diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 72a6966..c607606 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -124,7 +124,7 @@ -behaviour(gen_server). --include("../include/leveled.hrl"). +-include("include/leveled.hrl"). -export([init/1, handle_call/3, @@ -348,6 +348,17 @@ handle_call({return_folder, FolderType}, _From, State) -> State#state.ledger_cache, Bucket, ?RIAK_TAG), + State}; + {index_query, + Bucket, + {IdxField, StartValue, EndValue}, + {ReturnTerms, TermRegex}} -> + {reply, + index_query(State#state.penciller, + State#state.ledger_cache, + Bucket, + {IdxField, StartValue, EndValue}, + {ReturnTerms, TermRegex}), State} end; handle_call({compact_journal, Timeout}, _From, State) -> @@ -408,6 +419,41 @@ bucket_stats(Penciller, LedgerCache, Bucket, Tag) -> end, {async, Folder}. +index_query(Penciller, LedgerCache, + Bucket, + {IdxField, StartValue, EndValue}, + {ReturnTerms, TermRegex}) -> + PCLopts = #penciller_options{start_snapshot=true, + source_penciller=Penciller}, + {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), + Folder = fun() -> + Increment = gb_trees:to_list(LedgerCache), + io:format("Length of increment in snapshot is ~w~n", + [length(Increment)]), + ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, + {infinity, Increment}), + StartKey = leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, + IdxField, StartValue), + EndKey = leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, + IdxField, EndValue), + AddFun = case ReturnTerms of + true -> + fun add_terms/3; + _ -> + fun add_keys/3 + end, + AccFun = accumulate_index(TermRegex, AddFun), + Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, + StartKey, + EndKey, + AccFun, + []), + ok = leveled_penciller:pcl_close(LedgerSnapshot), + Acc + end, + {async, Folder}. + + shutdown_wait([], _Inker) -> false; shutdown_wait([TopPause|Rest], Inker) -> @@ -476,6 +522,47 @@ accumulate_size(Key, Value, {Size, Count}) -> {Size, Count} end. + +add_keys(ObjKey, _IdxValue, Acc) -> + Acc ++ [ObjKey]. + +add_terms(ObjKey, IdxValue, Acc) -> + Acc ++ [{IdxValue, ObjKey}]. + +accumulate_index(TermRe, AddFun) -> + case TermRe of + undefined -> + fun(Key, Value, Acc) -> + case leveled_codec:is_active(Key, Value) of + true -> + {_Bucket, + ObjKey, + IdxValue} = leveled_codec:from_ledgerkey(Key), + AddFun(ObjKey, IdxValue, Acc); + false -> + Acc + end end; + TermRe -> + fun(Key, Value, Acc) -> + case leveled_codec:is_active(Key, Value) of + true -> + {_Bucket, + ObjKey, + IdxValue} = leveled_codec:from_ledgerkey(Key), + case re:run(IdxValue, TermRe) of + nomatch -> + Acc; + _ -> + AddFun(ObjKey, IdxValue, Acc) + end; + false -> + Acc + end end + end. + + + + preparefor_ledgercache(PK, SQN, Obj, Size, IndexSpecs) -> {Bucket, Key, PrimaryChange} = leveled_codec:generate_ledgerkv(PK, SQN, diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index d368896..94d3a2f 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -44,7 +44,7 @@ -module(leveled_cdb). -behaviour(gen_server). --include("../include/leveled.hrl"). +-include("include/leveled.hrl"). -export([init/1, handle_call/3, diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index b8170c6..370133c 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -28,7 +28,7 @@ -module(leveled_codec). --include("../include/leveled.hrl"). +-include("include/leveled.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -43,14 +43,24 @@ key_dominates/2, print_key/1, to_ledgerkey/3, + to_ledgerkey/5, + from_ledgerkey/1, build_metadata_object/2, generate_ledgerkv/4, generate_ledgerkv/5, get_size/2, convert_indexspecs/4, - riakto_keydetails/1]). + riakto_keydetails/1, + generate_uuid/0]). +%% Credit to +%% https://github.com/afiskon/erlang-uuid-v4/blob/master/src/uuid.erl +generate_uuid() -> + <> = crypto:rand_bytes(16), + io_lib:format("~8.16.0b-~4.16.0b-4~3.16.0b-~4.16.0b-~12.16.0b", + [A, B, C band 16#0fff, D band 16#3fff bor 16#8000, E]). + strip_to_keyonly({keyonly, K}) -> K; strip_to_keyonly({K, _V}) -> K. @@ -87,6 +97,13 @@ is_active(Key, Value) -> false end. +from_ledgerkey({Tag, Bucket, {_IdxField, IdxValue}, Key}) + when Tag == ?IDX_TAG -> + {Bucket, Key, IdxValue}. + +to_ledgerkey(Bucket, Key, Tag, Field, Value) when Tag == ?IDX_TAG -> + {?IDX_TAG, Bucket, {Field, Value}, Key}. + to_ledgerkey(Bucket, Key, Tag) -> {Tag, Bucket, Key, null}. @@ -132,7 +149,7 @@ endkey_passed(EndKey, CheckingKey) -> EndKey < CheckingKey. convert_indexspecs(IndexSpecs, Bucket, Key, SQN) -> - lists:map(fun({IndexOp, IndexField, IndexValue}) -> + lists:map(fun({IndexOp, IdxField, IdxValue}) -> Status = case IndexOp of add -> %% TODO: timestamp support @@ -141,7 +158,8 @@ convert_indexspecs(IndexSpecs, Bucket, Key, SQN) -> %% TODO: timestamps for delayed reaping {tomb, infinity} end, - {{i, Bucket, {IndexField, IndexValue}, Key}, + {to_ledgerkey(Bucket, Key, ?IDX_TAG, + IdxField, IdxValue), {SQN, Status, null}} end, IndexSpecs). diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index 8b58c4b..ca1a6f3 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -4,7 +4,7 @@ -behaviour(gen_server). --include("../include/leveled.hrl"). +-include("include/leveled.hrl"). -export([init/1, handle_call/3, diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 55bb71b..a876fa8 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -76,7 +76,7 @@ -behaviour(gen_server). --include("../include/leveled.hrl"). +-include("include/leveled.hrl"). -export([init/1, handle_call/3, @@ -669,20 +669,14 @@ filepath(RootPath, journal_compact_dir) -> filepath(RootPath, NewSQN, new_journal) -> filename:join(filepath(RootPath, journal_dir), integer_to_list(NewSQN) ++ "_" - ++ generate_uuid() + ++ leveled_codec:generate_uuid() ++ "." ++ ?PENDING_FILEX); filepath(CompactFilePath, NewSQN, compact_journal) -> filename:join(CompactFilePath, integer_to_list(NewSQN) ++ "_" - ++ generate_uuid() + ++ leveled_codec:generate_uuid() ++ "." ++ ?PENDING_FILEX). -%% Credit to -%% https://github.com/afiskon/erlang-uuid-v4/blob/master/src/uuid.erl -generate_uuid() -> - <> = crypto:rand_bytes(16), - io_lib:format("~8.16.0b-~4.16.0b-4~3.16.0b-~4.16.0b-~12.16.0b", - [A, B, C band 16#0fff, D band 16#3fff bor 16#8000, E]). simple_manifest_reader(SQN, RootPath) -> ManifestPath = filepath(RootPath, manifest_dir), @@ -815,6 +809,9 @@ simple_inker_completeactivejournal_test() -> F2 = filename:join(JournalFP, "nursery_3.pnd"), {ok, PidW} = leveled_cdb:cdb_open_writer(F2), {ok, _F2} = leveled_cdb:cdb_complete(PidW), + F1 = filename:join(JournalFP, "nursery_1.cdb"), + F1r = filename:join(JournalFP, "nursery_1.pnd"), + ok = file:rename(F1, F1r), {ok, Ink1} = ink_start(#inker_options{root_path=RootPath, cdb_options=CDBopts}), Obj1 = ink_get(Ink1, "Key1", 1), diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 32d97ee..7ec6144 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -52,7 +52,7 @@ -behaviour(gen_server). --include("../include/leveled.hrl"). +-include("include/leveled.hrl"). -export([init/1, handle_call/3, diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 0ce0dd0..34a99a4 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -223,7 +223,7 @@ -behaviour(gen_server). --include("../include/leveled.hrl"). +-include("include/leveled.hrl"). -export([init/1, handle_call/3, diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 6df0b02..000dd45 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -143,7 +143,7 @@ -module(leveled_sft). -behaviour(gen_server). --include("../include/leveled.hrl"). +-include("include/leveled.hrl"). -export([init/1, handle_call/3, diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index cd5704f..bf5a700 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -7,7 +7,8 @@ journal_compaction/1, fetchput_snapshot/1, load_and_count/1, - load_and_count_withdelete/1]). + load_and_count_withdelete/1 + ]). all() -> [simple_put_fetch_head_delete, many_put_fetch_head, @@ -19,33 +20,33 @@ all() -> [simple_put_fetch_head_delete, simple_put_fetch_head_delete(_Config) -> - RootPath = reset_filestructure(), + RootPath = testutil:reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath}, {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), - {TestObject, TestSpec} = generate_testobject(), + {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), - check_bookie_forobject(Bookie1, TestObject), - check_bookie_formissingobject(Bookie1, "Bucket1", "Key2"), + testutil:check_forobject(Bookie1, TestObject), + testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"), ok = leveled_bookie:book_close(Bookie1), StartOpts2 = #bookie_options{root_path=RootPath, max_journalsize=3000000}, {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), - check_bookie_forobject(Bookie2, TestObject), - ObjList1 = generate_multiple_objects(5000, 2), + testutil:check_forobject(Bookie2, TestObject), + ObjList1 = testutil:generate_objects(5000, 2), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, ObjList1), ChkList1 = lists:sublist(lists:sort(ObjList1), 100), - check_bookie_forlist(Bookie2, ChkList1), - check_bookie_forobject(Bookie2, TestObject), - check_bookie_formissingobject(Bookie2, "Bucket1", "Key2"), + testutil:check_forlist(Bookie2, ChkList1), + testutil:check_forobject(Bookie2, TestObject), + testutil:check_formissingobject(Bookie2, "Bucket1", "Key2"), ok = leveled_bookie:book_put(Bookie2, "Bucket1", "Key2", "Value2", [{add, "Index1", "Term1"}]), {ok, "Value2"} = leveled_bookie:book_get(Bookie2, "Bucket1", "Key2"), {ok, {62888926, 43}} = leveled_bookie:book_head(Bookie2, "Bucket1", "Key2"), - check_bookie_formissingobject(Bookie2, "Bucket1", "Key2"), + testutil:check_formissingobject(Bookie2, "Bucket1", "Key2"), ok = leveled_bookie:book_put(Bookie2, "Bucket1", "Key2", <<"Value2">>, [{remove, "Index1", "Term1"}, {add, "Index1", <<"Term2">>}]), @@ -60,110 +61,111 @@ simple_put_fetch_head_delete(_Config) -> {ok, Bookie4} = leveled_bookie:book_start(StartOpts2), not_found = leveled_bookie:book_get(Bookie4, "Bucket1", "Key2"), ok = leveled_bookie:book_close(Bookie4), - reset_filestructure(). + testutil:reset_filestructure(). many_put_fetch_head(_Config) -> - RootPath = reset_filestructure(), + RootPath = testutil:reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath}, {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), - {TestObject, TestSpec} = generate_testobject(), + {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), - check_bookie_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject), ok = leveled_bookie:book_close(Bookie1), StartOpts2 = #bookie_options{root_path=RootPath, max_journalsize=1000000000}, {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), - check_bookie_forobject(Bookie2, TestObject), + testutil:check_forobject(Bookie2, TestObject), GenList = [2, 20002, 40002, 60002, 80002, 100002, 120002, 140002, 160002, 180002], - CLs = load_objects(20000, GenList, Bookie2, TestObject, - fun generate_multiple_smallobjects/2), + CLs = testutil:load_objects(20000, GenList, Bookie2, TestObject, + fun testutil:generate_smallobjects/2), CL1A = lists:nth(1, CLs), ChkListFixed = lists:nth(length(CLs), CLs), - check_bookie_forlist(Bookie2, CL1A), - ObjList2A = generate_multiple_objects(5000, 2), + testutil:check_forlist(Bookie2, CL1A), + ObjList2A = testutil:generate_objects(5000, 2), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, ObjList2A), ChkList2A = lists:sublist(lists:sort(ObjList2A), 1000), - check_bookie_forlist(Bookie2, ChkList2A), - check_bookie_forlist(Bookie2, ChkListFixed), - check_bookie_forobject(Bookie2, TestObject), - check_bookie_forlist(Bookie2, ChkList2A), - check_bookie_forlist(Bookie2, ChkListFixed), - check_bookie_forobject(Bookie2, TestObject), + testutil:check_forlist(Bookie2, ChkList2A), + testutil:check_forlist(Bookie2, ChkListFixed), + testutil:check_forobject(Bookie2, TestObject), + testutil:check_forlist(Bookie2, ChkList2A), + testutil:check_forlist(Bookie2, ChkListFixed), + testutil:check_forobject(Bookie2, TestObject), ok = leveled_bookie:book_close(Bookie2), {ok, Bookie3} = leveled_bookie:book_start(StartOpts2), - check_bookie_forlist(Bookie3, ChkList2A), - check_bookie_forobject(Bookie3, TestObject), + testutil:check_forlist(Bookie3, ChkList2A), + testutil:check_forobject(Bookie3, TestObject), ok = leveled_bookie:book_close(Bookie3), - reset_filestructure(). + testutil:reset_filestructure(). journal_compaction(_Config) -> - RootPath = reset_filestructure(), + RootPath = testutil:reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=4000000}, {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), - {TestObject, TestSpec} = generate_testobject(), + {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), - check_bookie_forobject(Bookie1, TestObject), - ObjList1 = generate_multiple_objects(5000, 2), + testutil:check_forobject(Bookie1, TestObject), + ObjList1 = testutil:generate_objects(5000, 2), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, ObjList1), ChkList1 = lists:sublist(lists:sort(ObjList1), 1000), - check_bookie_forlist(Bookie1, ChkList1), - check_bookie_forobject(Bookie1, TestObject), + testutil:check_forlist(Bookie1, ChkList1), + testutil:check_forobject(Bookie1, TestObject), {B2, K2, V2, Spec2, MD} = {"Bucket1", "Key1", "Value1", [], {"MDK1", "MDV1"}}, - {TestObject2, TestSpec2} = generate_testobject(B2, K2, V2, Spec2, MD), + {TestObject2, TestSpec2} = testutil:generate_testobject(B2, K2, + V2, Spec2, MD), ok = leveled_bookie:book_riakput(Bookie1, TestObject2, TestSpec2), ok = leveled_bookie:book_compactjournal(Bookie1, 30000), - check_bookie_forlist(Bookie1, ChkList1), - check_bookie_forobject(Bookie1, TestObject), - check_bookie_forobject(Bookie1, TestObject2), + testutil:check_forlist(Bookie1, ChkList1), + testutil:check_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject2), timer:sleep(5000), % Allow for compaction to complete io:format("Has journal completed?~n"), - check_bookie_forlist(Bookie1, ChkList1), - check_bookie_forobject(Bookie1, TestObject), - check_bookie_forobject(Bookie1, TestObject2), + testutil:check_forlist(Bookie1, ChkList1), + testutil:check_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject2), %% Now replace all the objects - ObjList2 = generate_multiple_objects(5000, 2), + ObjList2 = testutil:generate_objects(5000, 2), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, ObjList2), ok = leveled_bookie:book_compactjournal(Bookie1, 30000), ChkList3 = lists:sublist(lists:sort(ObjList2), 500), - check_bookie_forlist(Bookie1, ChkList3), + testutil:check_forlist(Bookie1, ChkList3), ok = leveled_bookie:book_close(Bookie1), % Restart {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), - check_bookie_forobject(Bookie2, TestObject), - check_bookie_forlist(Bookie2, ChkList3), + testutil:check_forobject(Bookie2, TestObject), + testutil:check_forlist(Bookie2, ChkList3), ok = leveled_bookie:book_close(Bookie2), - reset_filestructure(). + testutil:reset_filestructure(). fetchput_snapshot(_Config) -> - RootPath = reset_filestructure(), + RootPath = testutil:reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=3000000}, {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), - {TestObject, TestSpec} = generate_testobject(), + {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), - ObjList1 = generate_multiple_objects(5000, 2), + ObjList1 = testutil:generate_objects(5000, 2), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, ObjList1), SnapOpts1 = #bookie_options{snapshot_bookie=Bookie1}, {ok, SnapBookie1} = leveled_bookie:book_start(SnapOpts1), ChkList1 = lists:sublist(lists:sort(ObjList1), 100), - check_bookie_forlist(Bookie1, ChkList1), - check_bookie_forlist(SnapBookie1, ChkList1), + testutil:check_forlist(Bookie1, ChkList1), + testutil:check_forlist(SnapBookie1, ChkList1), ok = leveled_bookie:book_close(SnapBookie1), - check_bookie_forlist(Bookie1, ChkList1), + testutil:check_forlist(Bookie1, ChkList1), ok = leveled_bookie:book_close(Bookie1), io:format("Closed initial bookies~n"), @@ -172,89 +174,94 @@ fetchput_snapshot(_Config) -> {ok, SnapBookie2} = leveled_bookie:book_start(SnapOpts2), io:format("Bookies restarted~n"), - check_bookie_forlist(Bookie2, ChkList1), + testutil:check_forlist(Bookie2, ChkList1), io:format("Check active bookie still contains original data~n"), - check_bookie_forlist(SnapBookie2, ChkList1), + testutil:check_forlist(SnapBookie2, ChkList1), io:format("Check snapshot still contains original data~n"), - ObjList2 = generate_multiple_objects(5000, 2), + ObjList2 = testutil:generate_objects(5000, 2), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, ObjList2), io:format("Replacement objects put~n"), ChkList2 = lists:sublist(lists:sort(ObjList2), 100), - check_bookie_forlist(Bookie2, ChkList2), - check_bookie_forlist(SnapBookie2, ChkList1), + testutil:check_forlist(Bookie2, ChkList2), + testutil:check_forlist(SnapBookie2, ChkList1), io:format("Checked for replacement objects in active bookie" ++ ", old objects in snapshot~n"), {ok, FNsA} = file:list_dir(RootPath ++ "/ledger/ledger_files"), - ObjList3 = generate_multiple_objects(15000, 5002), + ObjList3 = testutil:generate_objects(15000, 5002), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, ObjList3), ChkList3 = lists:sublist(lists:sort(ObjList3), 100), - check_bookie_forlist(Bookie2, ChkList3), - check_bookie_formissinglist(SnapBookie2, ChkList3), + testutil:check_forlist(Bookie2, ChkList3), + testutil:check_formissinglist(SnapBookie2, ChkList3), GenList = [20002, 40002, 60002, 80002, 100002, 120002], - CLs2 = load_objects(20000, GenList, Bookie2, TestObject, - fun generate_multiple_smallobjects/2), + CLs2 = testutil:load_objects(20000, GenList, Bookie2, TestObject, + fun testutil:generate_smallobjects/2), io:format("Loaded significant numbers of new objects~n"), - check_bookie_forlist(Bookie2, lists:nth(length(CLs2), CLs2)), + testutil:check_forlist(Bookie2, lists:nth(length(CLs2), CLs2)), io:format("Checked active bookie has new objects~n"), {ok, SnapBookie3} = leveled_bookie:book_start(SnapOpts2), - check_bookie_forlist(SnapBookie3, lists:nth(length(CLs2), CLs2)), - check_bookie_formissinglist(SnapBookie2, ChkList3), - check_bookie_formissinglist(SnapBookie2, lists:nth(length(CLs2), CLs2)), - check_bookie_forlist(SnapBookie3, ChkList2), - check_bookie_forlist(SnapBookie2, ChkList1), + testutil:check_forlist(SnapBookie3, lists:nth(length(CLs2), CLs2)), + testutil:check_formissinglist(SnapBookie2, ChkList3), + testutil:check_formissinglist(SnapBookie2, lists:nth(length(CLs2), CLs2)), + testutil:check_forlist(SnapBookie3, ChkList2), + testutil:check_forlist(SnapBookie2, ChkList1), io:format("Started new snapshot and check for new objects~n"), - CLs3 = load_objects(20000, GenList, Bookie2, TestObject, - fun generate_multiple_smallobjects/2), - check_bookie_forlist(Bookie2, lists:nth(length(CLs3), CLs3)), - check_bookie_forlist(Bookie2, lists:nth(1, CLs3)), + CLs3 = testutil:load_objects(20000, GenList, Bookie2, TestObject, + fun testutil:generate_smallobjects/2), + testutil:check_forlist(Bookie2, lists:nth(length(CLs3), CLs3)), + testutil:check_forlist(Bookie2, lists:nth(1, CLs3)), {ok, FNsB} = file:list_dir(RootPath ++ "/ledger/ledger_files"), ok = leveled_bookie:book_close(SnapBookie2), - check_bookie_forlist(Bookie2, lists:nth(length(CLs3), CLs3)), + testutil:check_forlist(Bookie2, lists:nth(length(CLs3), CLs3)), ok = leveled_bookie:book_close(SnapBookie3), - check_bookie_forlist(Bookie2, lists:nth(length(CLs3), CLs3)), - check_bookie_forlist(Bookie2, lists:nth(1, CLs3)), + testutil:check_forlist(Bookie2, lists:nth(length(CLs3), CLs3)), + testutil:check_forlist(Bookie2, lists:nth(1, CLs3)), timer:sleep(90000), {ok, FNsC} = file:list_dir(RootPath ++ "/ledger/ledger_files"), true = length(FNsB) > length(FNsA), true = length(FNsB) > length(FNsC), - {B1Size, B1Count} = check_bucket_stats(Bookie2, "Bucket1"), + {B1Size, B1Count} = testutil:check_bucket_stats(Bookie2, "Bucket1"), true = B1Size > 0, true = B1Count == 1, - {B1Size, B1Count} = check_bucket_stats(Bookie2, "Bucket1"), - {BSize, BCount} = check_bucket_stats(Bookie2, "Bucket"), + {B1Size, B1Count} = testutil:check_bucket_stats(Bookie2, "Bucket1"), + {BSize, BCount} = testutil:check_bucket_stats(Bookie2, "Bucket"), true = BSize > 0, true = BCount == 140000, ok = leveled_bookie:book_close(Bookie2), - reset_filestructure(). + testutil:reset_filestructure(). load_and_count(_Config) -> % Use artificially small files, and the load keys, counting they're all % present - RootPath = reset_filestructure(), + RootPath = testutil:reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=50000000}, {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), - {TestObject, TestSpec} = generate_testobject(), + {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), - check_bookie_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject), io:format("Loading initial small objects~n"), + G1 = fun testutil:generate_smallobjects/2, lists:foldl(fun(_X, Acc) -> - load_objects(5000, [Acc + 2], Bookie1, TestObject, - fun generate_multiple_smallobjects/2), - {_Size, Count} = check_bucket_stats(Bookie1, "Bucket"), + testutil:load_objects(5000, + [Acc + 2], + Bookie1, + TestObject, + G1), + {_S, Count} = testutil:check_bucket_stats(Bookie1, + "Bucket"), if Acc + 5000 == Count -> ok @@ -262,12 +269,17 @@ load_and_count(_Config) -> Acc + 5000 end, 0, lists:seq(1, 20)), - check_bookie_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject), io:format("Loading larger compressible objects~n"), + G2 = fun testutil:generate_compressibleobjects/2, lists:foldl(fun(_X, Acc) -> - load_objects(5000, [Acc + 2], Bookie1, TestObject, - fun generate_multiple_compressibleobjects/2), - {_Size, Count} = check_bucket_stats(Bookie1, "Bucket"), + testutil:load_objects(5000, + [Acc + 2], + Bookie1, + TestObject, + G2), + {_S, Count} = testutil:check_bucket_stats(Bookie1, + "Bucket"), if Acc + 5000 == Count -> ok @@ -275,12 +287,16 @@ load_and_count(_Config) -> Acc + 5000 end, 100000, lists:seq(1, 20)), - check_bookie_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject), io:format("Replacing small objects~n"), lists:foldl(fun(_X, Acc) -> - load_objects(5000, [Acc + 2], Bookie1, TestObject, - fun generate_multiple_smallobjects/2), - {_Size, Count} = check_bucket_stats(Bookie1, "Bucket"), + testutil:load_objects(5000, + [Acc + 2], + Bookie1, + TestObject, + G1), + {_S, Count} = testutil:check_bucket_stats(Bookie1, + "Bucket"), if Count == 200000 -> ok @@ -288,12 +304,16 @@ load_and_count(_Config) -> Acc + 5000 end, 0, lists:seq(1, 20)), - check_bookie_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject), io:format("Loading more small objects~n"), lists:foldl(fun(_X, Acc) -> - load_objects(5000, [Acc + 2], Bookie1, TestObject, - fun generate_multiple_compressibleobjects/2), - {_Size, Count} = check_bucket_stats(Bookie1, "Bucket"), + testutil:load_objects(5000, + [Acc + 2], + Bookie1, + TestObject, + G2), + {_S, Count} = testutil:check_bucket_stats(Bookie1, + "Bucket"), if Acc + 5000 == Count -> ok @@ -301,25 +321,30 @@ load_and_count(_Config) -> Acc + 5000 end, 200000, lists:seq(1, 20)), - check_bookie_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject), ok = leveled_bookie:book_close(Bookie1), {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), - {_BSize, 300000} = check_bucket_stats(Bookie2, "Bucket"), + {_, 300000} = testutil:check_bucket_stats(Bookie2, "Bucket"), ok = leveled_bookie:book_close(Bookie2), - reset_filestructure(). + testutil:reset_filestructure(). load_and_count_withdelete(_Config) -> - RootPath = reset_filestructure(), + RootPath = testutil:reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=50000000}, {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), - {TestObject, TestSpec} = generate_testobject(), + {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), - check_bookie_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject), io:format("Loading initial small objects~n"), + G1 = fun testutil:generate_smallobjects/2, lists:foldl(fun(_X, Acc) -> - load_objects(5000, [Acc + 2], Bookie1, TestObject, - fun generate_multiple_smallobjects/2), - {_Size, Count} = check_bucket_stats(Bookie1, "Bucket"), + testutil:load_objects(5000, + [Acc + 2], + Bookie1, + TestObject, + G1), + {_S, Count} = testutil:check_bucket_stats(Bookie1, + "Bucket"), if Acc + 5000 == Count -> ok @@ -327,17 +352,22 @@ load_and_count_withdelete(_Config) -> Acc + 5000 end, 0, lists:seq(1, 20)), - check_bookie_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject), {BucketD, KeyD} = leveled_codec:riakto_keydetails(TestObject), - {_, 1} = check_bucket_stats(Bookie1, BucketD), + {_, 1} = testutil:check_bucket_stats(Bookie1, BucketD), ok = leveled_bookie:book_riakdelete(Bookie1, BucketD, KeyD, []), not_found = leveled_bookie:book_riakget(Bookie1, BucketD, KeyD), - {_, 0} = check_bucket_stats(Bookie1, BucketD), + {_, 0} = testutil:check_bucket_stats(Bookie1, BucketD), io:format("Loading larger compressible objects~n"), + G2 = fun testutil:generate_compressibleobjects/2, lists:foldl(fun(_X, Acc) -> - load_objects(5000, [Acc + 2], Bookie1, no_check, - fun generate_multiple_compressibleobjects/2), - {_Size, Count} = check_bucket_stats(Bookie1, "Bucket"), + testutil:load_objects(5000, + [Acc + 2], + Bookie1, + no_check, + G2), + {_S, Count} = testutil:check_bucket_stats(Bookie1, + "Bucket"), if Acc + 5000 == Count -> ok @@ -348,160 +378,8 @@ load_and_count_withdelete(_Config) -> not_found = leveled_bookie:book_riakget(Bookie1, BucketD, KeyD), ok = leveled_bookie:book_close(Bookie1), {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), - check_bookie_formissingobject(Bookie2, BucketD, KeyD), - {_BSize, 0} = check_bucket_stats(Bookie2, BucketD), - ok = leveled_bookie:book_close(Bookie2). + testutil:check_formissingobject(Bookie2, BucketD, KeyD), + {_BSize, 0} = testutil:check_bucket_stats(Bookie2, BucketD), + ok = leveled_bookie:book_close(Bookie2), + testutil:reset_filestructure(). - -reset_filestructure() -> - RootPath = "test", - filelib:ensure_dir(RootPath ++ "/journal/"), - filelib:ensure_dir(RootPath ++ "/ledger/"), - leveled_inker:clean_testdir(RootPath ++ "/journal"), - leveled_penciller:clean_testdir(RootPath ++ "/ledger"), - RootPath. - - - -check_bucket_stats(Bookie, Bucket) -> - FoldSW1 = os:timestamp(), - io:format("Checking bucket size~n"), - {async, Folder1} = leveled_bookie:book_returnfolder(Bookie, - {riakbucket_stats, - Bucket}), - {B1Size, B1Count} = Folder1(), - io:format("Bucket fold completed in ~w microseconds~n", - [timer:now_diff(os:timestamp(), FoldSW1)]), - io:format("Bucket ~s has size ~w and count ~w~n", - [Bucket, B1Size, B1Count]), - {B1Size, B1Count}. - - -check_bookie_forlist(Bookie, ChkList) -> - check_bookie_forlist(Bookie, ChkList, false). - -check_bookie_forlist(Bookie, ChkList, Log) -> - SW = os:timestamp(), - lists:foreach(fun({_RN, Obj, _Spc}) -> - if - Log == true -> - io:format("Fetching Key ~w~n", [Obj#r_object.key]); - true -> - ok - end, - R = leveled_bookie:book_riakget(Bookie, - Obj#r_object.bucket, - Obj#r_object.key), - R = {ok, Obj} end, - ChkList), - io:format("Fetch check took ~w microseconds checking list of length ~w~n", - [timer:now_diff(os:timestamp(), SW), length(ChkList)]). - -check_bookie_formissinglist(Bookie, ChkList) -> - SW = os:timestamp(), - lists:foreach(fun({_RN, Obj, _Spc}) -> - R = leveled_bookie:book_riakget(Bookie, - Obj#r_object.bucket, - Obj#r_object.key), - R = not_found end, - ChkList), - io:format("Miss check took ~w microseconds checking list of length ~w~n", - [timer:now_diff(os:timestamp(), SW), length(ChkList)]). - -check_bookie_forobject(Bookie, TestObject) -> - {ok, TestObject} = leveled_bookie:book_riakget(Bookie, - TestObject#r_object.bucket, - TestObject#r_object.key), - {ok, HeadObject} = leveled_bookie:book_riakhead(Bookie, - TestObject#r_object.bucket, - TestObject#r_object.key), - ok = case {HeadObject#r_object.bucket, - HeadObject#r_object.key, - HeadObject#r_object.vclock} of - {B1, K1, VC1} when B1 == TestObject#r_object.bucket, - K1 == TestObject#r_object.key, - VC1 == TestObject#r_object.vclock -> - ok - end. - -check_bookie_formissingobject(Bookie, Bucket, Key) -> - not_found = leveled_bookie:book_riakget(Bookie, Bucket, Key), - not_found = leveled_bookie:book_riakhead(Bookie, Bucket, Key). - - -generate_testobject() -> - {B1, K1, V1, Spec1, MD} = {"Bucket1", - "Key1", - "Value1", - [], - {"MDK1", "MDV1"}}, - generate_testobject(B1, K1, V1, Spec1, MD). - -generate_testobject(B, K, V, Spec, MD) -> - Content = #r_content{metadata=MD, value=V}, - {#r_object{bucket=B, key=K, contents=[Content], vclock=[{'a',1}]}, - Spec}. - - -generate_multiple_compressibleobjects(Count, KeyNumber) -> - S1 = "111111111111111", - S2 = "222222222222222", - S3 = "333333333333333", - S4 = "aaaaaaaaaaaaaaa", - S5 = "AAAAAAAAAAAAAAA", - S6 = "GGGGGGGGGGGGGGG", - S7 = "===============", - S8 = "...............", - Selector = [{1, S1}, {2, S2}, {3, S3}, {4, S4}, - {5, S5}, {6, S6}, {7, S7}, {8, S8}], - L = lists:seq(1, 1024), - V = lists:foldl(fun(_X, Acc) -> - {_, Str} = lists:keyfind(random:uniform(8), 1, Selector), - Acc ++ Str end, - "", - L), - generate_multiple_objects(Count, KeyNumber, [], V). - -generate_multiple_smallobjects(Count, KeyNumber) -> - generate_multiple_objects(Count, KeyNumber, [], crypto:rand_bytes(512)). - -generate_multiple_objects(Count, KeyNumber) -> - generate_multiple_objects(Count, KeyNumber, [], crypto:rand_bytes(4096)). - -generate_multiple_objects(0, _KeyNumber, ObjL, _Value) -> - ObjL; -generate_multiple_objects(Count, KeyNumber, ObjL, Value) -> - Obj = {"Bucket", - "Key" ++ integer_to_list(KeyNumber), - Value, - [], - [{"MDK", "MDV" ++ integer_to_list(KeyNumber)}, - {"MDK2", "MDV" ++ integer_to_list(KeyNumber)}]}, - {B1, K1, V1, Spec1, MD} = Obj, - Content = #r_content{metadata=MD, value=V1}, - Obj1 = #r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]}, - generate_multiple_objects(Count - 1, - KeyNumber + 1, - ObjL ++ [{random:uniform(), Obj1, Spec1}], - Value). - - -load_objects(ChunkSize, GenList, Bookie, TestObject, Generator) -> - lists:map(fun(KN) -> - ObjListA = Generator(ChunkSize, KN), - StartWatchA = os:timestamp(), - lists:foreach(fun({_RN, Obj, Spc}) -> - leveled_bookie:book_riakput(Bookie, Obj, Spc) - end, - ObjListA), - Time = timer:now_diff(os:timestamp(), StartWatchA), - io:format("~w objects loaded in ~w seconds~n", - [ChunkSize, Time/1000000]), - if - TestObject == no_check -> - ok; - true -> - check_bookie_forobject(Bookie, TestObject) - end, - lists:sublist(ObjListA, 1000) end, - GenList). diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl new file mode 100644 index 0000000..a7a9fd6 --- /dev/null +++ b/test/end_to_end/iterator_SUITE.erl @@ -0,0 +1,182 @@ +-module(iterator_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include("include/leveled.hrl"). + +-export([all/0]). +-export([simple_load_with2i/1, + simple_querycount/1]). + +all() -> [simple_load_with2i, + simple_querycount]. + + +simple_load_with2i(_Config) -> + RootPath = testutil:reset_filestructure(), + StartOpts1 = #bookie_options{root_path=RootPath, + max_journalsize=50000000}, + {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), + {TestObject, TestSpec} = testutil:generate_testobject(), + ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), + testutil:check_forobject(Bookie1, TestObject), + testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"), + testutil:check_forobject(Bookie1, TestObject), + ObjL1 = testutil:generate_objects(10000, + uuid, + [], + testutil:get_compressiblevalue(), + testutil:get_randomindexes_generator(8)), + lists:foreach(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, + ObjL1), + ChkList1 = lists:sublist(lists:sort(ObjL1), 100), + testutil:check_forlist(Bookie1, ChkList1), + testutil:check_forobject(Bookie1, TestObject), + ok = leveled_bookie:book_close(Bookie1), + testutil:reset_filestructure(). + + +simple_querycount(_Config) -> + RootPath = testutil:reset_filestructure(), + StartOpts1 = #bookie_options{root_path=RootPath, + max_journalsize=50000000}, + {ok, Book1} = leveled_bookie:book_start(StartOpts1), + {TestObject, TestSpec} = testutil:generate_testobject(), + ok = leveled_bookie:book_riakput(Book1, TestObject, TestSpec), + testutil:check_forobject(Book1, TestObject), + testutil:check_formissingobject(Book1, "Bucket1", "Key2"), + testutil:check_forobject(Book1, TestObject), + lists:foreach(fun(_X) -> + V = testutil:get_compressiblevalue(), + Indexes = testutil:get_randomindexes_generator(8), + SW = os:timestamp(), + ObjL1 = testutil:generate_objects(10000, + uuid, + [], + V, + Indexes), + lists:foreach(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Book1, + Obj, + Spc) + end, + ObjL1), + io:format("Put of 10000 objects with 8 index entries " + ++ + "each completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(), SW)]) + end, + lists:seq(1, 8)), + testutil:check_forobject(Book1, TestObject), + Total = lists:foldl(fun(X, Acc) -> + IdxF = "idx" ++ integer_to_list(X) ++ "_bin", + T = count_termsonindex("Bucket", + IdxF, + Book1, + {false, undefined}), + io:format("~w terms found on index ~s~n", + [T, IdxF]), + Acc + T + end, + 0, + lists:seq(1, 8)), + ok = case Total of + 640000 -> + ok + end, + Index1Count = count_termsonindex("Bucket", + "idx1_bin", + Book1, + {false, undefined}), + ok = leveled_bookie:book_close(Book1), + {ok, Book2} = leveled_bookie:book_start(StartOpts1), + Index1Count = count_termsonindex("Bucket", + "idx1_bin", + Book2, + {false, undefined}), + NameList = testutil:name_list(), + TotalNameByName = lists:foldl(fun({_X, Name}, Acc) -> + {ok, Regex} = re:compile("[0-9]+" ++ + Name), + SW = os:timestamp(), + T = count_termsonindex("Bucket", + "idx1_bin", + Book2, + {false, + Regex}), + TD = timer:now_diff(os:timestamp(), + SW), + io:format("~w terms found on " ++ + "index idx1 with a " ++ + "regex in ~w " ++ + "microseconds~n", + [T, TD]), + Acc + T + end, + 0, + NameList), + ok = case TotalNameByName of + Index1Count -> + ok + end, + RegMia = re:compile("[0-9]+Mia"), + {async, + Mia2KFolder1} = leveled_bookie:book_returnfolder(Book2, + {index_query, + "Bucket", + {"idx2_bin", + "2000L", + "2000N~"}, + {false, + RegMia}}), + Mia2000Count1 = length(Mia2KFolder1()), + {async, + Mia2KFolder2} = leveled_bookie:book_returnfolder(Book2, + {index_query, + "Bucket", + {"idx2_bin", + "2000Ma", + "2000Mz"}, + {true, + undefined}}), + Mia2000Count2 = lists:foldl(fun({Term, _Key}, Acc) -> + case Term of + "2000Mia" -> + Acc + 1; + _ -> + Acc + end end, + 0, + Mia2KFolder2()), + ok = case Mia2000Count2 of + Mia2000Count1 -> + ok + end, + ok = leveled_bookie:book_close(Book2), + testutil:reset_filestructure(). + + + +count_termsonindex(Bucket, IdxField, Book, QType) -> + lists:foldl(fun(X, Acc) -> + SW = os:timestamp(), + ST = integer_to_list(X), + ET = ST ++ "~", + R = leveled_bookie:book_returnfolder(Book, + {index_query, + Bucket, + {IdxField, + ST, + ET}, + QType}), + {async, Folder} = R, + Items = length(Folder()), + io:format("2i query from term ~s on index ~s took " ++ + "~w microseconds~n", + [ST, + IdxField, + timer:now_diff(os:timestamp(), SW)]), + Acc + Items + end, + 0, + lists:seq(1901, 2218)). diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl new file mode 100644 index 0000000..0d472cc --- /dev/null +++ b/test/end_to_end/testutil.erl @@ -0,0 +1,232 @@ +-module(testutil). + +-include("../include/leveled.hrl"). + +-export([reset_filestructure/0, + check_bucket_stats/2, + check_forlist/2, + check_forlist/3, + check_formissinglist/2, + check_forobject/2, + check_formissingobject/3, + generate_testobject/0, + generate_testobject/5, + generate_compressibleobjects/2, + generate_smallobjects/2, + generate_objects/2, + generate_objects/5, + get_compressiblevalue/0, + get_randomindexes_generator/1, + name_list/0, + load_objects/5]). + + +reset_filestructure() -> + RootPath = "test", + filelib:ensure_dir(RootPath ++ "/journal/"), + filelib:ensure_dir(RootPath ++ "/ledger/"), + leveled_inker:clean_testdir(RootPath ++ "/journal"), + leveled_penciller:clean_testdir(RootPath ++ "/ledger"), + RootPath. + + + +check_bucket_stats(Bookie, Bucket) -> + FoldSW1 = os:timestamp(), + io:format("Checking bucket size~n"), + {async, Folder1} = leveled_bookie:book_returnfolder(Bookie, + {riakbucket_stats, + Bucket}), + {B1Size, B1Count} = Folder1(), + io:format("Bucket fold completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(), FoldSW1)]), + io:format("Bucket ~s has size ~w and count ~w~n", + [Bucket, B1Size, B1Count]), + {B1Size, B1Count}. + + +check_forlist(Bookie, ChkList) -> + check_forlist(Bookie, ChkList, false). + +check_forlist(Bookie, ChkList, Log) -> + SW = os:timestamp(), + lists:foreach(fun({_RN, Obj, _Spc}) -> + if + Log == true -> + io:format("Fetching Key ~w~n", [Obj#r_object.key]); + true -> + ok + end, + R = leveled_bookie:book_riakget(Bookie, + Obj#r_object.bucket, + Obj#r_object.key), + R = {ok, Obj} end, + ChkList), + io:format("Fetch check took ~w microseconds checking list of length ~w~n", + [timer:now_diff(os:timestamp(), SW), length(ChkList)]). + +check_formissinglist(Bookie, ChkList) -> + SW = os:timestamp(), + lists:foreach(fun({_RN, Obj, _Spc}) -> + R = leveled_bookie:book_riakget(Bookie, + Obj#r_object.bucket, + Obj#r_object.key), + R = not_found end, + ChkList), + io:format("Miss check took ~w microseconds checking list of length ~w~n", + [timer:now_diff(os:timestamp(), SW), length(ChkList)]). + +check_forobject(Bookie, TestObject) -> + {ok, TestObject} = leveled_bookie:book_riakget(Bookie, + TestObject#r_object.bucket, + TestObject#r_object.key), + {ok, HeadObject} = leveled_bookie:book_riakhead(Bookie, + TestObject#r_object.bucket, + TestObject#r_object.key), + ok = case {HeadObject#r_object.bucket, + HeadObject#r_object.key, + HeadObject#r_object.vclock} of + {B1, K1, VC1} when B1 == TestObject#r_object.bucket, + K1 == TestObject#r_object.key, + VC1 == TestObject#r_object.vclock -> + ok + end. + +check_formissingobject(Bookie, Bucket, Key) -> + not_found = leveled_bookie:book_riakget(Bookie, Bucket, Key), + not_found = leveled_bookie:book_riakhead(Bookie, Bucket, Key). + + +generate_testobject() -> + {B1, K1, V1, Spec1, MD} = {"Bucket1", + "Key1", + "Value1", + [], + {"MDK1", "MDV1"}}, + generate_testobject(B1, K1, V1, Spec1, MD). + +generate_testobject(B, K, V, Spec, MD) -> + Content = #r_content{metadata=MD, value=V}, + {#r_object{bucket=B, key=K, contents=[Content], vclock=[{'a',1}]}, + Spec}. + + +generate_compressibleobjects(Count, KeyNumber) -> + V = get_compressiblevalue(), + generate_objects(Count, KeyNumber, [], V). + + +get_compressiblevalue() -> + S1 = "111111111111111", + S2 = "222222222222222", + S3 = "333333333333333", + S4 = "aaaaaaaaaaaaaaa", + S5 = "AAAAAAAAAAAAAAA", + S6 = "GGGGGGGGGGGGGGG", + S7 = "===============", + S8 = "...............", + Selector = [{1, S1}, {2, S2}, {3, S3}, {4, S4}, + {5, S5}, {6, S6}, {7, S7}, {8, S8}], + L = lists:seq(1, 1024), + lists:foldl(fun(_X, Acc) -> + {_, Str} = lists:keyfind(random:uniform(8), 1, Selector), + Acc ++ Str end, + "", + L). + +generate_smallobjects(Count, KeyNumber) -> + generate_objects(Count, KeyNumber, [], crypto:rand_bytes(512)). + +generate_objects(Count, KeyNumber) -> + generate_objects(Count, KeyNumber, [], crypto:rand_bytes(4096)). + + +generate_objects(Count, KeyNumber, ObjL, Value) -> + generate_objects(Count, KeyNumber, ObjL, Value, fun() -> [] end). + +generate_objects(0, _KeyNumber, ObjL, _Value, _IndexGen) -> + ObjL; +generate_objects(Count, uuid, ObjL, Value, IndexGen) -> + {Obj1, Spec1} = set_object(leveled_codec:generate_uuid(), + Value, + IndexGen), + generate_objects(Count - 1, + uuid, + ObjL ++ [{random:uniform(), Obj1, Spec1}], + Value, + IndexGen); +generate_objects(Count, KeyNumber, ObjL, Value, IndexGen) -> + {Obj1, Spec1} = set_object("Key" ++ integer_to_list(KeyNumber), + Value, + IndexGen), + generate_objects(Count - 1, + KeyNumber + 1, + ObjL ++ [{random:uniform(), Obj1, Spec1}], + Value, + IndexGen). + +set_object(Key, Value, IndexGen) -> + Obj = {"Bucket", + Key, + Value, + IndexGen(), + [{"MDK", "MDV" ++ Key}, + {"MDK2", "MDV" ++ Key}]}, + {B1, K1, V1, Spec1, MD} = Obj, + Content = #r_content{metadata=MD, value=V1}, + {#r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]}, + Spec1}. + + +load_objects(ChunkSize, GenList, Bookie, TestObject, Generator) -> + lists:map(fun(KN) -> + ObjListA = Generator(ChunkSize, KN), + StartWatchA = os:timestamp(), + lists:foreach(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Bookie, Obj, Spc) + end, + ObjListA), + Time = timer:now_diff(os:timestamp(), StartWatchA), + io:format("~w objects loaded in ~w seconds~n", + [ChunkSize, Time/1000000]), + if + TestObject == no_check -> + ok; + true -> + check_forobject(Bookie, TestObject) + end, + lists:sublist(ObjListA, 1000) end, + GenList). + + +get_randomindexes_generator(Count) -> + Generator = fun() -> + lists:map(fun(X) -> + {add, + "idx" ++ integer_to_list(X) ++ "_bin", + get_randomdate() ++ get_randomname()} end, + lists:seq(1, Count)) + end, + Generator. + +name_list() -> + [{1, "Sophia"}, {2, "Emma"}, {3, "Olivia"}, {4, "Ava"}, + {5, "Isabella"}, {6, "Mia"}, {7, "Zoe"}, {8, "Lily"}, + {9, "Emily"}, {10, "Madelyn"}, {11, "Madison"}, {12, "Chloe"}, + {13, "Charlotte"}, {14, "Aubrey"}, {15, "Avery"}, + {16, "Abigail"}]. + +get_randomname() -> + NameList = name_list(), + N = random:uniform(16), + {N, Name} = lists:keyfind(N, 1, NameList), + Name. + +get_randomdate() -> + LowTime = 60000000000, + HighTime = 70000000000, + RandPoint = LowTime + random:uniform(HighTime - LowTime), + Date = calendar:gregorian_seconds_to_datetime(RandPoint), + {{Year, Month, Day}, {Hour, Minute, Second}} = Date, + lists:flatten(io_lib:format("~4..0w~2..0w~2..0w~2..0w~2..0w~2..0w", + [Year, Month, Day, Hour, Minute, Second])). \ No newline at end of file