diff --git a/test/end_to_end/tictac_SUITE.erl b/test/end_to_end/tictac_SUITE.erl index 7e4d7a0..39caf5d 100644 --- a/test/end_to_end/tictac_SUITE.erl +++ b/test/end_to_end/tictac_SUITE.erl @@ -21,6 +21,8 @@ all() -> [ ]. -define(LMD_FORMAT, "~4..0w~2..0w~2..0w~2..0w~2..0w"). +-define(V1_VERS, 1). +-define(MAGIC, 53). % riak_kv -> riak_object many_put_compare(_Config) -> TreeSize = small, @@ -31,7 +33,7 @@ many_put_compare(_Config) -> RootPathB = testutil:reset_filestructure("testB"), RootPathC = testutil:reset_filestructure("testC"), RootPathD = testutil:reset_filestructure("testD"), - + % Start the first database, load a test object, close it, start it again StartOpts1 = [{root_path, RootPathA}, {max_pencillercachesize, 16000}, @@ -52,11 +54,11 @@ many_put_compare(_Config) -> {sync_strategy, testutil:sync_strategy()}], {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), testutil:check_forobject(Bookie2, TestObject), - + % Generate 200K objects to be sued within the test, and load them into % the first store (outputting the generated objects as a list of lists) % to be used elsewhere - + GenList = [2, 20002, 40002, 60002, 80002, 100002, 120002, 140002, 160002, 180002], CLs = testutil:load_objects(20000, @@ -65,20 +67,20 @@ many_put_compare(_Config) -> TestObject, fun testutil:generate_smallobjects/2, 20000), - + % Start a new store, and load the same objects (except fot the original % test object) into this store - + StartOpts3 = [{root_path, RootPathB}, {max_journalsize, 200000000}, {max_pencillercachesize, 16000}, {sync_strategy, testutil:sync_strategy()}], {ok, Bookie3} = leveled_bookie:book_start(StartOpts3), lists:foreach(fun(ObjL) -> testutil:riakload(Bookie3, ObjL) end, CLs), - - % Now run a tictac query against both stores to see th extent to which + + % Now run a tictac query against both stores to see the extent to which % state between stores is consistent - + TicTacQ = {tictactree_obj, {o_rkv, "Bucket", null, null, false}, TreeSize, @@ -99,15 +101,45 @@ many_put_compare(_Config) -> [timer:now_diff(os:timestamp(), SWC0)]), io:format("Tree comparison shows ~w different leaves~n", [length(SegList0)]), - AltList = leveled_tictac:find_dirtyleaves(TreeA, - leveled_tictac:new_tree(0)), + AltList = + leveled_tictac:find_dirtyleaves(TreeA, + leveled_tictac:new_tree(0, TreeSize)), io:format("Tree comparison shows ~w altered leaves~n", [length(AltList)]), - true = length(SegList0) == 1, + true = length(SegList0) == 1, % only the test object should be different - true = length(AltList) > 10000, + true = length(AltList) > 10000, % check there are a significant number of differences from empty - + + % Now run the same query by putting the tree-building responsibility onto + % the fold_objects_fun + HashFun = + fun(_Key, Value) -> + {proxy_object, HeadBin, _Size, _FetchFun} = binary_to_term(Value), + <> = HeadBin, + <> = Rest, + erlang:phash2(lists:sort(binary_to_term(VclockBin))) + end, + FoldObjectsFun = + fun(_Bucket, Key, Value, Acc) -> + leveled_tictac:add_kv(Acc, Key, Value, HashFun) + end, + FoldQ = {foldheads_bybucket, + o_rkv, + "Bucket", + {FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)}}, + {async, TreeAObjFolder} = leveled_bookie:book_returnfolder(Bookie2, FoldQ), + SWB0Obj = os:timestamp(), + TreeAObj = TreeAObjFolder(), + io:format("Build tictac tree via object foldwith 200K objects in ~w~n", + [timer:now_diff(os:timestamp(), SWB0Obj)]), + SegList0Obj = leveled_tictac:find_dirtyleaves(TreeA, TreeAObj), + io:format("Fold object compared with tictac fold has ~w diffs~n", + [length(SegList0Obj)]), + true = length(SegList0Obj) == 0, + + %% Finding differing keys FoldKeysFun = fun(SegListToFind) -> fun(_B, K, Acc) -> @@ -129,14 +161,14 @@ many_put_compare(_Config) -> [length(SegKeyList), length(SegList0), timer:now_diff(os:timestamp(), SWSKL0)]), - + true = length(SegKeyList) >= 1, true = length(SegKeyList) < 10, true = lists:member("Key1.1.4567.4321", SegKeyList), - + % Now remove the object which represents the difference between these % stores and confirm that the tictac trees will now match - + testutil:book_riakdelete(Bookie2, B1, K1, []), {async, TreeAFolder0} = leveled_bookie:book_returnfolder(Bookie2, TicTacQ), SWA1 = os:timestamp(), @@ -149,7 +181,7 @@ many_put_compare(_Config) -> [length(SegList1)]), true = length(SegList1) == 0, % Removed test object so tictac trees should match - + ok = testutil:book_riakput(Bookie3, TestObject, TestSpec), {async, TreeBFolder0} = leveled_bookie:book_returnfolder(Bookie3, TicTacQ), SWB1 = os:timestamp(), @@ -162,10 +194,10 @@ many_put_compare(_Config) -> % Bookie 2 (compared to it being in Bookie2 not Bookie3) ok = leveled_bookie:book_close(Bookie3), - + % Replace Bookie 3 with two stores Bookie 4 and Bookie 5 where the ojects % have been randomly split between the stores - + StartOpts4 = [{root_path, RootPathC}, {max_journalsize, 200000000}, {max_pencillercachesize, 24000}, @@ -176,7 +208,7 @@ many_put_compare(_Config) -> {max_pencillercachesize, 24000}, {sync_strategy, testutil:sync_strategy()}], {ok, Bookie5} = leveled_bookie:book_start(StartOpts5), - + SplitFun = fun(Obj) -> case erlang:phash2(Obj) rem 2 of @@ -192,11 +224,11 @@ many_put_compare(_Config) -> testutil:riakload(Bookie5, ObjLB) end, CLs), - + % query both the stores, then merge the trees - the result should be the % same as the result from the tree created aginst the store with both % partitions - + {async, TreeC0Folder} = leveled_bookie:book_returnfolder(Bookie4, TicTacQ), {async, TreeC1Folder} = leveled_bookie:book_returnfolder(Bookie5, TicTacQ), SWD0 = os:timestamp(), @@ -207,18 +239,18 @@ many_put_compare(_Config) -> TreeC1 = TreeC1Folder(), io:format("Build tictac tree with 100K objects in ~w~n", [timer:now_diff(os:timestamp(), SWD1)]), - + TreeC2 = leveled_tictac:merge_trees(TreeC0, TreeC1), SegList3 = leveled_tictac:find_dirtyleaves(TreeC2, TreeB), io:format("Tree comparison following delete shows ~w different leaves~n", [length(SegList3)]), true = length(SegList3) == 0, - - + + ok = leveled_bookie:book_close(Bookie2), ok = leveled_bookie:book_close(Bookie4), ok = leveled_bookie:book_close(Bookie5). - + index_compare(_Config) -> TreeSize = small, @@ -226,7 +258,7 @@ index_compare(_Config) -> JS = 50000000, SS = testutil:sync_strategy(), SegmentCount = 256 * 256, - + % Test requires multiple different databases, so want to mount them all % on individual file paths RootPathA = testutil:reset_filestructure("testA"), @@ -239,7 +271,7 @@ index_compare(_Config) -> {ok, Book1B} = leveled_bookie:book_start(RootPathB, LS, JS, SS), {ok, Book1C} = leveled_bookie:book_start(RootPathC, LS, JS, SS), {ok, Book1D} = leveled_bookie:book_start(RootPathD, LS, JS, SS), - + % Generate nine lists of objects BucketBin = list_to_binary("Bucket"), GenMapFun = @@ -248,13 +280,13 @@ index_compare(_Config) -> Indexes = testutil:get_randomindexes_generator(8), testutil:generate_objects(10000, binary_uuid, [], V, Indexes) end, - + ObjLists = lists:map(GenMapFun, lists:seq(1, 9)), - + % Load all nine lists into Book1A lists:foreach(fun(ObjL) -> testutil:riakload(Book1A, ObjL) end, - ObjLists), - + ObjLists), + % Split nine lists across Book1B to Book1D, three object lists in each lists:foreach(fun(ObjL) -> testutil:riakload(Book1B, ObjL) end, lists:sublist(ObjLists, 1, 3)), @@ -262,7 +294,7 @@ index_compare(_Config) -> lists:sublist(ObjLists, 4, 3)), lists:foreach(fun(ObjL) -> testutil:riakload(Book1D, ObjL) end, lists:sublist(ObjLists, 7, 3)), - + GetTicTacTreeFun = fun(X, Bookie) -> SW = os:timestamp(), @@ -279,18 +311,18 @@ index_compare(_Config) -> [X, timer:now_diff(os:timestamp(), SW)]), R end, - + % Get a TicTac tree representing one of the indexes in Bucket A TicTacTree1_Full = GetTicTacTreeFun(1, Book1A), TicTacTree1_P1 = GetTicTacTreeFun(1, Book1B), TicTacTree1_P2 = GetTicTacTreeFun(1, Book1C), TicTacTree1_P3 = GetTicTacTreeFun(1, Book1D), - + % Merge the tree across the partitions TicTacTree1_Joined = lists:foldl(fun leveled_tictac:merge_trees/2, TicTacTree1_P1, [TicTacTree1_P2, TicTacTree1_P3]), - + % Go compare! Also check we're not comparing empty trees DL1_0 = leveled_tictac:find_dirtyleaves(TicTacTree1_Full, TicTacTree1_Joined), @@ -303,7 +335,7 @@ index_compare(_Config) -> ok = leveled_bookie:book_close(Book1B), ok = leveled_bookie:book_close(Book1C), ok = leveled_bookie:book_close(Book1D), - + % Double chekc all is well still after a restart % Book1A to get all objects {ok, Book2A} = leveled_bookie:book_start(RootPathA, LS, JS, SS), @@ -316,12 +348,12 @@ index_compare(_Config) -> TicTacTree2_P1 = GetTicTacTreeFun(2, Book2B), TicTacTree2_P2 = GetTicTacTreeFun(2, Book2C), TicTacTree2_P3 = GetTicTacTreeFun(2, Book2D), - + % Merge the tree across the partitions TicTacTree2_Joined = lists:foldl(fun leveled_tictac:merge_trees/2, TicTacTree2_P1, [TicTacTree2_P2, TicTacTree2_P3]), - + % Go compare! Also check we're not comparing empty trees DL2_0 = leveled_tictac:find_dirtyleaves(TicTacTree2_Full, TicTacTree2_Joined), @@ -329,7 +361,7 @@ index_compare(_Config) -> DL2_1 = leveled_tictac:find_dirtyleaves(TicTacTree2_Full, EmptyTree), true = DL2_0 == [], true = length(DL2_1) > 100, - + IdxSpc = {add, "idx2_bin", "zz999"}, {TestObj, TestSpc} = testutil:generate_testobject(BucketBin, term_to_binary("K9.Z"), @@ -338,17 +370,17 @@ index_compare(_Config) -> [{"MDK1", "MDV1"}]), ok = testutil:book_riakput(Book2C, TestObj, TestSpc), testutil:check_forobject(Book2C, TestObj), - + TicTacTree3_Full = GetTicTacTreeFun(2, Book2A), TicTacTree3_P1 = GetTicTacTreeFun(2, Book2B), TicTacTree3_P2 = GetTicTacTreeFun(2, Book2C), TicTacTree3_P3 = GetTicTacTreeFun(2, Book2D), - + % Merge the tree across the partitions TicTacTree3_Joined = lists:foldl(fun leveled_tictac:merge_trees/2, TicTacTree3_P1, [TicTacTree3_P2, TicTacTree3_P3]), - + % Find all keys index, and then just the last key IdxQ1 = {index_query, BucketBin, @@ -357,7 +389,7 @@ index_compare(_Config) -> {true, undefined}}, {async, IdxFolder1} = leveled_bookie:book_returnfolder(Book2C, IdxQ1), true = IdxFolder1() >= 1, - + DL_3to2B = leveled_tictac:find_dirtyleaves(TicTacTree2_P1, TicTacTree3_P1), DL_3to2C = leveled_tictac:find_dirtyleaves(TicTacTree2_P2, @@ -366,23 +398,23 @@ index_compare(_Config) -> TicTacTree3_P3), io:format("Individual tree comparison found dirty leaves of ~w ~w ~w~n", [DL_3to2B, DL_3to2C, DL_3to2D]), - + true = length(DL_3to2B) == 0, true = length(DL_3to2C) == 1, true = length(DL_3to2D) == 0, - + % Go compare! Should find a difference in one leaf DL3_0 = leveled_tictac:find_dirtyleaves(TicTacTree3_Full, TicTacTree3_Joined), io:format("Different leaves count ~w~n", [length(DL3_0)]), true = length(DL3_0) == 1, - + % Now we want to find for the {Term, Key} pairs that make up the segment % diferrence (there should only be one) % % We want the database to filter on segment - so this doesn't have the % overheads of key listing - + FoldKeysIndexQFun = fun(_Bucket, {Term, Key}, Acc) -> Seg = leveled_tictac:get_segment(Key, SegmentCount), @@ -393,7 +425,7 @@ index_compare(_Config) -> Acc end end, - + MismatchQ = {index_query, BucketBin, {FoldKeysIndexQFun, []}, @@ -403,21 +435,21 @@ index_compare(_Config) -> {async, MMFldr_2B} = leveled_bookie:book_returnfolder(Book2B, MismatchQ), {async, MMFldr_2C} = leveled_bookie:book_returnfolder(Book2C, MismatchQ), {async, MMFldr_2D} = leveled_bookie:book_returnfolder(Book2D, MismatchQ), - + SWSS = os:timestamp(), SL_Joined = MMFldr_2B() ++ MMFldr_2C() ++ MMFldr_2D(), SL_Full = MMFldr_2A(), io:format("Segment search across both clusters took ~w~n", [timer:now_diff(os:timestamp(), SWSS)]), - + io:format("Joined SegList ~w~n", [SL_Joined]), io:format("Full SegList ~w~n", [SL_Full]), - + Diffs = lists:subtract(SL_Full, SL_Joined) ++ lists:subtract(SL_Joined, SL_Full), - + io:format("Differences between lists ~w~n", [Diffs]), - + % The actual difference is discovered true = lists:member({"zz999", term_to_binary("K9.Z")}, Diffs), % Without discovering too many others @@ -433,11 +465,11 @@ index_compare(_Config) -> recent_aae_noaae(_Config) -> % Starts databases with recent_aae tables, and attempt to query to fetch % recent aae trees returns empty trees as no index entries are found. - + TreeSize = small, % SegmentCount = 256 * 256, UnitMins = 2, - + % Test requires multiple different databases, so want to mount them all % on individual file paths RootPathA = testutil:reset_filestructure("testA"), @@ -448,28 +480,28 @@ recent_aae_noaae(_Config) -> StartOptsB = aae_startopts(RootPathB, false), StartOptsC = aae_startopts(RootPathC, false), StartOptsD = aae_startopts(RootPathD, false), - + % Book1A to get all objects {ok, Book1A} = leveled_bookie:book_start(StartOptsA), % Book1B/C/D will have objects partitioned across it {ok, Book1B} = leveled_bookie:book_start(StartOptsB), {ok, Book1C} = leveled_bookie:book_start(StartOptsC), {ok, Book1D} = leveled_bookie:book_start(StartOptsD), - + {B1, K1, V1, S1, MD} = {"Bucket", "Key1.1.4567.4321", "Value1", [], [{"MDK1", "MDV1"}]}, {TestObject, TestSpec} = testutil:generate_testobject(B1, K1, V1, S1, MD), - + SW_StartLoad = os:timestamp(), - + ok = testutil:book_riakput(Book1A, TestObject, TestSpec), ok = testutil:book_riakput(Book1B, TestObject, TestSpec), testutil:check_forobject(Book1A, TestObject), testutil:check_forobject(Book1B, TestObject), - + {TicTacTreeJoined, TicTacTreeFull, EmptyTree, _LMDIndexes} = load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D, SW_StartLoad, TreeSize, UnitMins, @@ -477,7 +509,7 @@ recent_aae_noaae(_Config) -> % Go compare! Also confirm we're not comparing empty trees DL1_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, TicTacTreeJoined), - + DL1_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree), true = DL1_0 == [], true = length(DL1_1) == 0, @@ -489,7 +521,7 @@ recent_aae_noaae(_Config) -> recent_aae_allaae(_Config) -> - % Leveled is started in blacklisted mode with no buckets blacklisted. + % Leveled is started in blacklisted mode with no buckets blacklisted. % % A number of changes are then loaded into a store, and also partitioned % across a separate set of three stores. A merge tree is returned from @@ -503,12 +535,12 @@ recent_aae_allaae(_Config) -> % The segment Id found is then used in a query to find the Keys that make % up that segment, and the delta discovered should be just that one key % which was known to have been changed - + TreeSize = small, % SegmentCount = 256 * 256, UnitMins = 2, AAE = {blacklist, [], 60, UnitMins}, - + % Test requires multiple different databases, so want to mount them all % on individual file paths RootPathA = testutil:reset_filestructure("testA"), @@ -519,28 +551,28 @@ recent_aae_allaae(_Config) -> StartOptsB = aae_startopts(RootPathB, AAE), StartOptsC = aae_startopts(RootPathC, AAE), StartOptsD = aae_startopts(RootPathD, AAE), - + % Book1A to get all objects {ok, Book1A} = leveled_bookie:book_start(StartOptsA), % Book1B/C/D will have objects partitioned across it {ok, Book1B} = leveled_bookie:book_start(StartOptsB), {ok, Book1C} = leveled_bookie:book_start(StartOptsC), {ok, Book1D} = leveled_bookie:book_start(StartOptsD), - + {B1, K1, V1, S1, MD} = {"Bucket", "Key1.1.4567.4321", "Value1", [], [{"MDK1", "MDV1"}]}, {TestObject, TestSpec} = testutil:generate_testobject(B1, K1, V1, S1, MD), - + SW_StartLoad = os:timestamp(), - + ok = testutil:book_riakput(Book1A, TestObject, TestSpec), ok = testutil:book_riakput(Book1B, TestObject, TestSpec), testutil:check_forobject(Book1A, TestObject), testutil:check_forobject(Book1B, TestObject), - + {TicTacTreeJoined, TicTacTreeFull, EmptyTree, LMDIndexes} = load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D, SW_StartLoad, TreeSize, UnitMins, @@ -548,7 +580,7 @@ recent_aae_allaae(_Config) -> % Go compare! Also confirm we're not comparing empty trees DL1_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, TicTacTreeJoined), - + DL1_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree), true = DL1_0 == [], true = length(DL1_1) > 100, @@ -557,14 +589,14 @@ recent_aae_allaae(_Config) -> ok = leveled_bookie:book_close(Book1B), ok = leveled_bookie:book_close(Book1C), ok = leveled_bookie:book_close(Book1D), - + % Book2A to get all objects {ok, Book2A} = leveled_bookie:book_start(StartOptsA), % Book2B/C/D will have objects partitioned across it {ok, Book2B} = leveled_bookie:book_start(StartOptsB), {ok, Book2C} = leveled_bookie:book_start(StartOptsC), {ok, Book2D} = leveled_bookie:book_start(StartOptsD), - + {TicTacTreeJoined, TicTacTreeFull, EmptyTree, LMDIndexes} = load_and_check_recentaae(Book2A, Book2B, Book2C, Book2D, SW_StartLoad, TreeSize, UnitMins, @@ -572,21 +604,21 @@ recent_aae_allaae(_Config) -> % Go compare! Also confirm we're not comparing empty trees DL1_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, TicTacTreeJoined), - + DL1_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree), true = DL1_0 == [], true = length(DL1_1) > 100, - + V2 = "Value2", {TestObject2, TestSpec2} = testutil:generate_testobject(B1, K1, V2, S1, MD), - + New_startTS = os:timestamp(), - + ok = testutil:book_riakput(Book2B, TestObject2, TestSpec2), testutil:check_forobject(Book2B, TestObject2), testutil:check_forobject(Book2A, TestObject), - + New_endTS = os:timestamp(), NewLMDIndexes = determine_lmd_indexes(New_startTS, New_endTS, UnitMins), {TicTacTreeJoined2, TicTacTreeFull2, _EmptyTree, NewLMDIndexes} = @@ -595,10 +627,10 @@ recent_aae_allaae(_Config) -> NewLMDIndexes), DL2_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull2, TicTacTreeJoined2), - + % DL2_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree), true = length(DL2_0) == 1, - + [DirtySeg] = DL2_0, TermPrefix = string:right(integer_to_list(DirtySeg), 8, $0), @@ -631,18 +663,18 @@ recent_aae_allaae(_Config) -> {KeysTerms2D, _} = lists:foldl(LMDSegFolder, {[], Book2D}, lists:usort(LMDIndexes ++ NewLMDIndexes)), - + KeysTerms2Joined = KeysTerms2B ++ KeysTerms2C ++ KeysTerms2D, DeltaX = lists:subtract(KeysTerms2A, KeysTerms2Joined), DeltaY = lists:subtract(KeysTerms2Joined, KeysTerms2A), - + io:format("DeltaX ~w~n", [DeltaX]), io:format("DeltaY ~w~n", [DeltaY]), - + true = length(DeltaX) == 0, % This hasn't seen any extra changes true = length(DeltaY) == 1, % This has seen an extra change [{_, {B1, K1}}] = DeltaY, - + ok = leveled_bookie:book_close(Book2A), ok = leveled_bookie:book_close(Book2B), ok = leveled_bookie:book_close(Book2C), @@ -654,12 +686,12 @@ recent_aae_bucketaae(_Config) -> % Configure AAE to work only on a single whitelisted bucket % Confirm that we can spot a delta in this bucket, but not % in another bucket - + TreeSize = small, % SegmentCount = 256 * 256, UnitMins = 2, AAE = {whitelist, [<<"Bucket">>], 60, UnitMins}, - + % Test requires multiple different databases, so want to mount them all % on individual file paths RootPathA = testutil:reset_filestructure("testA"), @@ -670,28 +702,28 @@ recent_aae_bucketaae(_Config) -> StartOptsB = aae_startopts(RootPathB, AAE), StartOptsC = aae_startopts(RootPathC, AAE), StartOptsD = aae_startopts(RootPathD, AAE), - + % Book1A to get all objects {ok, Book1A} = leveled_bookie:book_start(StartOptsA), % Book1B/C/D will have objects partitioned across it {ok, Book1B} = leveled_bookie:book_start(StartOptsB), {ok, Book1C} = leveled_bookie:book_start(StartOptsC), {ok, Book1D} = leveled_bookie:book_start(StartOptsD), - + {B1, K1, V1, S1, MD} = {<<"Bucket">>, "Key1.1.4567.4321", "Value1", [], [{"MDK1", "MDV1"}]}, {TestObject, TestSpec} = testutil:generate_testobject(B1, K1, V1, S1, MD), - + SW_StartLoad = os:timestamp(), - + ok = testutil:book_riakput(Book1A, TestObject, TestSpec), ok = testutil:book_riakput(Book1B, TestObject, TestSpec), testutil:check_forobject(Book1A, TestObject), testutil:check_forobject(Book1B, TestObject), - + {TicTacTreeJoined, TicTacTreeFull, EmptyTree, LMDIndexes} = load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D, SW_StartLoad, TreeSize, UnitMins, @@ -699,7 +731,7 @@ recent_aae_bucketaae(_Config) -> % Go compare! Also confirm we're not comparing empty trees DL1_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, TicTacTreeJoined), - + DL1_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree), true = DL1_0 == [], true = length(DL1_1) > 100, @@ -708,27 +740,27 @@ recent_aae_bucketaae(_Config) -> ok = leveled_bookie:book_close(Book1B), ok = leveled_bookie:book_close(Book1C), ok = leveled_bookie:book_close(Book1D), - + % Book2A to get all objects {ok, Book2A} = leveled_bookie:book_start(StartOptsA), % Book2B/C/D will have objects partitioned across it {ok, Book2B} = leveled_bookie:book_start(StartOptsB), {ok, Book2C} = leveled_bookie:book_start(StartOptsC), {ok, Book2D} = leveled_bookie:book_start(StartOptsD), - + % Change the value for a key in another bucket % If we get trees for this period, no difference should be found - + V2 = "Value2", {TestObject2, TestSpec2} = testutil:generate_testobject(<<"NotBucket">>, K1, V2, S1, MD), - + New_startTS2 = os:timestamp(), - + ok = testutil:book_riakput(Book2B, TestObject2, TestSpec2), testutil:check_forobject(Book2B, TestObject2), testutil:check_forobject(Book2A, TestObject), - + New_endTS2 = os:timestamp(), NewLMDIndexes2 = determine_lmd_indexes(New_startTS2, New_endTS2, UnitMins), {TicTacTreeJoined2, TicTacTreeFull2, _EmptyTree, NewLMDIndexes2} = @@ -738,19 +770,19 @@ recent_aae_bucketaae(_Config) -> DL2_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull2, TicTacTreeJoined2), true = length(DL2_0) == 0, - + % Now create an object that is a change to an existing key in the % monitored bucket. A differrence should be found - + {TestObject3, TestSpec3} = testutil:generate_testobject(B1, K1, V2, S1, MD), - + New_startTS3 = os:timestamp(), - + ok = testutil:book_riakput(Book2B, TestObject3, TestSpec3), testutil:check_forobject(Book2B, TestObject3), testutil:check_forobject(Book2A, TestObject), - + New_endTS3 = os:timestamp(), NewLMDIndexes3 = determine_lmd_indexes(New_startTS3, New_endTS3, UnitMins), {TicTacTreeJoined3, TicTacTreeFull3, _EmptyTree, NewLMDIndexes3} = @@ -759,15 +791,15 @@ recent_aae_bucketaae(_Config) -> NewLMDIndexes3, <<"Bucket">>), DL3_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull3, TicTacTreeJoined3), - + % DL2_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree), true = length(DL3_0) == 1, - + % Find the dirty segment, and use that to find the dirty key % % Note that unlike when monitoring $all, fold_keys can be used as there % is no need to return the Bucket (as hte bucket is known) - + [DirtySeg] = DL3_0, TermPrefix = string:right(integer_to_list(DirtySeg), 8, $0), @@ -800,18 +832,18 @@ recent_aae_bucketaae(_Config) -> {KeysTerms2D, _} = lists:foldl(LMDSegFolder, {[], Book2D}, lists:usort(LMDIndexes ++ NewLMDIndexes3)), - + KeysTerms2Joined = KeysTerms2B ++ KeysTerms2C ++ KeysTerms2D, DeltaX = lists:subtract(KeysTerms2A, KeysTerms2Joined), DeltaY = lists:subtract(KeysTerms2Joined, KeysTerms2A), - + io:format("DeltaX ~w~n", [DeltaX]), io:format("DeltaY ~w~n", [DeltaY]), - + true = length(DeltaX) == 0, % This hasn't seen any extra changes true = length(DeltaY) == 1, % This has seen an extra change [{_, K1}] = DeltaY, - + ok = leveled_bookie:book_close(Book2A), ok = leveled_bookie:book_close(Book2B), ok = leveled_bookie:book_close(Book2C), @@ -820,21 +852,21 @@ recent_aae_bucketaae(_Config) -> recent_aae_expiry(_Config) -> % Proof that the index entries are indeed expired - + TreeSize = small, % SegmentCount = 256 * 256, UnitMins = 1, TotalMins = 2, AAE = {blacklist, [], TotalMins, UnitMins}, - + % Test requires multiple different databases, so want to mount them all % on individual file paths RootPathA = testutil:reset_filestructure("testA"), StartOptsA = aae_startopts(RootPathA, AAE), - + % Book1A to get all objects {ok, Book1A} = leveled_bookie:book_start(StartOptsA), - + GenMapFun = fun(_X) -> V = testutil:get_compressiblevalue(), @@ -845,13 +877,13 @@ recent_aae_expiry(_Config) -> V, Indexes) end, - + ObjLists = lists:map(GenMapFun, lists:seq(1, 3)), - + SW0 = os:timestamp(), % Load all nine lists into Book1A lists:foreach(fun(ObjL) -> testutil:riakload(Book1A, ObjL) end, - ObjLists), + ObjLists), SW1 = os:timestamp(), % sleep for two minutes, so all index entries will have expired GetTicTacTreeFun = @@ -860,19 +892,19 @@ recent_aae_expiry(_Config) -> end, EmptyTree = leveled_tictac:new_tree(empty, TreeSize), LMDIndexes = determine_lmd_indexes(SW0, SW1, UnitMins), - + % Should get a non-empty answer to the query TicTacTree1_Full = - lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes), + lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes), DL3_0 = leveled_tictac:find_dirtyleaves(TicTacTree1_Full, EmptyTree), io:format("Dirty leaves found before expiry ~w~n", [length(DL3_0)]), true = length(DL3_0) > 0, - + SecondsSinceLMD = timer:now_diff(os:timestamp(), SW0) div 1000000, SecondsToExpiry = (TotalMins + UnitMins) * 60, - - io:format("SecondsToExpiry ~w SecondsSinceLMD ~w~n", + + io:format("SecondsToExpiry ~w SecondsSinceLMD ~w~n", [SecondsToExpiry, SecondsSinceLMD]), io:format("LMDIndexes ~w~n", [LMDIndexes]), @@ -882,17 +914,17 @@ recent_aae_expiry(_Config) -> false -> timer:sleep(1000) end, - + % Should now get an empty answer - all entries have expired TicTacTree2_Full = - lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes), + lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes), DL4_0 = leveled_tictac:find_dirtyleaves(TicTacTree2_Full, EmptyTree), io:format("Dirty leaves found after expiry ~w~n", [length(DL4_0)]), timer:sleep(10000), TicTacTree3_Full = - lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes), + lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes), DL5_0 = leveled_tictac:find_dirtyleaves(TicTacTree3_Full, EmptyTree), io:format("Dirty leaves found after expiry plus 10s ~w~n", [length(DL5_0)]), @@ -913,7 +945,7 @@ load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D, load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D, SW_StartLoad, TreeSize, UnitMins, LMDIndexes_Loaded, Bucket) -> - LMDIndexes = + LMDIndexes = case LMDIndexes_Loaded of false -> % Generate nine lists of objects @@ -928,13 +960,13 @@ load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D, V, Indexes) end, - + ObjLists = lists:map(GenMapFun, lists:seq(1, 9)), - + % Load all nine lists into Book1A lists:foreach(fun(ObjL) -> testutil:riakload(Book1A, ObjL) end, - ObjLists), - + ObjLists), + % Split nine lists across Book1B to Book1D, three object lists % in each lists:foreach(fun(ObjL) -> testutil:riakload(Book1B, ObjL) end, @@ -943,38 +975,38 @@ load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D, lists:sublist(ObjLists, 4, 3)), lists:foreach(fun(ObjL) -> testutil:riakload(Book1D, ObjL) end, lists:sublist(ObjLists, 7, 3)), - + SW_EndLoad = os:timestamp(), determine_lmd_indexes(SW_StartLoad, SW_EndLoad, UnitMins); _ -> LMDIndexes_Loaded end, - + EmptyTree = leveled_tictac:new_tree(empty, TreeSize), - + GetTicTacTreeFun = fun(Bookie) -> get_tictactree_fun(Bookie, Bucket, TreeSize) end, - + % Get a TicTac tree representing one of the indexes in Bucket A TicTacTree1_Full = lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes), - + TicTacTree1_P1 = lists:foldl(GetTicTacTreeFun(Book1B), EmptyTree, LMDIndexes), TicTacTree1_P2 = lists:foldl(GetTicTacTreeFun(Book1C), EmptyTree, LMDIndexes), TicTacTree1_P3 = lists:foldl(GetTicTacTreeFun(Book1D), EmptyTree, LMDIndexes), - + % Merge the tree across the partitions TicTacTree1_Joined = lists:foldl(fun leveled_tictac:merge_trees/2, TicTacTree1_P1, [TicTacTree1_P2, TicTacTree1_P3]), - + {TicTacTree1_Full, TicTacTree1_Joined, EmptyTree, LMDIndexes}. - + aae_startopts(RootPath, AAE) -> LS = 2000, @@ -992,7 +1024,7 @@ determine_lmd_indexes(StartTS, EndTS, UnitMins) -> EndDT = calendar:now_to_datetime(EndTS), StartTimeStr = get_strtime(StartDT, UnitMins), EndTimeStr = get_strtime(EndDT, UnitMins), - + AddTimeFun = fun(X, Acc) -> case lists:member(EndTimeStr, Acc) of @@ -1007,10 +1039,10 @@ determine_lmd_indexes(StartTS, EndTS, UnitMins) -> Acc ++ [get_strtime(NextDT, UnitMins)] end end, - - lists:foldl(AddTimeFun, [StartTimeStr], lists:seq(1, 10)). - - + + lists:foldl(AddTimeFun, [StartTimeStr], lists:seq(1, 10)). + + get_strtime(DateTime, UnitMins) -> {{Y, M, D}, {Hour, Minute, _Second}} = DateTime, RoundMins =