From dd2013289260c0f085a0580e803e16b119d2f768 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Sat, 5 Aug 2017 16:43:03 +0100 Subject: [PATCH] Add test with fold_heads Build the AAE tree equally using fold_heads. This is a pre-cursor to running this within Riak. In part this leans on some of the work done to improve standard Riak AAE with leveled. When rebuilding the standard AAE store only the head is required, and so this process was switched in riak_kv_sweeper to make a fold_heads request if supported by the backend. The head response is a proxy object, which when loaded into a riak_object will allow for access to object metadata, but will use the passed function if access to object contents is requested. --- test/end_to_end/tictac_SUITE.erl | 318 +++++++++++++++++-------------- 1 file changed, 175 insertions(+), 143 deletions(-) diff --git a/test/end_to_end/tictac_SUITE.erl b/test/end_to_end/tictac_SUITE.erl index 7e4d7a0..39caf5d 100644 --- a/test/end_to_end/tictac_SUITE.erl +++ b/test/end_to_end/tictac_SUITE.erl @@ -21,6 +21,8 @@ all() -> [ ]. -define(LMD_FORMAT, "~4..0w~2..0w~2..0w~2..0w~2..0w"). +-define(V1_VERS, 1). +-define(MAGIC, 53). % riak_kv -> riak_object many_put_compare(_Config) -> TreeSize = small, @@ -31,7 +33,7 @@ many_put_compare(_Config) -> RootPathB = testutil:reset_filestructure("testB"), RootPathC = testutil:reset_filestructure("testC"), RootPathD = testutil:reset_filestructure("testD"), - + % Start the first database, load a test object, close it, start it again StartOpts1 = [{root_path, RootPathA}, {max_pencillercachesize, 16000}, @@ -52,11 +54,11 @@ many_put_compare(_Config) -> {sync_strategy, testutil:sync_strategy()}], {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), testutil:check_forobject(Bookie2, TestObject), - + % Generate 200K objects to be sued within the test, and load them into % the first store (outputting the generated objects as a list of lists) % to be used elsewhere - + GenList = [2, 20002, 40002, 60002, 80002, 100002, 120002, 140002, 160002, 180002], CLs = testutil:load_objects(20000, @@ -65,20 +67,20 @@ many_put_compare(_Config) -> TestObject, fun testutil:generate_smallobjects/2, 20000), - + % Start a new store, and load the same objects (except fot the original % test object) into this store - + StartOpts3 = [{root_path, RootPathB}, {max_journalsize, 200000000}, {max_pencillercachesize, 16000}, {sync_strategy, testutil:sync_strategy()}], {ok, Bookie3} = leveled_bookie:book_start(StartOpts3), lists:foreach(fun(ObjL) -> testutil:riakload(Bookie3, ObjL) end, CLs), - - % Now run a tictac query against both stores to see th extent to which + + % Now run a tictac query against both stores to see the extent to which % state between stores is consistent - + TicTacQ = {tictactree_obj, {o_rkv, "Bucket", null, null, false}, TreeSize, @@ -99,15 +101,45 @@ many_put_compare(_Config) -> [timer:now_diff(os:timestamp(), SWC0)]), io:format("Tree comparison shows ~w different leaves~n", [length(SegList0)]), - AltList = leveled_tictac:find_dirtyleaves(TreeA, - leveled_tictac:new_tree(0)), + AltList = + leveled_tictac:find_dirtyleaves(TreeA, + leveled_tictac:new_tree(0, TreeSize)), io:format("Tree comparison shows ~w altered leaves~n", [length(AltList)]), - true = length(SegList0) == 1, + true = length(SegList0) == 1, % only the test object should be different - true = length(AltList) > 10000, + true = length(AltList) > 10000, % check there are a significant number of differences from empty - + + % Now run the same query by putting the tree-building responsibility onto + % the fold_objects_fun + HashFun = + fun(_Key, Value) -> + {proxy_object, HeadBin, _Size, _FetchFun} = binary_to_term(Value), + <> = HeadBin, + <> = Rest, + erlang:phash2(lists:sort(binary_to_term(VclockBin))) + end, + FoldObjectsFun = + fun(_Bucket, Key, Value, Acc) -> + leveled_tictac:add_kv(Acc, Key, Value, HashFun) + end, + FoldQ = {foldheads_bybucket, + o_rkv, + "Bucket", + {FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)}}, + {async, TreeAObjFolder} = leveled_bookie:book_returnfolder(Bookie2, FoldQ), + SWB0Obj = os:timestamp(), + TreeAObj = TreeAObjFolder(), + io:format("Build tictac tree via object foldwith 200K objects in ~w~n", + [timer:now_diff(os:timestamp(), SWB0Obj)]), + SegList0Obj = leveled_tictac:find_dirtyleaves(TreeA, TreeAObj), + io:format("Fold object compared with tictac fold has ~w diffs~n", + [length(SegList0Obj)]), + true = length(SegList0Obj) == 0, + + %% Finding differing keys FoldKeysFun = fun(SegListToFind) -> fun(_B, K, Acc) -> @@ -129,14 +161,14 @@ many_put_compare(_Config) -> [length(SegKeyList), length(SegList0), timer:now_diff(os:timestamp(), SWSKL0)]), - + true = length(SegKeyList) >= 1, true = length(SegKeyList) < 10, true = lists:member("Key1.1.4567.4321", SegKeyList), - + % Now remove the object which represents the difference between these % stores and confirm that the tictac trees will now match - + testutil:book_riakdelete(Bookie2, B1, K1, []), {async, TreeAFolder0} = leveled_bookie:book_returnfolder(Bookie2, TicTacQ), SWA1 = os:timestamp(), @@ -149,7 +181,7 @@ many_put_compare(_Config) -> [length(SegList1)]), true = length(SegList1) == 0, % Removed test object so tictac trees should match - + ok = testutil:book_riakput(Bookie3, TestObject, TestSpec), {async, TreeBFolder0} = leveled_bookie:book_returnfolder(Bookie3, TicTacQ), SWB1 = os:timestamp(), @@ -162,10 +194,10 @@ many_put_compare(_Config) -> % Bookie 2 (compared to it being in Bookie2 not Bookie3) ok = leveled_bookie:book_close(Bookie3), - + % Replace Bookie 3 with two stores Bookie 4 and Bookie 5 where the ojects % have been randomly split between the stores - + StartOpts4 = [{root_path, RootPathC}, {max_journalsize, 200000000}, {max_pencillercachesize, 24000}, @@ -176,7 +208,7 @@ many_put_compare(_Config) -> {max_pencillercachesize, 24000}, {sync_strategy, testutil:sync_strategy()}], {ok, Bookie5} = leveled_bookie:book_start(StartOpts5), - + SplitFun = fun(Obj) -> case erlang:phash2(Obj) rem 2 of @@ -192,11 +224,11 @@ many_put_compare(_Config) -> testutil:riakload(Bookie5, ObjLB) end, CLs), - + % query both the stores, then merge the trees - the result should be the % same as the result from the tree created aginst the store with both % partitions - + {async, TreeC0Folder} = leveled_bookie:book_returnfolder(Bookie4, TicTacQ), {async, TreeC1Folder} = leveled_bookie:book_returnfolder(Bookie5, TicTacQ), SWD0 = os:timestamp(), @@ -207,18 +239,18 @@ many_put_compare(_Config) -> TreeC1 = TreeC1Folder(), io:format("Build tictac tree with 100K objects in ~w~n", [timer:now_diff(os:timestamp(), SWD1)]), - + TreeC2 = leveled_tictac:merge_trees(TreeC0, TreeC1), SegList3 = leveled_tictac:find_dirtyleaves(TreeC2, TreeB), io:format("Tree comparison following delete shows ~w different leaves~n", [length(SegList3)]), true = length(SegList3) == 0, - - + + ok = leveled_bookie:book_close(Bookie2), ok = leveled_bookie:book_close(Bookie4), ok = leveled_bookie:book_close(Bookie5). - + index_compare(_Config) -> TreeSize = small, @@ -226,7 +258,7 @@ index_compare(_Config) -> JS = 50000000, SS = testutil:sync_strategy(), SegmentCount = 256 * 256, - + % Test requires multiple different databases, so want to mount them all % on individual file paths RootPathA = testutil:reset_filestructure("testA"), @@ -239,7 +271,7 @@ index_compare(_Config) -> {ok, Book1B} = leveled_bookie:book_start(RootPathB, LS, JS, SS), {ok, Book1C} = leveled_bookie:book_start(RootPathC, LS, JS, SS), {ok, Book1D} = leveled_bookie:book_start(RootPathD, LS, JS, SS), - + % Generate nine lists of objects BucketBin = list_to_binary("Bucket"), GenMapFun = @@ -248,13 +280,13 @@ index_compare(_Config) -> Indexes = testutil:get_randomindexes_generator(8), testutil:generate_objects(10000, binary_uuid, [], V, Indexes) end, - + ObjLists = lists:map(GenMapFun, lists:seq(1, 9)), - + % Load all nine lists into Book1A lists:foreach(fun(ObjL) -> testutil:riakload(Book1A, ObjL) end, - ObjLists), - + ObjLists), + % Split nine lists across Book1B to Book1D, three object lists in each lists:foreach(fun(ObjL) -> testutil:riakload(Book1B, ObjL) end, lists:sublist(ObjLists, 1, 3)), @@ -262,7 +294,7 @@ index_compare(_Config) -> lists:sublist(ObjLists, 4, 3)), lists:foreach(fun(ObjL) -> testutil:riakload(Book1D, ObjL) end, lists:sublist(ObjLists, 7, 3)), - + GetTicTacTreeFun = fun(X, Bookie) -> SW = os:timestamp(), @@ -279,18 +311,18 @@ index_compare(_Config) -> [X, timer:now_diff(os:timestamp(), SW)]), R end, - + % Get a TicTac tree representing one of the indexes in Bucket A TicTacTree1_Full = GetTicTacTreeFun(1, Book1A), TicTacTree1_P1 = GetTicTacTreeFun(1, Book1B), TicTacTree1_P2 = GetTicTacTreeFun(1, Book1C), TicTacTree1_P3 = GetTicTacTreeFun(1, Book1D), - + % Merge the tree across the partitions TicTacTree1_Joined = lists:foldl(fun leveled_tictac:merge_trees/2, TicTacTree1_P1, [TicTacTree1_P2, TicTacTree1_P3]), - + % Go compare! Also check we're not comparing empty trees DL1_0 = leveled_tictac:find_dirtyleaves(TicTacTree1_Full, TicTacTree1_Joined), @@ -303,7 +335,7 @@ index_compare(_Config) -> ok = leveled_bookie:book_close(Book1B), ok = leveled_bookie:book_close(Book1C), ok = leveled_bookie:book_close(Book1D), - + % Double chekc all is well still after a restart % Book1A to get all objects {ok, Book2A} = leveled_bookie:book_start(RootPathA, LS, JS, SS), @@ -316,12 +348,12 @@ index_compare(_Config) -> TicTacTree2_P1 = GetTicTacTreeFun(2, Book2B), TicTacTree2_P2 = GetTicTacTreeFun(2, Book2C), TicTacTree2_P3 = GetTicTacTreeFun(2, Book2D), - + % Merge the tree across the partitions TicTacTree2_Joined = lists:foldl(fun leveled_tictac:merge_trees/2, TicTacTree2_P1, [TicTacTree2_P2, TicTacTree2_P3]), - + % Go compare! Also check we're not comparing empty trees DL2_0 = leveled_tictac:find_dirtyleaves(TicTacTree2_Full, TicTacTree2_Joined), @@ -329,7 +361,7 @@ index_compare(_Config) -> DL2_1 = leveled_tictac:find_dirtyleaves(TicTacTree2_Full, EmptyTree), true = DL2_0 == [], true = length(DL2_1) > 100, - + IdxSpc = {add, "idx2_bin", "zz999"}, {TestObj, TestSpc} = testutil:generate_testobject(BucketBin, term_to_binary("K9.Z"), @@ -338,17 +370,17 @@ index_compare(_Config) -> [{"MDK1", "MDV1"}]), ok = testutil:book_riakput(Book2C, TestObj, TestSpc), testutil:check_forobject(Book2C, TestObj), - + TicTacTree3_Full = GetTicTacTreeFun(2, Book2A), TicTacTree3_P1 = GetTicTacTreeFun(2, Book2B), TicTacTree3_P2 = GetTicTacTreeFun(2, Book2C), TicTacTree3_P3 = GetTicTacTreeFun(2, Book2D), - + % Merge the tree across the partitions TicTacTree3_Joined = lists:foldl(fun leveled_tictac:merge_trees/2, TicTacTree3_P1, [TicTacTree3_P2, TicTacTree3_P3]), - + % Find all keys index, and then just the last key IdxQ1 = {index_query, BucketBin, @@ -357,7 +389,7 @@ index_compare(_Config) -> {true, undefined}}, {async, IdxFolder1} = leveled_bookie:book_returnfolder(Book2C, IdxQ1), true = IdxFolder1() >= 1, - + DL_3to2B = leveled_tictac:find_dirtyleaves(TicTacTree2_P1, TicTacTree3_P1), DL_3to2C = leveled_tictac:find_dirtyleaves(TicTacTree2_P2, @@ -366,23 +398,23 @@ index_compare(_Config) -> TicTacTree3_P3), io:format("Individual tree comparison found dirty leaves of ~w ~w ~w~n", [DL_3to2B, DL_3to2C, DL_3to2D]), - + true = length(DL_3to2B) == 0, true = length(DL_3to2C) == 1, true = length(DL_3to2D) == 0, - + % Go compare! Should find a difference in one leaf DL3_0 = leveled_tictac:find_dirtyleaves(TicTacTree3_Full, TicTacTree3_Joined), io:format("Different leaves count ~w~n", [length(DL3_0)]), true = length(DL3_0) == 1, - + % Now we want to find for the {Term, Key} pairs that make up the segment % diferrence (there should only be one) % % We want the database to filter on segment - so this doesn't have the % overheads of key listing - + FoldKeysIndexQFun = fun(_Bucket, {Term, Key}, Acc) -> Seg = leveled_tictac:get_segment(Key, SegmentCount), @@ -393,7 +425,7 @@ index_compare(_Config) -> Acc end end, - + MismatchQ = {index_query, BucketBin, {FoldKeysIndexQFun, []}, @@ -403,21 +435,21 @@ index_compare(_Config) -> {async, MMFldr_2B} = leveled_bookie:book_returnfolder(Book2B, MismatchQ), {async, MMFldr_2C} = leveled_bookie:book_returnfolder(Book2C, MismatchQ), {async, MMFldr_2D} = leveled_bookie:book_returnfolder(Book2D, MismatchQ), - + SWSS = os:timestamp(), SL_Joined = MMFldr_2B() ++ MMFldr_2C() ++ MMFldr_2D(), SL_Full = MMFldr_2A(), io:format("Segment search across both clusters took ~w~n", [timer:now_diff(os:timestamp(), SWSS)]), - + io:format("Joined SegList ~w~n", [SL_Joined]), io:format("Full SegList ~w~n", [SL_Full]), - + Diffs = lists:subtract(SL_Full, SL_Joined) ++ lists:subtract(SL_Joined, SL_Full), - + io:format("Differences between lists ~w~n", [Diffs]), - + % The actual difference is discovered true = lists:member({"zz999", term_to_binary("K9.Z")}, Diffs), % Without discovering too many others @@ -433,11 +465,11 @@ index_compare(_Config) -> recent_aae_noaae(_Config) -> % Starts databases with recent_aae tables, and attempt to query to fetch % recent aae trees returns empty trees as no index entries are found. - + TreeSize = small, % SegmentCount = 256 * 256, UnitMins = 2, - + % Test requires multiple different databases, so want to mount them all % on individual file paths RootPathA = testutil:reset_filestructure("testA"), @@ -448,28 +480,28 @@ recent_aae_noaae(_Config) -> StartOptsB = aae_startopts(RootPathB, false), StartOptsC = aae_startopts(RootPathC, false), StartOptsD = aae_startopts(RootPathD, false), - + % Book1A to get all objects {ok, Book1A} = leveled_bookie:book_start(StartOptsA), % Book1B/C/D will have objects partitioned across it {ok, Book1B} = leveled_bookie:book_start(StartOptsB), {ok, Book1C} = leveled_bookie:book_start(StartOptsC), {ok, Book1D} = leveled_bookie:book_start(StartOptsD), - + {B1, K1, V1, S1, MD} = {"Bucket", "Key1.1.4567.4321", "Value1", [], [{"MDK1", "MDV1"}]}, {TestObject, TestSpec} = testutil:generate_testobject(B1, K1, V1, S1, MD), - + SW_StartLoad = os:timestamp(), - + ok = testutil:book_riakput(Book1A, TestObject, TestSpec), ok = testutil:book_riakput(Book1B, TestObject, TestSpec), testutil:check_forobject(Book1A, TestObject), testutil:check_forobject(Book1B, TestObject), - + {TicTacTreeJoined, TicTacTreeFull, EmptyTree, _LMDIndexes} = load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D, SW_StartLoad, TreeSize, UnitMins, @@ -477,7 +509,7 @@ recent_aae_noaae(_Config) -> % Go compare! Also confirm we're not comparing empty trees DL1_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, TicTacTreeJoined), - + DL1_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree), true = DL1_0 == [], true = length(DL1_1) == 0, @@ -489,7 +521,7 @@ recent_aae_noaae(_Config) -> recent_aae_allaae(_Config) -> - % Leveled is started in blacklisted mode with no buckets blacklisted. + % Leveled is started in blacklisted mode with no buckets blacklisted. % % A number of changes are then loaded into a store, and also partitioned % across a separate set of three stores. A merge tree is returned from @@ -503,12 +535,12 @@ recent_aae_allaae(_Config) -> % The segment Id found is then used in a query to find the Keys that make % up that segment, and the delta discovered should be just that one key % which was known to have been changed - + TreeSize = small, % SegmentCount = 256 * 256, UnitMins = 2, AAE = {blacklist, [], 60, UnitMins}, - + % Test requires multiple different databases, so want to mount them all % on individual file paths RootPathA = testutil:reset_filestructure("testA"), @@ -519,28 +551,28 @@ recent_aae_allaae(_Config) -> StartOptsB = aae_startopts(RootPathB, AAE), StartOptsC = aae_startopts(RootPathC, AAE), StartOptsD = aae_startopts(RootPathD, AAE), - + % Book1A to get all objects {ok, Book1A} = leveled_bookie:book_start(StartOptsA), % Book1B/C/D will have objects partitioned across it {ok, Book1B} = leveled_bookie:book_start(StartOptsB), {ok, Book1C} = leveled_bookie:book_start(StartOptsC), {ok, Book1D} = leveled_bookie:book_start(StartOptsD), - + {B1, K1, V1, S1, MD} = {"Bucket", "Key1.1.4567.4321", "Value1", [], [{"MDK1", "MDV1"}]}, {TestObject, TestSpec} = testutil:generate_testobject(B1, K1, V1, S1, MD), - + SW_StartLoad = os:timestamp(), - + ok = testutil:book_riakput(Book1A, TestObject, TestSpec), ok = testutil:book_riakput(Book1B, TestObject, TestSpec), testutil:check_forobject(Book1A, TestObject), testutil:check_forobject(Book1B, TestObject), - + {TicTacTreeJoined, TicTacTreeFull, EmptyTree, LMDIndexes} = load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D, SW_StartLoad, TreeSize, UnitMins, @@ -548,7 +580,7 @@ recent_aae_allaae(_Config) -> % Go compare! Also confirm we're not comparing empty trees DL1_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, TicTacTreeJoined), - + DL1_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree), true = DL1_0 == [], true = length(DL1_1) > 100, @@ -557,14 +589,14 @@ recent_aae_allaae(_Config) -> ok = leveled_bookie:book_close(Book1B), ok = leveled_bookie:book_close(Book1C), ok = leveled_bookie:book_close(Book1D), - + % Book2A to get all objects {ok, Book2A} = leveled_bookie:book_start(StartOptsA), % Book2B/C/D will have objects partitioned across it {ok, Book2B} = leveled_bookie:book_start(StartOptsB), {ok, Book2C} = leveled_bookie:book_start(StartOptsC), {ok, Book2D} = leveled_bookie:book_start(StartOptsD), - + {TicTacTreeJoined, TicTacTreeFull, EmptyTree, LMDIndexes} = load_and_check_recentaae(Book2A, Book2B, Book2C, Book2D, SW_StartLoad, TreeSize, UnitMins, @@ -572,21 +604,21 @@ recent_aae_allaae(_Config) -> % Go compare! Also confirm we're not comparing empty trees DL1_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, TicTacTreeJoined), - + DL1_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree), true = DL1_0 == [], true = length(DL1_1) > 100, - + V2 = "Value2", {TestObject2, TestSpec2} = testutil:generate_testobject(B1, K1, V2, S1, MD), - + New_startTS = os:timestamp(), - + ok = testutil:book_riakput(Book2B, TestObject2, TestSpec2), testutil:check_forobject(Book2B, TestObject2), testutil:check_forobject(Book2A, TestObject), - + New_endTS = os:timestamp(), NewLMDIndexes = determine_lmd_indexes(New_startTS, New_endTS, UnitMins), {TicTacTreeJoined2, TicTacTreeFull2, _EmptyTree, NewLMDIndexes} = @@ -595,10 +627,10 @@ recent_aae_allaae(_Config) -> NewLMDIndexes), DL2_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull2, TicTacTreeJoined2), - + % DL2_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree), true = length(DL2_0) == 1, - + [DirtySeg] = DL2_0, TermPrefix = string:right(integer_to_list(DirtySeg), 8, $0), @@ -631,18 +663,18 @@ recent_aae_allaae(_Config) -> {KeysTerms2D, _} = lists:foldl(LMDSegFolder, {[], Book2D}, lists:usort(LMDIndexes ++ NewLMDIndexes)), - + KeysTerms2Joined = KeysTerms2B ++ KeysTerms2C ++ KeysTerms2D, DeltaX = lists:subtract(KeysTerms2A, KeysTerms2Joined), DeltaY = lists:subtract(KeysTerms2Joined, KeysTerms2A), - + io:format("DeltaX ~w~n", [DeltaX]), io:format("DeltaY ~w~n", [DeltaY]), - + true = length(DeltaX) == 0, % This hasn't seen any extra changes true = length(DeltaY) == 1, % This has seen an extra change [{_, {B1, K1}}] = DeltaY, - + ok = leveled_bookie:book_close(Book2A), ok = leveled_bookie:book_close(Book2B), ok = leveled_bookie:book_close(Book2C), @@ -654,12 +686,12 @@ recent_aae_bucketaae(_Config) -> % Configure AAE to work only on a single whitelisted bucket % Confirm that we can spot a delta in this bucket, but not % in another bucket - + TreeSize = small, % SegmentCount = 256 * 256, UnitMins = 2, AAE = {whitelist, [<<"Bucket">>], 60, UnitMins}, - + % Test requires multiple different databases, so want to mount them all % on individual file paths RootPathA = testutil:reset_filestructure("testA"), @@ -670,28 +702,28 @@ recent_aae_bucketaae(_Config) -> StartOptsB = aae_startopts(RootPathB, AAE), StartOptsC = aae_startopts(RootPathC, AAE), StartOptsD = aae_startopts(RootPathD, AAE), - + % Book1A to get all objects {ok, Book1A} = leveled_bookie:book_start(StartOptsA), % Book1B/C/D will have objects partitioned across it {ok, Book1B} = leveled_bookie:book_start(StartOptsB), {ok, Book1C} = leveled_bookie:book_start(StartOptsC), {ok, Book1D} = leveled_bookie:book_start(StartOptsD), - + {B1, K1, V1, S1, MD} = {<<"Bucket">>, "Key1.1.4567.4321", "Value1", [], [{"MDK1", "MDV1"}]}, {TestObject, TestSpec} = testutil:generate_testobject(B1, K1, V1, S1, MD), - + SW_StartLoad = os:timestamp(), - + ok = testutil:book_riakput(Book1A, TestObject, TestSpec), ok = testutil:book_riakput(Book1B, TestObject, TestSpec), testutil:check_forobject(Book1A, TestObject), testutil:check_forobject(Book1B, TestObject), - + {TicTacTreeJoined, TicTacTreeFull, EmptyTree, LMDIndexes} = load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D, SW_StartLoad, TreeSize, UnitMins, @@ -699,7 +731,7 @@ recent_aae_bucketaae(_Config) -> % Go compare! Also confirm we're not comparing empty trees DL1_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, TicTacTreeJoined), - + DL1_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree), true = DL1_0 == [], true = length(DL1_1) > 100, @@ -708,27 +740,27 @@ recent_aae_bucketaae(_Config) -> ok = leveled_bookie:book_close(Book1B), ok = leveled_bookie:book_close(Book1C), ok = leveled_bookie:book_close(Book1D), - + % Book2A to get all objects {ok, Book2A} = leveled_bookie:book_start(StartOptsA), % Book2B/C/D will have objects partitioned across it {ok, Book2B} = leveled_bookie:book_start(StartOptsB), {ok, Book2C} = leveled_bookie:book_start(StartOptsC), {ok, Book2D} = leveled_bookie:book_start(StartOptsD), - + % Change the value for a key in another bucket % If we get trees for this period, no difference should be found - + V2 = "Value2", {TestObject2, TestSpec2} = testutil:generate_testobject(<<"NotBucket">>, K1, V2, S1, MD), - + New_startTS2 = os:timestamp(), - + ok = testutil:book_riakput(Book2B, TestObject2, TestSpec2), testutil:check_forobject(Book2B, TestObject2), testutil:check_forobject(Book2A, TestObject), - + New_endTS2 = os:timestamp(), NewLMDIndexes2 = determine_lmd_indexes(New_startTS2, New_endTS2, UnitMins), {TicTacTreeJoined2, TicTacTreeFull2, _EmptyTree, NewLMDIndexes2} = @@ -738,19 +770,19 @@ recent_aae_bucketaae(_Config) -> DL2_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull2, TicTacTreeJoined2), true = length(DL2_0) == 0, - + % Now create an object that is a change to an existing key in the % monitored bucket. A differrence should be found - + {TestObject3, TestSpec3} = testutil:generate_testobject(B1, K1, V2, S1, MD), - + New_startTS3 = os:timestamp(), - + ok = testutil:book_riakput(Book2B, TestObject3, TestSpec3), testutil:check_forobject(Book2B, TestObject3), testutil:check_forobject(Book2A, TestObject), - + New_endTS3 = os:timestamp(), NewLMDIndexes3 = determine_lmd_indexes(New_startTS3, New_endTS3, UnitMins), {TicTacTreeJoined3, TicTacTreeFull3, _EmptyTree, NewLMDIndexes3} = @@ -759,15 +791,15 @@ recent_aae_bucketaae(_Config) -> NewLMDIndexes3, <<"Bucket">>), DL3_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull3, TicTacTreeJoined3), - + % DL2_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree), true = length(DL3_0) == 1, - + % Find the dirty segment, and use that to find the dirty key % % Note that unlike when monitoring $all, fold_keys can be used as there % is no need to return the Bucket (as hte bucket is known) - + [DirtySeg] = DL3_0, TermPrefix = string:right(integer_to_list(DirtySeg), 8, $0), @@ -800,18 +832,18 @@ recent_aae_bucketaae(_Config) -> {KeysTerms2D, _} = lists:foldl(LMDSegFolder, {[], Book2D}, lists:usort(LMDIndexes ++ NewLMDIndexes3)), - + KeysTerms2Joined = KeysTerms2B ++ KeysTerms2C ++ KeysTerms2D, DeltaX = lists:subtract(KeysTerms2A, KeysTerms2Joined), DeltaY = lists:subtract(KeysTerms2Joined, KeysTerms2A), - + io:format("DeltaX ~w~n", [DeltaX]), io:format("DeltaY ~w~n", [DeltaY]), - + true = length(DeltaX) == 0, % This hasn't seen any extra changes true = length(DeltaY) == 1, % This has seen an extra change [{_, K1}] = DeltaY, - + ok = leveled_bookie:book_close(Book2A), ok = leveled_bookie:book_close(Book2B), ok = leveled_bookie:book_close(Book2C), @@ -820,21 +852,21 @@ recent_aae_bucketaae(_Config) -> recent_aae_expiry(_Config) -> % Proof that the index entries are indeed expired - + TreeSize = small, % SegmentCount = 256 * 256, UnitMins = 1, TotalMins = 2, AAE = {blacklist, [], TotalMins, UnitMins}, - + % Test requires multiple different databases, so want to mount them all % on individual file paths RootPathA = testutil:reset_filestructure("testA"), StartOptsA = aae_startopts(RootPathA, AAE), - + % Book1A to get all objects {ok, Book1A} = leveled_bookie:book_start(StartOptsA), - + GenMapFun = fun(_X) -> V = testutil:get_compressiblevalue(), @@ -845,13 +877,13 @@ recent_aae_expiry(_Config) -> V, Indexes) end, - + ObjLists = lists:map(GenMapFun, lists:seq(1, 3)), - + SW0 = os:timestamp(), % Load all nine lists into Book1A lists:foreach(fun(ObjL) -> testutil:riakload(Book1A, ObjL) end, - ObjLists), + ObjLists), SW1 = os:timestamp(), % sleep for two minutes, so all index entries will have expired GetTicTacTreeFun = @@ -860,19 +892,19 @@ recent_aae_expiry(_Config) -> end, EmptyTree = leveled_tictac:new_tree(empty, TreeSize), LMDIndexes = determine_lmd_indexes(SW0, SW1, UnitMins), - + % Should get a non-empty answer to the query TicTacTree1_Full = - lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes), + lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes), DL3_0 = leveled_tictac:find_dirtyleaves(TicTacTree1_Full, EmptyTree), io:format("Dirty leaves found before expiry ~w~n", [length(DL3_0)]), true = length(DL3_0) > 0, - + SecondsSinceLMD = timer:now_diff(os:timestamp(), SW0) div 1000000, SecondsToExpiry = (TotalMins + UnitMins) * 60, - - io:format("SecondsToExpiry ~w SecondsSinceLMD ~w~n", + + io:format("SecondsToExpiry ~w SecondsSinceLMD ~w~n", [SecondsToExpiry, SecondsSinceLMD]), io:format("LMDIndexes ~w~n", [LMDIndexes]), @@ -882,17 +914,17 @@ recent_aae_expiry(_Config) -> false -> timer:sleep(1000) end, - + % Should now get an empty answer - all entries have expired TicTacTree2_Full = - lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes), + lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes), DL4_0 = leveled_tictac:find_dirtyleaves(TicTacTree2_Full, EmptyTree), io:format("Dirty leaves found after expiry ~w~n", [length(DL4_0)]), timer:sleep(10000), TicTacTree3_Full = - lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes), + lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes), DL5_0 = leveled_tictac:find_dirtyleaves(TicTacTree3_Full, EmptyTree), io:format("Dirty leaves found after expiry plus 10s ~w~n", [length(DL5_0)]), @@ -913,7 +945,7 @@ load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D, load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D, SW_StartLoad, TreeSize, UnitMins, LMDIndexes_Loaded, Bucket) -> - LMDIndexes = + LMDIndexes = case LMDIndexes_Loaded of false -> % Generate nine lists of objects @@ -928,13 +960,13 @@ load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D, V, Indexes) end, - + ObjLists = lists:map(GenMapFun, lists:seq(1, 9)), - + % Load all nine lists into Book1A lists:foreach(fun(ObjL) -> testutil:riakload(Book1A, ObjL) end, - ObjLists), - + ObjLists), + % Split nine lists across Book1B to Book1D, three object lists % in each lists:foreach(fun(ObjL) -> testutil:riakload(Book1B, ObjL) end, @@ -943,38 +975,38 @@ load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D, lists:sublist(ObjLists, 4, 3)), lists:foreach(fun(ObjL) -> testutil:riakload(Book1D, ObjL) end, lists:sublist(ObjLists, 7, 3)), - + SW_EndLoad = os:timestamp(), determine_lmd_indexes(SW_StartLoad, SW_EndLoad, UnitMins); _ -> LMDIndexes_Loaded end, - + EmptyTree = leveled_tictac:new_tree(empty, TreeSize), - + GetTicTacTreeFun = fun(Bookie) -> get_tictactree_fun(Bookie, Bucket, TreeSize) end, - + % Get a TicTac tree representing one of the indexes in Bucket A TicTacTree1_Full = lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes), - + TicTacTree1_P1 = lists:foldl(GetTicTacTreeFun(Book1B), EmptyTree, LMDIndexes), TicTacTree1_P2 = lists:foldl(GetTicTacTreeFun(Book1C), EmptyTree, LMDIndexes), TicTacTree1_P3 = lists:foldl(GetTicTacTreeFun(Book1D), EmptyTree, LMDIndexes), - + % Merge the tree across the partitions TicTacTree1_Joined = lists:foldl(fun leveled_tictac:merge_trees/2, TicTacTree1_P1, [TicTacTree1_P2, TicTacTree1_P3]), - + {TicTacTree1_Full, TicTacTree1_Joined, EmptyTree, LMDIndexes}. - + aae_startopts(RootPath, AAE) -> LS = 2000, @@ -992,7 +1024,7 @@ determine_lmd_indexes(StartTS, EndTS, UnitMins) -> EndDT = calendar:now_to_datetime(EndTS), StartTimeStr = get_strtime(StartDT, UnitMins), EndTimeStr = get_strtime(EndDT, UnitMins), - + AddTimeFun = fun(X, Acc) -> case lists:member(EndTimeStr, Acc) of @@ -1007,10 +1039,10 @@ determine_lmd_indexes(StartTS, EndTS, UnitMins) -> Acc ++ [get_strtime(NextDT, UnitMins)] end end, - - lists:foldl(AddTimeFun, [StartTimeStr], lists:seq(1, 10)). - - + + lists:foldl(AddTimeFun, [StartTimeStr], lists:seq(1, 10)). + + get_strtime(DateTime, UnitMins) -> {{Y, M, D}, {Hour, Minute, _Second}} = DateTime, RoundMins =