From c8ad39b33b55f56fa8367aac0293c45cfba8f3a6 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 1 Nov 2017 22:00:12 +0000 Subject: [PATCH] foldheads_bybucket adds segment list support Accelerate queries for foldheads_bybucket as well --- src/leveled_bookie.erl | 34 ++++--------- src/leveled_runner.erl | 16 +++++-- test/end_to_end/recovery_SUITE.erl | 6 +-- test/end_to_end/riak_SUITE.erl | 76 ++++++++++++++++++------------ test/end_to_end/tictac_SUITE.erl | 7 ++- 5 files changed, 73 insertions(+), 66 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index c09087b..a74c734 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -679,12 +679,6 @@ get_runner(State, {keylist, Tag, Bucket, FoldAccT}) -> leveled_runner:bucketkey_query(SnapFun, Tag, Bucket, FoldAccT); %% Set of runners for object or metadata folds -get_runner(State, - {foldheads_allkeys, Tag, FoldFun, JournalCheck, SnapPreFold}) -> - SnapType = snaptype_by_presence(JournalCheck), - SnapFun = return_snapfun(State, SnapType, no_lookup, true, SnapPreFold), - leveled_runner:foldheads_allkeys(SnapFun, Tag, FoldFun, - JournalCheck, false); get_runner(State, {foldheads_allkeys, Tag, FoldFun, @@ -702,14 +696,14 @@ get_runner(State, {foldheads_bybucket, Tag, Bucket, KeyRange, FoldFun, - JournalCheck, SnapPreFold}) -> + JournalCheck, SnapPreFold, SegmentList}) -> {StartKey, EndKey, SnapQ} = return_ledger_keyrange(Tag, Bucket, KeyRange), SnapType = snaptype_by_presence(JournalCheck), SnapFun = return_snapfun(State, SnapType, SnapQ, true, SnapPreFold), leveled_runner:foldheads_bybucket(SnapFun, {Tag, StartKey, EndKey}, FoldFun, - JournalCheck); + JournalCheck, SegmentList); get_runner(State, {foldobjects_bybucket, Tag, Bucket, KeyRange, @@ -1322,8 +1316,7 @@ foldobjects_vs_hashtree_testto() -> {foldheads_allkeys, ?STD_TAG, FoldHeadsFun, - true, - true}), + true, true, false}), KeyHashList3 = HTFolder3(), ?assertMatch(KeyHashList1, lists:usort(KeyHashList3)), @@ -1342,8 +1335,7 @@ foldobjects_vs_hashtree_testto() -> {foldheads_allkeys, ?STD_TAG, FoldHeadsFun2, - false, - false}), + false, false, false}), KeyHashList4 = HTFolder4(), ?assertMatch(KeyHashList1, lists:usort(KeyHashList4)), @@ -1413,8 +1405,7 @@ foldobjects_vs_foldheads_bybucket_testto() -> "BucketA", all, FoldHeadsFun, - true, - true}), + true, true, false}), KeyHashList2A = HTFolder2A(), {async, HTFolder2B} = book_returnfolder(Bookie1, @@ -1423,8 +1414,7 @@ foldobjects_vs_foldheads_bybucket_testto() -> "BucketB", all, FoldHeadsFun, - true, - false}), + true, false, false}), KeyHashList2B = HTFolder2B(), ?assertMatch(true, @@ -1439,8 +1429,7 @@ foldobjects_vs_foldheads_bybucket_testto() -> "BucketB", {"Key", <<"$all">>}, FoldHeadsFun, - true, - false}), + true, false, false}), KeyHashList2C = HTFolder2C(), {async, HTFolder2D} = book_returnfolder(Bookie1, @@ -1449,8 +1438,7 @@ foldobjects_vs_foldheads_bybucket_testto() -> "BucketB", {"Key", "Keyzzzzz"}, FoldHeadsFun, - true, - true}), + true, true, false}), KeyHashList2D = HTFolder2D(), ?assertMatch(true, lists:usort(KeyHashList2B) == lists:usort(KeyHashList2C)), @@ -1464,8 +1452,7 @@ foldobjects_vs_foldheads_bybucket_testto() -> "BucketB", {"Key", "Key4zzzz"}, FoldHeadsFun, - true, - false}), + true, false, false}), KeyHashList2E = HTFolder2E(), {async, HTFolder2F} = book_returnfolder(Bookie1, @@ -1474,8 +1461,7 @@ foldobjects_vs_foldheads_bybucket_testto() -> "BucketB", {"Key5", <<"all">>}, FoldHeadsFun, - true, - false}), + true, false, false}), KeyHashList2F = HTFolder2F(), ?assertMatch(true, length(KeyHashList2E) > 0), diff --git a/src/leveled_runner.erl b/src/leveled_runner.erl index 1cdfc10..710aba7 100644 --- a/src/leveled_runner.erl +++ b/src/leveled_runner.erl @@ -30,7 +30,7 @@ tictactree/5, foldheads_allkeys/5, foldobjects_allkeys/3, - foldheads_bybucket/4, + foldheads_bybucket/5, foldobjects_bybucket/3, foldobjects_byindex/3 ]). @@ -247,15 +247,21 @@ foldobjects_bybucket(SnapFun, {Tag, StartKey, EndKey}, FoldFun) -> FoldFun, false, false). --spec foldheads_bybucket(fun(), {atom(), any(), any()}, fun(), boolean()) -> - {async, fun()}. +-spec foldheads_bybucket(fun(), + {atom(), any(), any()}, + fun(), + boolean(), false|list(integer())) + -> {async, fun()}. %% @doc %% Fold over all object metadata within a given key range in a bucket -foldheads_bybucket(SnapFun, {Tag, StartKey, EndKey}, FoldFun, JournalCheck) -> +foldheads_bybucket(SnapFun, + {Tag, StartKey, EndKey}, + FoldFun, + JournalCheck, SegmentList) -> foldobjects(SnapFun, Tag, StartKey, EndKey, FoldFun, - {true, JournalCheck}, false). + {true, JournalCheck}, SegmentList). -spec foldobjects_byindex(fun(), tuple(), fun()) -> {async, fun()}. %% @doc diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index 8df2e21..842ec7f 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -117,8 +117,7 @@ aae_missingjournal(_Config) -> {foldheads_allkeys, ?RIAK_TAG, FoldHeadsFun, - true, - true}), + true, true, false}), HeadL1 = length(AllHeadF1()), io:format("Fold head returned ~w objects~n", [HeadL1]), @@ -135,8 +134,7 @@ aae_missingjournal(_Config) -> {foldheads_allkeys, ?RIAK_TAG, FoldHeadsFun, - true, - true}), + true, true, false}), HeadL2 = length(AllHeadF2()), io:format("Fold head returned ~w objects~n", [HeadL2]), true = HeadL2 < HeadL1, diff --git a/test/end_to_end/riak_SUITE.erl b/test/end_to_end/riak_SUITE.erl index 1459966..7c979da 100644 --- a/test/end_to_end/riak_SUITE.erl +++ b/test/end_to_end/riak_SUITE.erl @@ -78,47 +78,37 @@ crossbucket_aae(_Config) -> test_singledelta_stores(BookA, BookB, TreeSize, DeltaKey) -> io:format("Test for single delta with tree size ~w~n", [TreeSize]), - {B1, K1} = DeltaKey, % Now run a tictac query against both stores to see the extent to which % state between stores is consistent - HeadTicTacFolder = + TicTacFolder = {foldheads_allkeys, ?RIAK_TAG, {fun head_tictac_foldfun/4, {0, leveled_tictac:new_tree(test, TreeSize)}}, - false, - true}, + false, true, false}, + % tictac query by bucket (should be same result as all stores) + TicTacByBucketFolder = + {foldheads_bybucket, + ?RIAK_TAG, <<"Bucket">>, + all, + {fun head_tictac_foldfun/4, + {0, leveled_tictac:new_tree(test, TreeSize)}}, + false, false, false}, - SW_TT0 = os:timestamp(), - {async, BookATreeFolder} = - leveled_bookie:book_returnfolder(BookA, HeadTicTacFolder), - {async, BookBTreeFolder} = - leveled_bookie:book_returnfolder(BookB, HeadTicTacFolder), - {CountA, BookATree} = BookATreeFolder(), - {CountB, BookBTree} = BookBTreeFolder(), - Time_TT0 = timer:now_diff(os:timestamp(), SW_TT0)/1000, - io:format("Two tree folds took ~w milliseconds ~n", [Time_TT0]), - - io:format("Fold over keys revealed counts of ~w and ~w~n", - [CountA, CountB]), - - % There should be a single delta between the stores - 1 = CountA - CountB, - - DLs = leveled_tictac:find_dirtyleaves(BookATree, BookBTree), - io:format("Found dirty leaves with Riak fold_heads of ~w~n", - [length(DLs)]), - true = length(DLs) == 1, - ExpSeg = leveled_tictac:keyto_segment32(<>), - TreeSeg = leveled_tictac:get_segment(ExpSeg, TreeSize), - [ActualSeg] = DLs, - true = TreeSeg == ActualSeg, + DLs = check_tictacfold(BookA, BookB, + TicTacFolder, + DeltaKey, + TreeSize), + DLs = check_tictacfold(BookA, BookB, + TicTacByBucketFolder, + DeltaKey, + TreeSize), HeadSegmentFolder = {foldheads_allkeys, ?RIAK_TAG, {get_segment_folder(DLs, TreeSize), []}, - false, true}, + false, true, false}, SW_SL0 = os:timestamp(), {async, BookASegFolder} = @@ -229,6 +219,34 @@ head_tictac_foldfun(B, K, PO, {Count, TreeAcc}) -> leveled_tictac:add_kv(TreeAcc, {B, K}, PO, ExtractFun)}. +check_tictacfold(BookA, BookB, HeadTicTacFolder, {B1, K1}, TreeSize) -> + SW_TT0 = os:timestamp(), + {async, BookATreeFolder} = + leveled_bookie:book_returnfolder(BookA, HeadTicTacFolder), + {async, BookBTreeFolder} = + leveled_bookie:book_returnfolder(BookB, HeadTicTacFolder), + {CountA, BookATree} = BookATreeFolder(), + {CountB, BookBTree} = BookBTreeFolder(), + Time_TT0 = timer:now_diff(os:timestamp(), SW_TT0)/1000, + io:format("Two tree folds took ~w milliseconds ~n", [Time_TT0]), + + io:format("Fold over keys revealed counts of ~w and ~w~n", + [CountA, CountB]), + + % There should be a single delta between the stores + 1 = CountA - CountB, + + DLs = leveled_tictac:find_dirtyleaves(BookATree, BookBTree), + io:format("Found dirty leaves with Riak fold_heads of ~w~n", + [length(DLs)]), + true = length(DLs) == 1, + ExpSeg = leveled_tictac:keyto_segment32(<>), + TreeSeg = leveled_tictac:get_segment(ExpSeg, TreeSize), + [ActualSeg] = DLs, + true = TreeSeg == ActualSeg, + DLs. + + summary_from_binary(<<131, _Rest/binary>>=ObjBin) -> {proxy_object, HeadBin, ObjSize, _Fetcher} = binary_to_term(ObjBin), summary_from_binary(HeadBin, ObjSize); diff --git a/test/end_to_end/tictac_SUITE.erl b/test/end_to_end/tictac_SUITE.erl index 5ef92d4..506704c 100644 --- a/test/end_to_end/tictac_SUITE.erl +++ b/test/end_to_end/tictac_SUITE.erl @@ -150,7 +150,7 @@ many_put_compare(_Config) -> "Bucket", all, {FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)}, - false, true}, + false, true, false}, {async, TreeAObjFolder0} = leveled_bookie:book_returnfolder(Bookie2, FoldQ0), SWB0Obj = os:timestamp(), @@ -165,7 +165,7 @@ many_put_compare(_Config) -> "Bucket", all, {FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)}, - true, true}, + true, true, false}, {async, TreeAObjFolder1} = leveled_bookie:book_returnfolder(Bookie2, FoldQ1), SWB1Obj = os:timestamp(), @@ -193,8 +193,7 @@ many_put_compare(_Config) -> "Bucket", all, {AltFoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)}, - false, - true}, + false, true, false}, {async, TreeAAltObjFolder0} = leveled_bookie:book_returnfolder(Bookie2, AltFoldQ0), SWB2Obj = os:timestamp(),