foldheads_bybucket adds segment list support

Accelerate queries for foldheads_bybucket as well

src/leveled_bookie.erl

@@ -679,12 +679,6 @@ get_runner(State, {keylist, Tag, Bucket, FoldAccT}) ->
     leveled_runner:bucketkey_query(SnapFun, Tag, Bucket, FoldAccT);
 
 %% Set of runners for object or metadata folds
-get_runner(State, 
-            {foldheads_allkeys, Tag, FoldFun, JournalCheck, SnapPreFold}) ->
-    SnapType = snaptype_by_presence(JournalCheck),
-    SnapFun = return_snapfun(State, SnapType, no_lookup, true, SnapPreFold),
-    leveled_runner:foldheads_allkeys(SnapFun, Tag, FoldFun, 
-                                        JournalCheck, false);
 get_runner(State, 
             {foldheads_allkeys, 
                 Tag, FoldFun, 
@@ -702,14 +696,14 @@ get_runner(State,
             {foldheads_bybucket, 
                 Tag, Bucket, KeyRange, 
                 FoldFun, 
-                JournalCheck, SnapPreFold}) ->
+                JournalCheck, SnapPreFold, SegmentList}) ->
     {StartKey, EndKey, SnapQ} = return_ledger_keyrange(Tag, Bucket, KeyRange),
     SnapType = snaptype_by_presence(JournalCheck),
     SnapFun = return_snapfun(State, SnapType, SnapQ, true, SnapPreFold),
     leveled_runner:foldheads_bybucket(SnapFun, 
                                         {Tag, StartKey, EndKey}, 
                                         FoldFun, 
-                                        JournalCheck);
+                                        JournalCheck, SegmentList);
 get_runner(State,
             {foldobjects_bybucket, 
                 Tag, Bucket, KeyRange, 
@@ -1322,8 +1316,7 @@ foldobjects_vs_hashtree_testto() ->
                             {foldheads_allkeys, 
                                 ?STD_TAG, 
                                 FoldHeadsFun, 
-                                true,
+                                true, true, false}),
-                                true}),
     KeyHashList3 = HTFolder3(),
     ?assertMatch(KeyHashList1, lists:usort(KeyHashList3)),
 
@@ -1342,8 +1335,7 @@ foldobjects_vs_hashtree_testto() ->
                             {foldheads_allkeys, 
                                 ?STD_TAG, 
                                 FoldHeadsFun2,
-                                false,
+                                false, false, false}),
-                                false}),
     KeyHashList4 = HTFolder4(),
     ?assertMatch(KeyHashList1, lists:usort(KeyHashList4)),
 
@@ -1413,8 +1405,7 @@ foldobjects_vs_foldheads_bybucket_testto() ->
                                 "BucketA",
                                 all,
                                 FoldHeadsFun,
-                                true,
+                                true, true, false}),
-                                true}),
     KeyHashList2A = HTFolder2A(),
     {async, HTFolder2B} =
         book_returnfolder(Bookie1,
@@ -1423,8 +1414,7 @@ foldobjects_vs_foldheads_bybucket_testto() ->
                                 "BucketB",
                                 all,
                                 FoldHeadsFun,
-                                true,
+                                true, false, false}),
-                                false}),
     KeyHashList2B = HTFolder2B(),
     
     ?assertMatch(true,
@@ -1439,8 +1429,7 @@ foldobjects_vs_foldheads_bybucket_testto() ->
                                 "BucketB",
                                 {"Key", <<"$all">>},
                                 FoldHeadsFun,
-                                true,
+                                true, false, false}),
-                                false}),
     KeyHashList2C = HTFolder2C(),
     {async, HTFolder2D} =
         book_returnfolder(Bookie1,
@@ -1449,8 +1438,7 @@ foldobjects_vs_foldheads_bybucket_testto() ->
                                 "BucketB",
                                 {"Key", "Keyzzzzz"},
                                 FoldHeadsFun,
-                                true,
+                                true, true, false}),
-                                true}),
     KeyHashList2D = HTFolder2D(),
     ?assertMatch(true,
                     lists:usort(KeyHashList2B) == lists:usort(KeyHashList2C)),
@@ -1464,8 +1452,7 @@ foldobjects_vs_foldheads_bybucket_testto() ->
                                 "BucketB",
                                 {"Key", "Key4zzzz"},
                                 FoldHeadsFun,
-                                true,
+                                true, false, false}),
-                                false}),
     KeyHashList2E = HTFolder2E(),
     {async, HTFolder2F} =
         book_returnfolder(Bookie1,
@@ -1474,8 +1461,7 @@ foldobjects_vs_foldheads_bybucket_testto() ->
                                 "BucketB",
                                 {"Key5", <<"all">>},
                                 FoldHeadsFun,
-                                true,
+                                true, false, false}),
-                                false}),
     KeyHashList2F = HTFolder2F(),
 
     ?assertMatch(true, length(KeyHashList2E) > 0),

src/leveled_runner.erl

@@ -30,7 +30,7 @@
             tictactree/5,
             foldheads_allkeys/5,
             foldobjects_allkeys/3,
-            foldheads_bybucket/4,
+            foldheads_bybucket/5,
             foldobjects_bybucket/3,
             foldobjects_byindex/3
         ]).
@@ -247,15 +247,21 @@ foldobjects_bybucket(SnapFun, {Tag, StartKey, EndKey}, FoldFun) ->
                 FoldFun, 
                 false, false).
 
--spec foldheads_bybucket(fun(), {atom(), any(), any()}, fun(), boolean()) -> 
+-spec foldheads_bybucket(fun(), 
-                                                                {async, fun()}.
+                            {atom(), any(), any()}, 
+                            fun(), 
+                            boolean(), false|list(integer())) 
+                                                        -> {async, fun()}.
 %% @doc
 %% Fold over all object metadata within a given key range in a bucket
-foldheads_bybucket(SnapFun, {Tag, StartKey, EndKey}, FoldFun, JournalCheck) ->
+foldheads_bybucket(SnapFun, 
+                    {Tag, StartKey, EndKey}, 
+                    FoldFun, 
+                    JournalCheck, SegmentList) ->
     foldobjects(SnapFun, 
                 Tag, StartKey, EndKey, 
                 FoldFun, 
-                {true, JournalCheck}, false).
+                {true, JournalCheck}, SegmentList).
 
 -spec foldobjects_byindex(fun(), tuple(), fun()) -> {async, fun()}.
 %% @doc

test/end_to_end/recovery_SUITE.erl

@@ -117,8 +117,7 @@ aae_missingjournal(_Config) ->
                                             {foldheads_allkeys,
                                                 ?RIAK_TAG,
                                                 FoldHeadsFun,
-                                                true,
+                                                true, true, false}),
-                                                true}),
     HeadL1 = length(AllHeadF1()),
     io:format("Fold head returned ~w objects~n", [HeadL1]),
     
@@ -135,8 +134,7 @@ aae_missingjournal(_Config) ->
                                             {foldheads_allkeys,
                                                 ?RIAK_TAG,
                                                 FoldHeadsFun,
-                                                true,
+                                                true, true, false}),
-                                                true}),
     HeadL2 = length(AllHeadF2()),
     io:format("Fold head returned ~w objects~n", [HeadL2]),
     true = HeadL2 < HeadL1,

test/end_to_end/riak_SUITE.erl

@@ -78,47 +78,37 @@ crossbucket_aae(_Config) ->
 
 test_singledelta_stores(BookA, BookB, TreeSize, DeltaKey) ->
     io:format("Test for single delta with tree size ~w~n", [TreeSize]),
-    {B1, K1} = DeltaKey,
     % Now run a tictac query against both stores to see the extent to which
     % state between stores is consistent
-    HeadTicTacFolder = 
+    TicTacFolder = 
         {foldheads_allkeys,
             ?RIAK_TAG,
             {fun head_tictac_foldfun/4, 
                 {0, leveled_tictac:new_tree(test, TreeSize)}},
-            false,
+            false, true, false},
-            true},
+    % tictac query by bucket (should be same result as all stores)
+    TicTacByBucketFolder = 
+        {foldheads_bybucket, 
+                ?RIAK_TAG, <<"Bucket">>, 
+                all, 
+                {fun head_tictac_foldfun/4, 
+                    {0, leveled_tictac:new_tree(test, TreeSize)}},
+                false, false, false},
 
-    SW_TT0 = os:timestamp(),
+    DLs = check_tictacfold(BookA, BookB, 
-    {async, BookATreeFolder} =
+                            TicTacFolder, 
-        leveled_bookie:book_returnfolder(BookA, HeadTicTacFolder),
+                            DeltaKey, 
-    {async, BookBTreeFolder} =
+                            TreeSize),
-        leveled_bookie:book_returnfolder(BookB, HeadTicTacFolder),
+    DLs = check_tictacfold(BookA, BookB, 
-    {CountA, BookATree} = BookATreeFolder(),
+                            TicTacByBucketFolder, 
-    {CountB, BookBTree} = BookBTreeFolder(),
+                            DeltaKey, 
-    Time_TT0 = timer:now_diff(os:timestamp(), SW_TT0)/1000,
+                            TreeSize),
-    io:format("Two tree folds took ~w milliseconds ~n", [Time_TT0]),
-
-    io:format("Fold over keys revealed counts of ~w and ~w~n", 
-                [CountA, CountB]),
-
-    % There should be a single delta between the stores
-    1 = CountA - CountB,
-
-    DLs = leveled_tictac:find_dirtyleaves(BookATree, BookBTree),
-    io:format("Found dirty leaves with Riak fold_heads of ~w~n",
-                [length(DLs)]),
-    true = length(DLs) == 1,
-    ExpSeg = leveled_tictac:keyto_segment32(<<B1/binary, K1/binary>>),
-    TreeSeg = leveled_tictac:get_segment(ExpSeg, TreeSize),
-    [ActualSeg] = DLs,
-    true = TreeSeg == ActualSeg,
     
     HeadSegmentFolder = 
         {foldheads_allkeys,
             ?RIAK_TAG,
             {get_segment_folder(DLs, TreeSize),  []},
-            false, true},
+            false, true, false},
     
     SW_SL0 = os:timestamp(),
     {async, BookASegFolder} =
@@ -229,6 +219,34 @@ head_tictac_foldfun(B, K, PO, {Count, TreeAcc}) ->
         leveled_tictac:add_kv(TreeAcc, {B, K}, PO, ExtractFun)}.
 
 
+check_tictacfold(BookA, BookB, HeadTicTacFolder, {B1, K1}, TreeSize) ->
+    SW_TT0 = os:timestamp(),
+    {async, BookATreeFolder} =
+        leveled_bookie:book_returnfolder(BookA, HeadTicTacFolder),
+    {async, BookBTreeFolder} =
+        leveled_bookie:book_returnfolder(BookB, HeadTicTacFolder),
+    {CountA, BookATree} = BookATreeFolder(),
+    {CountB, BookBTree} = BookBTreeFolder(),
+    Time_TT0 = timer:now_diff(os:timestamp(), SW_TT0)/1000,
+    io:format("Two tree folds took ~w milliseconds ~n", [Time_TT0]),
+
+    io:format("Fold over keys revealed counts of ~w and ~w~n", 
+                [CountA, CountB]),
+
+    % There should be a single delta between the stores
+    1 = CountA - CountB,
+
+    DLs = leveled_tictac:find_dirtyleaves(BookATree, BookBTree),
+    io:format("Found dirty leaves with Riak fold_heads of ~w~n",
+                [length(DLs)]),
+    true = length(DLs) == 1,
+    ExpSeg = leveled_tictac:keyto_segment32(<<B1/binary, K1/binary>>),
+    TreeSeg = leveled_tictac:get_segment(ExpSeg, TreeSize),
+    [ActualSeg] = DLs,
+    true = TreeSeg == ActualSeg,
+    DLs.
+
+
 summary_from_binary(<<131, _Rest/binary>>=ObjBin) ->
     {proxy_object, HeadBin, ObjSize, _Fetcher} = binary_to_term(ObjBin),
     summary_from_binary(HeadBin, ObjSize);

test/end_to_end/tictac_SUITE.erl

@@ -150,7 +150,7 @@ many_put_compare(_Config) ->
               "Bucket",
               all,
               {FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)},
-              false, true},
+              false, true, false},
     {async, TreeAObjFolder0} =
         leveled_bookie:book_returnfolder(Bookie2, FoldQ0),
     SWB0Obj = os:timestamp(),
@@ -165,7 +165,7 @@ many_put_compare(_Config) ->
               "Bucket",
               all,
               {FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)},
-              true, true},
+              true, true, false},
     {async, TreeAObjFolder1} =
         leveled_bookie:book_returnfolder(Bookie2, FoldQ1),
     SWB1Obj = os:timestamp(),
@@ -193,8 +193,7 @@ many_put_compare(_Config) ->
                     "Bucket",
                     all,
                     {AltFoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)},
-                    false, 
+                    false, true, false},
-                    true},
     {async, TreeAAltObjFolder0} =
         leveled_bookie:book_returnfolder(Bookie2, AltFoldQ0),
     SWB2Obj = os:timestamp(),