diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index e0d12fa..dcfb6cd 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -864,8 +864,9 @@ book_objectfold(Pid, Tag, Bucket, Limiter, FoldAccT, SnapPreFold) -> SegmentList :: false | list(integer()), Runner :: fun(() -> Acc). book_headfold(Pid, Tag, FoldAccT, JournalCheck, SnapPreFold, SegmentList) -> - RunnerType = {foldheads_allkeys, Tag, FoldAccT, JournalCheck, SnapPreFold, SegmentList}, - book_returnfolder(Pid, RunnerType). + book_headfold(Pid, Tag, all, + FoldAccT, JournalCheck, SnapPreFold, + SegmentList, false, false). %% @doc as book_headfold/6, but with the addition of a `Limiter' that %% restricts the set of objects folded over. `Limiter' can either be a @@ -897,16 +898,10 @@ book_headfold(Pid, Tag, FoldAccT, JournalCheck, SnapPreFold, SegmentList) -> SnapPreFold :: boolean(), SegmentList :: false | list(integer()), Runner :: fun(() -> Acc). -book_headfold(Pid, Tag, {bucket_list, BucketList}, FoldAccT, JournalCheck, SnapPreFold, SegmentList) -> - RunnerType = - {foldheads_bybucket, Tag, BucketList, bucket_list, FoldAccT, - JournalCheck, SnapPreFold, SegmentList, false, false}, - book_returnfolder(Pid, RunnerType); -book_headfold(Pid, Tag, {range, Bucket, KeyRange}, FoldAccT, JournalCheck, SnapPreFold, SegmentList) -> - RunnerType = - {foldheads_bybucket, Tag, Bucket, KeyRange, FoldAccT, - JournalCheck, SnapPreFold, SegmentList, false, false}, - book_returnfolder(Pid, RunnerType). +book_headfold(Pid, Tag, Limiter, FoldAccT, JournalCheck, SnapPreFold, SegmentList) -> + book_headfold(Pid, Tag, Limiter, + FoldAccT, JournalCheck, SnapPreFold, + SegmentList, false, false). %% @doc as book_headfold/7, but with the addition of a Last Modified Date %% Range and Max Object Count. For version 2 objects this will filter out @@ -927,7 +922,7 @@ book_headfold(Pid, Tag, {range, Bucket, KeyRange}, FoldAccT, JournalCheck, SnapP SegmentList, LastModRange, MaxObjectCount) -> {async, Runner} when Tag :: leveled_codec:tag(), - Limiter :: BucketList | BucketKeyRange, + Limiter :: BucketList | BucketKeyRange | all, BucketList :: {bucket_list, list(Bucket)}, BucketKeyRange :: {range, Bucket, KeyRange}, KeyRange :: {StartKey, EndKey} | all, @@ -953,12 +948,17 @@ book_headfold(Pid, Tag, {bucket_list, BucketList}, FoldAccT, JournalCheck, SnapP SegmentList, LastModRange, MaxObjectCount}, book_returnfolder(Pid, RunnerType); book_headfold(Pid, Tag, {range, Bucket, KeyRange}, FoldAccT, JournalCheck, SnapPreFold, - SegmentList, LastModRange, MaxObjectCount) -> - + SegmentList, LastModRange, MaxObjectCount) -> RunnerType = {foldheads_bybucket, Tag, Bucket, KeyRange, FoldAccT, JournalCheck, SnapPreFold, SegmentList, LastModRange, MaxObjectCount}, + book_returnfolder(Pid, RunnerType); +book_headfold(Pid, Tag, all, FoldAccT, JournalCheck, SnapPreFold, + SegmentList, LastModRange, MaxObjectCount) -> + RunnerType = {foldheads_allkeys, Tag, FoldAccT, + JournalCheck, SnapPreFold, + SegmentList, LastModRange, MaxObjectCount}, book_returnfolder(Pid, RunnerType). -spec book_snapshot(pid(), @@ -1615,12 +1615,14 @@ get_runner(State, {keylist, Tag, Bucket, KeyRange, FoldAccT, TermRegex}) -> get_runner(State, {foldheads_allkeys, Tag, FoldFun, - JournalCheck, SnapPreFold, SegmentList}) -> + JournalCheck, SnapPreFold, SegmentList, + LastModRange, MaxObjectCount}) -> SnapType = snaptype_by_presence(JournalCheck), SnapFun = return_snapfun(State, SnapType, no_lookup, true, SnapPreFold), leveled_runner:foldheads_allkeys(SnapFun, Tag, FoldFun, - JournalCheck, SegmentList); + JournalCheck, SegmentList, + LastModRange, MaxObjectCount); get_runner(State, {foldobjects_allkeys, Tag, FoldFun, SnapPreFold}) -> get_runner(State, @@ -2494,7 +2496,7 @@ foldobjects_vs_hashtree_testto() -> {foldheads_allkeys, ?STD_TAG, FoldHeadsFun, - true, true, false}), + true, true, false, false, false}), KeyHashList3 = HTFolder3(), ?assertMatch(KeyHashList1, lists:usort(KeyHashList3)), @@ -2513,7 +2515,7 @@ foldobjects_vs_hashtree_testto() -> {foldheads_allkeys, ?STD_TAG, FoldHeadsFun2, - false, false, false}), + false, false, false, false, false}), KeyHashList4 = HTFolder4(), ?assertMatch(KeyHashList1, lists:usort(KeyHashList4)), diff --git a/src/leveled_runner.erl b/src/leveled_runner.erl index ffaf509..9f4c25e 100644 --- a/src/leveled_runner.erl +++ b/src/leveled_runner.erl @@ -30,7 +30,7 @@ bucketkey_query/6, hashlist_query/3, tictactree/5, - foldheads_allkeys/5, + foldheads_allkeys/7, foldobjects_allkeys/4, foldheads_bybucket/8, foldobjects_bybucket/4, @@ -270,12 +270,14 @@ tictactree(SnapFun, {Tag, Bucket, Query}, JournalCheck, TreeSize, Filter) -> {async, Runner}. -spec foldheads_allkeys(fun(), leveled_codec:tag(), - fun(), boolean(), false|list(integer())) - -> {async, fun()}. + fun(), boolean(), false|list(integer()), + false|leveled_codec:lastmod_range(), + false|pos_integer()) -> {async, fun()}. %% @doc %% Fold over all heads in the store for a given tag - applying the passed %% function to each proxy object -foldheads_allkeys(SnapFun, Tag, FoldFun, JournalCheck, SegmentList) -> +foldheads_allkeys(SnapFun, Tag, FoldFun, JournalCheck, + SegmentList, LastModRange, MaxObjectCount) -> StartKey = leveled_codec:to_ledgerkey(null, null, Tag), EndKey = leveled_codec:to_ledgerkey(null, null, Tag), foldobjects(SnapFun, @@ -283,7 +285,9 @@ foldheads_allkeys(SnapFun, Tag, FoldFun, JournalCheck, SegmentList) -> [{StartKey, EndKey}], FoldFun, {true, JournalCheck}, - SegmentList). + SegmentList, + LastModRange, + MaxObjectCount). -spec foldobjects_allkeys(fun(), leveled_codec:tag(), fun(), key_order|sqn_order) -> {async, fun()}. diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index 0f2f81a..eb748cc 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -227,7 +227,8 @@ aae_missingjournal(_Config) -> {foldheads_allkeys, ?RIAK_TAG, FoldHeadsFun, - true, true, false}), + true, true, false, + false, false}), HeadL2 = length(AllHeadF2()), io:format("Fold head returned ~w objects~n", [HeadL2]), true = HeadL2 < HeadL1, diff --git a/test/end_to_end/riak_SUITE.erl b/test/end_to_end/riak_SUITE.erl index 280b7da..6eb1962 100644 --- a/test/end_to_end/riak_SUITE.erl +++ b/test/end_to_end/riak_SUITE.erl @@ -422,7 +422,7 @@ test_segfilter_query(Bookie, CLs) -> Acc end end, 0}, - false, true, SegL} + false, true, SegL, false, false} end, {async, SL1Folder} = @@ -455,7 +455,7 @@ test_singledelta_stores(BookA, BookB, TreeSize, DeltaKey) -> ?RIAK_TAG, {fun head_tictac_foldfun/4, {0, leveled_tictac:new_tree(test, TreeSize)}}, - false, true, false}, + false, true, false, false, false}, % tictac query by bucket (should be same result as all stores) TicTacByBucketFolder = {foldheads_bybucket, @@ -478,7 +478,7 @@ test_singledelta_stores(BookA, BookB, TreeSize, DeltaKey) -> {foldheads_allkeys, ?RIAK_TAG, {get_segment_folder(DLs, TreeSize), []}, - false, true, false}, + false, true, false, false, false}, SW_SL0 = os:timestamp(), {async, BookASegFolder} = @@ -502,7 +502,7 @@ test_singledelta_stores(BookA, BookB, TreeSize, DeltaKey) -> {foldheads_allkeys, ?RIAK_TAG, {get_segment_folder(DLs, TreeSize), []}, - false, true, SegFilterList}, + false, true, SegFilterList, false, false}, SW_SL1 = os:timestamp(), {async, BookASegFolder1} = @@ -521,7 +521,7 @@ test_singledelta_stores(BookA, BookB, TreeSize, DeltaKey) -> {foldheads_allkeys, ?RIAK_TAG, {get_segment_folder(DLs, TreeSize), []}, - true, true, SegFilterList}, + true, true, SegFilterList, false, false}, SW_SL1CP = os:timestamp(), {async, BookASegFolder1CP} = @@ -545,7 +545,7 @@ test_singledelta_stores(BookA, BookB, TreeSize, DeltaKey) -> {foldheads_allkeys, ?RIAK_TAG, {get_segment_folder(DLs, TreeSize), []}, - false, true, SegFilterListF}, + false, true, SegFilterListF, false, false}, SW_SL1F = os:timestamp(), {async, BookASegFolder1F} = @@ -749,7 +749,7 @@ handoff(_Config) -> ?RIAK_TAG, {fun head_tictac_foldfun/4, {0, leveled_tictac:new_tree(test, TreeSize)}}, - false, true, false}, + false, true, false, false, false}, check_tictacfold(Bookie1, Bookie2, TicTacFolder, none, TreeSize), check_tictacfold(Bookie2, Bookie3, TicTacFolder, none, TreeSize), check_tictacfold(Bookie3, Bookie4, TicTacFolder, none, TreeSize), diff --git a/test/end_to_end/tictac_SUITE.erl b/test/end_to_end/tictac_SUITE.erl index 690160d..20c748c 100644 --- a/test/end_to_end/tictac_SUITE.erl +++ b/test/end_to_end/tictac_SUITE.erl @@ -595,7 +595,8 @@ basic_headonly_test(ObjectCount, RemoveCount, HeadOnly) -> InitAcc = {0, 0}, RunnerDefinition = - {foldheads_allkeys, h, {FoldFun, InitAcc}, false, false, false}, + {foldheads_allkeys, h, {FoldFun, InitAcc}, + false, false, false, false, false}, {async, Runner1} = leveled_bookie:book_returnfolder(Bookie1, RunnerDefinition),