umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Extend API

Browse source 
To support max_keys and the last modified date range.

This applies the last modified date check on all ledger folds.  This is hard to avoid, but ultimately a very low cost.

The limit on the number of heads to fold, is the limit based on passing to the accumulator - not on the limit being added to the accumulator.  So if the FoldFun perfoms a filter (e.g. for the preflist), then those filtered results will still count towards the maximum.

There needs to be someway at the end of signalling from the fold if the outcome was  or was not 'constrained' by max_keys - as the fold cannot simply tel by lenght checking the outcome.

Note this is used rather than length checking the buffer and throwing a 'stop_fold' message when the limit is reached.  The choice is made for simplicity, and ease of testing.  The throw mechanism is necessary if there is a need to stop parallel folds across the the cluster - but in this case the node_worker_pool will be used.
This commit is contained in:
Martin Sumner 2018-10-31 00:09:24 +00:00
parent 1f976948a1
commit 11627bbdd9
4 changed files with 132 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

32
src/leveled_runner.erl
View file

@ -32,7 +32,7 @@
tictactree/5,
foldheads_allkeys/5,
foldobjects_allkeys/4,
foldheads_bybucket/6,
foldheads_bybucket/8,
foldobjects_bybucket/4,
foldobjects_byindex/3
]).
@ -49,6 +49,7 @@
:: {fun(), any()}.
-type term_regex() :: re:mp()|undefined.
%%%============================================================================
%%% External functions
%%%============================================================================
@ -399,7 +400,10 @@ foldobjects_bybucket(SnapFun, Tag, KeyRanges, FoldFun) ->
atom(),
list({any(), any()}),
fun(),
boolean(), false|list(integer()))
boolean(),
false|list(integer()),
false|leveled_codec:lastmod_range(),
false|pos_integer())
-> {async, fun()}.
%% @doc
%% Fold over all object metadata within a given key range in a bucket
@ -407,13 +411,16 @@ foldheads_bybucket(SnapFun,
Tag,
KeyRanges,
FoldFun,
JournalCheck, SegmentList) ->
JournalCheck,
SegmentList, LastModRange, MaxObjectCount) ->
foldobjects(SnapFun,
Tag,
KeyRanges,
FoldFun,
{true, JournalCheck},
SegmentList).
SegmentList,
LastModRange,
MaxObjectCount).
-spec foldobjects_byindex(fun(), tuple(), fun()) -> {async, fun()}.
%% @doc
@ -484,6 +491,16 @@ get_nextbucket(NextBucket, NextKey, Tag, LedgerSnapshot, BKList, {C, L}) ->
-spec foldobjects(fun(), atom(), list(), fun(),
false|{true, boolean()}, false|list(integer())) ->
{async, fun()}.
foldobjects(SnapFun, Tag, KeyRanges, FoldObjFun, DeferredFetch, SegmentList) ->
foldobjects(SnapFun, Tag, KeyRanges,
FoldObjFun, DeferredFetch, SegmentList, false, false).
-spec foldobjects(fun(), atom(), list(), fun(),
false|{true, boolean()},
false|list(integer()),
false|leveled_codec:lastmod_range(),
false|pos_integer()) ->
{async, fun()}.
%% @doc
%% The object folder should be passed DeferredFetch.
%% DeferredFetch can either be false (which will return to the fold function
@ -491,7 +508,8 @@ get_nextbucket(NextBucket, NextKey, Tag, LedgerSnapshot, BKList, {C, L}) ->
%% will be created that if understood by the fold function will allow the fold
%% function to work on the head of the object, and defer fetching the body in
%% case such a fetch is unecessary.
foldobjects(SnapFun, Tag, KeyRanges, FoldObjFun, DeferredFetch, SegmentList) ->
foldobjects(SnapFun, Tag, KeyRanges, FoldObjFun, DeferredFetch,
SegmentList, LastModRange, MaxObjectCount) ->
{FoldFun, InitAcc} =
case is_tuple(FoldObjFun) of
true ->
@ -519,7 +537,9 @@ foldobjects(SnapFun, Tag, KeyRanges, FoldObjFun, DeferredFetch, SegmentList) ->
EndKey,
AccFun,
FoldAcc,
SegmentList)
SegmentList,
LastModRange,
MaxObjectCount)
end,
Acc = lists:foldl(ListFoldFun, InitAcc, KeyRanges),
ok = leveled_penciller:pcl_close(LedgerSnapshot),