umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

foldheads_bybucket adds segment list support

Browse source 
Accelerate queries for foldheads_bybucket as well
This commit is contained in:
Martin Sumner 2017-11-01 22:00:12 +00:00
parent 6beeadc7d8
commit c8ad39b33b
5 changed files with 73 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

34
src/leveled_bookie.erl
View file

@ -679,12 +679,6 @@ get_runner(State, {keylist, Tag, Bucket, FoldAccT}) ->
leveled_runner:bucketkey_query(SnapFun, Tag, Bucket, FoldAccT);
%% Set of runners for object or metadata folds
get_runner(State,
{foldheads_allkeys, Tag, FoldFun, JournalCheck, SnapPreFold}) ->
SnapType = snaptype_by_presence(JournalCheck),
SnapFun = return_snapfun(State, SnapType, no_lookup, true, SnapPreFold),
leveled_runner:foldheads_allkeys(SnapFun, Tag, FoldFun,
JournalCheck, false);
get_runner(State,
{foldheads_allkeys,
Tag, FoldFun,
@ -702,14 +696,14 @@ get_runner(State,
{foldheads_bybucket,
Tag, Bucket, KeyRange,
FoldFun,
JournalCheck, SnapPreFold}) ->
JournalCheck, SnapPreFold, SegmentList}) ->
{StartKey, EndKey, SnapQ} = return_ledger_keyrange(Tag, Bucket, KeyRange),
SnapType = snaptype_by_presence(JournalCheck),
SnapFun = return_snapfun(State, SnapType, SnapQ, true, SnapPreFold),
leveled_runner:foldheads_bybucket(SnapFun,
{Tag, StartKey, EndKey},
FoldFun,
JournalCheck);
JournalCheck, SegmentList);
get_runner(State,
{foldobjects_bybucket,
Tag, Bucket, KeyRange,
@ -1322,8 +1316,7 @@ foldobjects_vs_hashtree_testto() ->
{foldheads_allkeys,
?STD_TAG,
FoldHeadsFun,
true,
true}),
true, true, false}),
KeyHashList3 = HTFolder3(),
?assertMatch(KeyHashList1, lists:usort(KeyHashList3)),
@ -1342,8 +1335,7 @@ foldobjects_vs_hashtree_testto() ->
{foldheads_allkeys,
?STD_TAG,
FoldHeadsFun2,
false,
false}),
false, false, false}),
KeyHashList4 = HTFolder4(),
?assertMatch(KeyHashList1, lists:usort(KeyHashList4)),
@ -1413,8 +1405,7 @@ foldobjects_vs_foldheads_bybucket_testto() ->
"BucketA",
all,
FoldHeadsFun,
true,
true}),
true, true, false}),
KeyHashList2A = HTFolder2A(),
{async, HTFolder2B} =
book_returnfolder(Bookie1,
@ -1423,8 +1414,7 @@ foldobjects_vs_foldheads_bybucket_testto() ->
"BucketB",
all,
FoldHeadsFun,
true,
false}),
true, false, false}),
KeyHashList2B = HTFolder2B(),
?assertMatch(true,
@ -1439,8 +1429,7 @@ foldobjects_vs_foldheads_bybucket_testto() ->
"BucketB",
{"Key", <<"$all">>},
FoldHeadsFun,
true,
false}),
true, false, false}),
KeyHashList2C = HTFolder2C(),
{async, HTFolder2D} =
book_returnfolder(Bookie1,
@ -1449,8 +1438,7 @@ foldobjects_vs_foldheads_bybucket_testto() ->
"BucketB",
{"Key", "Keyzzzzz"},
FoldHeadsFun,
true,
true}),
true, true, false}),
KeyHashList2D = HTFolder2D(),
?assertMatch(true,
lists:usort(KeyHashList2B) == lists:usort(KeyHashList2C)),
@ -1464,8 +1452,7 @@ foldobjects_vs_foldheads_bybucket_testto() ->
"BucketB",
{"Key", "Key4zzzz"},
FoldHeadsFun,
true,
false}),
true, false, false}),
KeyHashList2E = HTFolder2E(),
{async, HTFolder2F} =
book_returnfolder(Bookie1,
@ -1474,8 +1461,7 @@ foldobjects_vs_foldheads_bybucket_testto() ->
"BucketB",
{"Key5", <<"all">>},
FoldHeadsFun,
true,
false}),
true, false, false}),
KeyHashList2F = HTFolder2F(),
?assertMatch(true, length(KeyHashList2E) > 0),

16
src/leveled_runner.erl
View file

@ -30,7 +30,7 @@
tictactree/5,
foldheads_allkeys/5,
foldobjects_allkeys/3,
foldheads_bybucket/4,
foldheads_bybucket/5,
foldobjects_bybucket/3,
foldobjects_byindex/3
]).
@ -247,15 +247,21 @@ foldobjects_bybucket(SnapFun, {Tag, StartKey, EndKey}, FoldFun) ->
FoldFun,
false, false).
-spec foldheads_bybucket(fun(), {atom(), any(), any()}, fun(), boolean()) ->
{async, fun()}.
-spec foldheads_bybucket(fun(),
{atom(), any(), any()},
fun(),
boolean(), false|list(integer()))
-> {async, fun()}.
%% @doc
%% Fold over all object metadata within a given key range in a bucket
foldheads_bybucket(SnapFun, {Tag, StartKey, EndKey}, FoldFun, JournalCheck) ->
foldheads_bybucket(SnapFun,
{Tag, StartKey, EndKey},
FoldFun,
JournalCheck, SegmentList) ->
foldobjects(SnapFun,
Tag, StartKey, EndKey,
FoldFun,
{true, JournalCheck}, false).
{true, JournalCheck}, SegmentList).
-spec foldobjects_byindex(fun(), tuple(), fun()) -> {async, fun()}.
%% @doc

6
test/end_to_end/recovery_SUITE.erl
View file

@ -117,8 +117,7 @@ aae_missingjournal(_Config) ->
{foldheads_allkeys,
?RIAK_TAG,
FoldHeadsFun,
true,
true}),
true, true, false}),
HeadL1 = length(AllHeadF1()),
io:format("Fold head returned ~w objects~n", [HeadL1]),
@ -135,8 +134,7 @@ aae_missingjournal(_Config) ->
{foldheads_allkeys,
?RIAK_TAG,
FoldHeadsFun,
true,
true}),
true, true, false}),
HeadL2 = length(AllHeadF2()),
io:format("Fold head returned ~w objects~n", [HeadL2]),
true = HeadL2 < HeadL1,

76
test/end_to_end/riak_SUITE.erl
View file

@ -78,47 +78,37 @@ crossbucket_aae(_Config) ->
test_singledelta_stores(BookA, BookB, TreeSize, DeltaKey) ->
io:format("Test for single delta with tree size ~w~n", [TreeSize]),
{B1, K1} = DeltaKey,
% Now run a tictac query against both stores to see the extent to which
% state between stores is consistent
HeadTicTacFolder =
TicTacFolder =
{foldheads_allkeys,
?RIAK_TAG,
{fun head_tictac_foldfun/4,
{0, leveled_tictac:new_tree(test, TreeSize)}},
false,
true},
false, true, false},
% tictac query by bucket (should be same result as all stores)
TicTacByBucketFolder =
{foldheads_bybucket,
?RIAK_TAG, <<"Bucket">>,
all,
{fun head_tictac_foldfun/4,
{0, leveled_tictac:new_tree(test, TreeSize)}},
false, false, false},
SW_TT0 = os:timestamp(),
{async, BookATreeFolder} =
leveled_bookie:book_returnfolder(BookA, HeadTicTacFolder),
{async, BookBTreeFolder} =
leveled_bookie:book_returnfolder(BookB, HeadTicTacFolder),
{CountA, BookATree} = BookATreeFolder(),
{CountB, BookBTree} = BookBTreeFolder(),
Time_TT0 = timer:now_diff(os:timestamp(), SW_TT0)/1000,
io:format("Two tree folds took ~w milliseconds ~n", [Time_TT0]),
io:format("Fold over keys revealed counts of ~w and ~w~n",
[CountA, CountB]),
% There should be a single delta between the stores
1 = CountA - CountB,
DLs = leveled_tictac:find_dirtyleaves(BookATree, BookBTree),
io:format("Found dirty leaves with Riak fold_heads of ~w~n",
[length(DLs)]),
true = length(DLs) == 1,
ExpSeg = leveled_tictac:keyto_segment32(<<B1/binary, K1/binary>>),
TreeSeg = leveled_tictac:get_segment(ExpSeg, TreeSize),
[ActualSeg] = DLs,
true = TreeSeg == ActualSeg,
DLs = check_tictacfold(BookA, BookB,
TicTacFolder,
DeltaKey,
TreeSize),
DLs = check_tictacfold(BookA, BookB,
TicTacByBucketFolder,
DeltaKey,
TreeSize),
HeadSegmentFolder =
{foldheads_allkeys,
?RIAK_TAG,
{get_segment_folder(DLs, TreeSize), []},
false, true},
false, true, false},
SW_SL0 = os:timestamp(),
{async, BookASegFolder} =
@ -229,6 +219,34 @@ head_tictac_foldfun(B, K, PO, {Count, TreeAcc}) ->
leveled_tictac:add_kv(TreeAcc, {B, K}, PO, ExtractFun)}.
check_tictacfold(BookA, BookB, HeadTicTacFolder, {B1, K1}, TreeSize) ->
SW_TT0 = os:timestamp(),
{async, BookATreeFolder} =
leveled_bookie:book_returnfolder(BookA, HeadTicTacFolder),
{async, BookBTreeFolder} =
leveled_bookie:book_returnfolder(BookB, HeadTicTacFolder),
{CountA, BookATree} = BookATreeFolder(),
{CountB, BookBTree} = BookBTreeFolder(),
Time_TT0 = timer:now_diff(os:timestamp(), SW_TT0)/1000,
io:format("Two tree folds took ~w milliseconds ~n", [Time_TT0]),
io:format("Fold over keys revealed counts of ~w and ~w~n",
[CountA, CountB]),
% There should be a single delta between the stores
1 = CountA - CountB,
DLs = leveled_tictac:find_dirtyleaves(BookATree, BookBTree),
io:format("Found dirty leaves with Riak fold_heads of ~w~n",
[length(DLs)]),
true = length(DLs) == 1,
ExpSeg = leveled_tictac:keyto_segment32(<<B1/binary, K1/binary>>),
TreeSeg = leveled_tictac:get_segment(ExpSeg, TreeSize),
[ActualSeg] = DLs,
true = TreeSeg == ActualSeg,
DLs.
summary_from_binary(<<131, _Rest/binary>>=ObjBin) ->
{proxy_object, HeadBin, ObjSize, _Fetcher} = binary_to_term(ObjBin),
summary_from_binary(HeadBin, ObjSize);

7
test/end_to_end/tictac_SUITE.erl
View file

@ -150,7 +150,7 @@ many_put_compare(_Config) ->
"Bucket",
all,
{FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)},
false, true},
false, true, false},
{async, TreeAObjFolder0} =
leveled_bookie:book_returnfolder(Bookie2, FoldQ0),
SWB0Obj = os:timestamp(),
@ -165,7 +165,7 @@ many_put_compare(_Config) ->
"Bucket",
all,
{FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)},
true, true},
true, true, false},
{async, TreeAObjFolder1} =
leveled_bookie:book_returnfolder(Bookie2, FoldQ1),
SWB1Obj = os:timestamp(),
@ -193,8 +193,7 @@ many_put_compare(_Config) ->
"Bucket",
all,
{AltFoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)},
false,
true},
false, true, false},
{async, TreeAAltObjFolder0} =
leveled_bookie:book_returnfolder(Bookie2, AltFoldQ0),
SWB2Obj = os:timestamp(),