umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Provide a top level API for folds

Browse source 
As the fold functions have been added to get_runner in an ad hoc way,
naturally, given the ongoing development of levelEd to support Riak,
it was difficult for a new user (in this case Quviq) to see what folds
are supported, and with what arguments, and expectations.

This PR is for discussion. It is one of many ways to group, spec, and
document the fold functions.

A test is also added for coverage of range queries.
This commit is contained in:
Russell Brown 2018-08-23 10:27:43 +01:00
parent a1269e5274
commit b7bd65d11f
6 changed files with 448 additions and 101 deletions
Download patch file Download diff file Expand all files Collapse all files

26
test/end_to_end/basic_SUITE.erl
View file

@ -579,8 +579,9 @@ space_clear_ondelete(_Config) ->
G2),
FoldKeysFun = fun(B, K, Acc) -> [{B, K}|Acc] end,
AllKeyQuery = {keylist, o_rkv, {FoldKeysFun, []}},
{async, F1} = leveled_bookie:book_returnfolder(Book1, AllKeyQuery),
{async, F1} = leveled_bookie:book_keylist(Book1, o_rkv, {FoldKeysFun, []}),
SW1 = os:timestamp(),
KL1 = F1(),
ok = case length(KL1) of
@ -594,19 +595,20 @@ space_clear_ondelete(_Config) ->
{ok, FNsA_J} = file:list_dir(RootPath ++ "/journal/journal_files"),
io:format("FNsA - Bookie created ~w journal files and ~w ledger files~n",
[length(FNsA_J), length(FNsA_L)]),
% Get an iterator to lock the inker during compaction
FoldObjectsFun = fun(B, K, ObjBin, Acc) ->
[{B, K, erlang:phash2(ObjBin)}|Acc] end,
{async, HTreeF1} = leveled_bookie:book_returnfolder(Book1,
{foldobjects_allkeys,
?RIAK_TAG,
FoldObjectsFun,
false}),
{async, HTreeF1} = leveled_bookie:book_objectfold(Book1,
?RIAK_TAG,
{FoldObjectsFun, []},
false),
% This query does not Snap PreFold - and so will not prevent
% pending deletes from prompting actual deletes
{async, KF1} = leveled_bookie:book_returnfolder(Book1, AllKeyQuery),
{async, KF1} = leveled_bookie:book_keylist(Book1, o_rkv, {FoldKeysFun, []}),
% This query does Snap PreFold, and so will prevent deletes from
% the ledger
@ -662,7 +664,7 @@ space_clear_ondelete(_Config) ->
"after deletes~n",
[PointB_Journals, length(FNsB_L)]),
{async, F2} = leveled_bookie:book_returnfolder(Book1, AllKeyQuery),
{async, F2} = leveled_bookie:book_keylist(Book1, o_rkv, {FoldKeysFun, []}),
SW3 = os:timestamp(),
KL2 = F2(),
ok = case length(KL2) of
@ -674,7 +676,7 @@ space_clear_ondelete(_Config) ->
ok = leveled_bookie:book_close(Book1),
{ok, Book2} = leveled_bookie:book_start(StartOpts1),
{async, F3} = leveled_bookie:book_returnfolder(Book2, AllKeyQuery),
{async, F3} = leveled_bookie:book_keylist(Book2, o_rkv, {FoldKeysFun, []}),
SW4 = os:timestamp(),
KL3 = F3(),
ok = case length(KL3) of
@ -842,4 +844,4 @@ many_put_fetch_switchcompression(_Config) ->
testutil:check_forlist(Bookie3, ChkListFixed),
testutil:check_forobject(Bookie3, TestObject),
testutil:check_formissingobject(Bookie3, "Bookie1", "MissingKey0123"),
ok = leveled_bookie:book_destroy(Bookie3).
ok = leveled_bookie:book_destroy(Bookie3).

134
test/end_to_end/iterator_SUITE.erl
View file

@ -10,6 +10,7 @@
small_load_with2i/1,
query_count/1,
multibucket_fold/1,
foldobjects_bybucket_range/1,
rotating_objects/1]).
all() -> [
@ -34,14 +35,15 @@ single_object_with2i(_Config) ->
TestSpec = [{add, list_to_binary("integer_int"), 100},
{add, list_to_binary("binary_bin"), <<100:32/integer>>}],
ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
IdxQ1 = {index_query,
"Bucket1",
{fun testutil:foldkeysfun/3, []},
{list_to_binary("binary_bin"),
<<99:32/integer>>, <<101:32/integer>>},
{true, undefined}},
{async, IdxFolder1} = leveled_bookie:book_returnfolder(Bookie1, IdxQ1),
%% @TODO replace all index queries with new Top-Level API if tests
%% pass
{async, IdxFolder1} = leveled_bookie:book_indexfold(Bookie1,
"Bucket1",
{fun testutil:foldkeysfun/3, []},
{list_to_binary("binary_bin"),
<<99:32/integer>>, <<101:32/integer>>},
{true, undefined}),
R1 = IdxFolder1(),
io:format("R1 of ~w~n", [R1]),
true = [{<<100:32/integer>>,"Key1"}] == R1,
@ -127,28 +129,26 @@ small_load_with2i(_Config) ->
%% Get the Buckets Keys and Hashes for the whole bucket
FoldObjectsFun = fun(B, K, V, Acc) -> [{B, K, erlang:phash2(V)}|Acc]
end,
{async, HTreeF1} = leveled_bookie:book_returnfolder(Bookie1,
{foldobjects_allkeys,
?RIAK_TAG,
FoldObjectsFun,
false}),
{async, HTreeF1} = leveled_bookie:book_objectfold(Bookie1,
?RIAK_TAG,
{FoldObjectsFun, []},
false),
KeyHashList1 = HTreeF1(),
{async, HTreeF2} = leveled_bookie:book_returnfolder(Bookie1,
{foldobjects_bybucket,
?RIAK_TAG,
"Bucket",
all,
FoldObjectsFun,
false}),
{async, HTreeF2} = leveled_bookie:book_objectfold(Bookie1,
?RIAK_TAG,
"Bucket",
all,
{FoldObjectsFun, []},
false),
KeyHashList2 = HTreeF2(),
{async, HTreeF3} = leveled_bookie:book_returnfolder(Bookie1,
{foldobjects_byindex,
?RIAK_TAG,
"Bucket",
{"idx1_bin",
"#", "|"},
FoldObjectsFun,
false}),
{async, HTreeF3} = leveled_bookie:book_objectfold(Bookie1,
?RIAK_TAG,
"Bucket",
{"idx1_bin", "#", "|"},
{FoldObjectsFun, []},
false),
KeyHashList3 = HTreeF3(),
true = 9901 == length(KeyHashList1), % also includes the test object
true = 9900 == length(KeyHashList2),
@ -173,14 +173,12 @@ small_load_with2i(_Config) ->
true = Total2 == Total1,
FoldBucketsFun = fun(B, Acc) -> sets:add_element(B, Acc) end,
% this should find Bucket and Bucket1 - as we can now find string-based
% buckets using bucket_list - i.e. it isn't just binary buckets now
BucketListQuery = {bucket_list,
?RIAK_TAG,
{FoldBucketsFun, sets:new()}},
{async, BL} = leveled_bookie:book_returnfolder(Bookie2, BucketListQuery),
{async, BL} = leveled_bookie:book_bucketlist(Bookie2, ?RIAK_TAG, {FoldBucketsFun, sets:new()}, all),
true = sets:size(BL()) == 2,
ok = leveled_bookie:book_close(Bookie2),
testutil:reset_filestructure().
@ -511,16 +509,20 @@ multibucket_fold(_Config) ->
IndexGen,
<<"Bucket4">>),
testutil:riakload(Bookie1, ObjL4),
Q1 = {foldheads_bybucket,
?RIAK_TAG,
[<<"Bucket1">>, <<"Bucket4">>], bucket_list,
fun(B, K, _PO, Acc) ->
FF = fun(B, K, _PO, Acc) ->
[{B, K}|Acc]
end,
false,
true,
false},
{async, R1} = leveled_bookie:book_returnfolder(Bookie1, Q1),
end,
FoldAccT = {FF, []},
{async, R1} = leveled_bookie:book_headfold(Bookie1,
?RIAK_TAG,
{bucket_list, [<<"Bucket1">>, <<"Bucket4">>]},
FoldAccT,
false,
true,
false),
O1 = length(R1()),
io:format("Result R1 of length ~w~n", [O1]),
@ -544,10 +546,6 @@ multibucket_fold(_Config) ->
ok = leveled_bookie:book_close(Bookie1),
testutil:reset_filestructure().
rotating_objects(_Config) ->
RootPath = testutil:reset_filestructure(),
ok = testutil:rotating_object_check(RootPath, "Bucket1", 10),
@ -558,8 +556,50 @@ rotating_objects(_Config) ->
ok = testutil:rotating_object_check(RootPath, "Bucket6", 9600),
testutil:reset_filestructure().
foldobjects_bybucket_range(_Config) ->
RootPath = testutil:reset_filestructure(),
{ok, Bookie1} = leveled_bookie:book_start(RootPath,
2000,
50000000,
testutil:sync_strategy()),
ObjectGen = testutil:get_compressiblevalue_andinteger(),
IndexGen = fun() -> [] end,
ObjL1 = testutil:generate_objects(1300,
{fixed_binary, 1},
[],
ObjectGen,
IndexGen,
<<"Bucket1">>),
testutil:riakload(Bookie1, ObjL1),
FoldKeysFun = fun(_B, K, Acc) ->
[ K |Acc]
end,
StartKey = testutil:fixed_bin_key(123),
EndKey = testutil:fixed_bin_key(779),
{async, Folder} = leveled_bookie:book_objectfold(Bookie1,
?RIAK_TAG,
<<"Bucket1">>,
{StartKey, EndKey}, {FoldKeysFun, []},
true
),
ResLen = length(Folder()),
io:format("Length of Result of folder ~w~n", [ResLen]),
true = 657 == ResLen,
{async, AllFolder} = leveled_bookie:book_objectfold(Bookie1,
?RIAK_TAG,
<<"Bucket1">>,
all,
{FoldKeysFun, []},
true
),
AllResLen = length(AllFolder()),
io:format("Length of Result of all keys folder ~w~n", [AllResLen]),
true = 1300 == AllResLen,
ok = leveled_bookie:book_close(Bookie1),
testutil:reset_filestructure().

11
test/end_to_end/recovery_SUITE.erl
View file

@ -113,11 +113,12 @@ aae_missingjournal(_Config) ->
fun(B, K, _V, Acc) -> [{B, K}|Acc] end,
{async, AllHeadF1} =
leveled_bookie:book_returnfolder(Bookie1,
{foldheads_allkeys,
?RIAK_TAG,
FoldHeadsFun,
true, true, false}),
leveled_bookie:book_headfold(Bookie1,
?RIAK_TAG,
{FoldHeadsFun, []},
true,
true,
false),
HeadL1 = length(AllHeadF1()),
io:format("Fold head returned ~w objects~n", [HeadL1]),

49
test/end_to_end/riak_SUITE.erl
View file

@ -425,26 +425,12 @@ handoff(_Config) ->
end,
% Handoff the data from the first store to the other three stores
HandoffFolder2 =
{foldobjects_allkeys,
?RIAK_TAG,
{FoldObjectsFun(Bookie2), ok},
false,
key_order},
HandoffFolder3 =
{foldobjects_allkeys,
?RIAK_TAG,
{FoldObjectsFun(Bookie3), ok},
true,
sqn_order},
HandoffFolder4 =
{foldobjects_allkeys,
?RIAK_TAG,
{FoldObjectsFun(Bookie4), ok},
true,
sqn_order},
{async, Handoff2} =
leveled_bookie:book_returnfolder(Bookie1, HandoffFolder2),
leveled_bookie:book_objectfold(Bookie1,
?RIAK_TAG,
{FoldObjectsFun(Bookie2), ok},
false,
key_order),
SW2 = os:timestamp(),
ok = Handoff2(),
Time_HO2 = timer:now_diff(os:timestamp(), SW2)/1000,
@ -452,14 +438,23 @@ handoff(_Config) ->
[Time_HO2]),
SW3 = os:timestamp(),
{async, Handoff3} =
leveled_bookie:book_returnfolder(Bookie1, HandoffFolder3),
leveled_bookie:book_objectfold(Bookie1,
?RIAK_TAG,
{FoldObjectsFun(Bookie3), ok},
true,
sqn_order),
ok = Handoff3(),
Time_HO3 = timer:now_diff(os:timestamp(), SW3)/1000,
io:format("Handoff to Book3 in sqn_order took ~w milliseconds ~n",
[Time_HO3]),
SW4 = os:timestamp(),
{async, Handoff4} =
leveled_bookie:book_returnfolder(Bookie1, HandoffFolder4),
leveled_bookie:book_objectfold(Bookie1,
?RIAK_TAG,
{FoldObjectsFun(Bookie4), ok},
true,
sqn_order),
ok = Handoff4(),
Time_HO4 = timer:now_diff(os:timestamp(), SW4)/1000,
io:format("Handoff to Book4 in sqn_order took ~w milliseconds ~n",
@ -529,9 +524,12 @@ dollar_key_index(_Config) ->
StartKey = testutil:fixed_bin_key(123),
EndKey = testutil:fixed_bin_key(779),
Query = {keylist, ?RIAK_TAG, <<"Bucket1">>, {StartKey, EndKey}, {FoldKeysFun, []}},
{async, Folder} = leveled_bookie:book_returnfolder(Bookie1, Query),
{async, Folder} = leveled_bookie:book_keylist(Bookie1,
?RIAK_TAG,
<<"Bucket1">>,
{StartKey, EndKey}, {FoldKeysFun, []}
),
ResLen = length(Folder()),
io:format("Length of Result of folder ~w~n", [ResLen]),
true = 657 == ResLen,
@ -575,10 +573,9 @@ dollar_bucket_index(_Config) ->
FoldKeysFun = fun(B, K, Acc) ->
[{B, K}|Acc]
end,
FoldAccT = {FoldKeysFun, []},
Query = {keylist, ?RIAK_TAG, <<"Bucket2">>, {FoldKeysFun, []}},
{async, Folder} = leveled_bookie:book_returnfolder(Bookie1, Query),
{async, Folder} = leveled_bookie:book_keylist(Bookie1, ?RIAK_TAG, <<"Bucket2">>, FoldAccT),
ResLen = length(Folder()),
io:format("Length of Result of folder ~w~n", [ResLen]),

18
test/end_to_end/tictac_SUITE.erl
View file

@ -147,14 +147,16 @@ many_put_compare(_Config) ->
leveled_tictac:add_kv(Acc, Key, Value, ExtractClockFun)
end,
FoldQ0 = {foldheads_bybucket,
o_rkv,
"Bucket",
all,
{FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)},
false, true, false},
FoldAccT = {FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)},
{async, TreeAObjFolder0} =
leveled_bookie:book_returnfolder(Bookie2, FoldQ0),
leveled_bookie:book_headfold(Bookie2,
o_rkv,
{range, "Bucket", all},
FoldAccT,
false,
true,
false),
SWB0Obj = os:timestamp(),
TreeAObj0 = TreeAObjFolder0(),
io:format("Build tictac tree via object fold with no "++
@ -1338,4 +1340,4 @@ get_segment(K, SegmentCount) ->
end,
{SegmentID, ExtraHash} = leveled_codec:segment_hash(BinKey),
SegHash = (ExtraHash band 65535) bsl 16 + SegmentID,
leveled_tictac:get_segment(SegHash, SegmentCount).
leveled_tictac:get_segment(SegHash, SegmentCount).