umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Merge branch 'master' into mas-i164-hotbackup

Browse source
This commit is contained in:
Martin Sumner 2018-09-07 12:27:30 +01:00
commit b99cde9599
9 changed files with 574 additions and 104 deletions
Download patch file Download diff file Expand all files Collapse all files

26
test/end_to_end/basic_SUITE.erl
View file

@ -579,8 +579,9 @@ space_clear_ondelete(_Config) ->
G2),
FoldKeysFun = fun(B, K, Acc) -> [{B, K}|Acc] end,
AllKeyQuery = {keylist, o_rkv, {FoldKeysFun, []}},
{async, F1} = leveled_bookie:book_returnfolder(Book1, AllKeyQuery),
{async, F1} = leveled_bookie:book_keylist(Book1, o_rkv, {FoldKeysFun, []}),
SW1 = os:timestamp(),
KL1 = F1(),
ok = case length(KL1) of
@ -594,19 +595,20 @@ space_clear_ondelete(_Config) ->
{ok, FNsA_J} = file:list_dir(RootPath ++ "/journal/journal_files"),
io:format("FNsA - Bookie created ~w journal files and ~w ledger files~n",
[length(FNsA_J), length(FNsA_L)]),
% Get an iterator to lock the inker during compaction
FoldObjectsFun = fun(B, K, ObjBin, Acc) ->
[{B, K, erlang:phash2(ObjBin)}|Acc] end,
{async, HTreeF1} = leveled_bookie:book_returnfolder(Book1,
{foldobjects_allkeys,
?RIAK_TAG,
FoldObjectsFun,
false}),
{async, HTreeF1} = leveled_bookie:book_objectfold(Book1,
?RIAK_TAG,
{FoldObjectsFun, []},
false),
% This query does not Snap PreFold - and so will not prevent
% pending deletes from prompting actual deletes
{async, KF1} = leveled_bookie:book_returnfolder(Book1, AllKeyQuery),
{async, KF1} = leveled_bookie:book_keylist(Book1, o_rkv, {FoldKeysFun, []}),
% This query does Snap PreFold, and so will prevent deletes from
% the ledger
@ -662,7 +664,7 @@ space_clear_ondelete(_Config) ->
"after deletes~n",
[PointB_Journals, length(FNsB_L)]),
{async, F2} = leveled_bookie:book_returnfolder(Book1, AllKeyQuery),
{async, F2} = leveled_bookie:book_keylist(Book1, o_rkv, {FoldKeysFun, []}),
SW3 = os:timestamp(),
KL2 = F2(),
ok = case length(KL2) of
@ -674,7 +676,7 @@ space_clear_ondelete(_Config) ->
ok = leveled_bookie:book_close(Book1),
{ok, Book2} = leveled_bookie:book_start(StartOpts1),
{async, F3} = leveled_bookie:book_returnfolder(Book2, AllKeyQuery),
{async, F3} = leveled_bookie:book_keylist(Book2, o_rkv, {FoldKeysFun, []}),
SW4 = os:timestamp(),
KL3 = F3(),
ok = case length(KL3) of
@ -842,4 +844,4 @@ many_put_fetch_switchcompression(_Config) ->
testutil:check_forlist(Bookie3, ChkListFixed),
testutil:check_forobject(Bookie3, TestObject),
testutil:check_formissingobject(Bookie3, "Bookie1", "MissingKey0123"),
ok = leveled_bookie:book_destroy(Bookie3).
ok = leveled_bookie:book_destroy(Bookie3).

137
test/end_to_end/iterator_SUITE.erl
View file

@ -10,6 +10,7 @@
small_load_with2i/1,
query_count/1,
multibucket_fold/1,
foldobjects_bybucket_range/1,
rotating_objects/1]).
all() -> [
@ -17,7 +18,8 @@ all() -> [
small_load_with2i,
query_count,
multibucket_fold,
rotating_objects
rotating_objects,
foldobjects_bybucket_range
].
@ -34,14 +36,15 @@ single_object_with2i(_Config) ->
TestSpec = [{add, list_to_binary("integer_int"), 100},
{add, list_to_binary("binary_bin"), <<100:32/integer>>}],
ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
IdxQ1 = {index_query,
"Bucket1",
{fun testutil:foldkeysfun/3, []},
{list_to_binary("binary_bin"),
<<99:32/integer>>, <<101:32/integer>>},
{true, undefined}},
{async, IdxFolder1} = leveled_bookie:book_returnfolder(Bookie1, IdxQ1),
%% @TODO replace all index queries with new Top-Level API if tests
%% pass
{async, IdxFolder1} = leveled_bookie:book_indexfold(Bookie1,
"Bucket1",
{fun testutil:foldkeysfun/3, []},
{list_to_binary("binary_bin"),
<<99:32/integer>>, <<101:32/integer>>},
{true, undefined}),
R1 = IdxFolder1(),
io:format("R1 of ~w~n", [R1]),
true = [{<<100:32/integer>>,"Key1"}] == R1,
@ -127,28 +130,26 @@ small_load_with2i(_Config) ->
%% Get the Buckets Keys and Hashes for the whole bucket
FoldObjectsFun = fun(B, K, V, Acc) -> [{B, K, erlang:phash2(V)}|Acc]
end,
{async, HTreeF1} = leveled_bookie:book_returnfolder(Bookie1,
{foldobjects_allkeys,
?RIAK_TAG,
FoldObjectsFun,
false}),
{async, HTreeF1} = leveled_bookie:book_objectfold(Bookie1,
?RIAK_TAG,
{FoldObjectsFun, []},
false),
KeyHashList1 = HTreeF1(),
{async, HTreeF2} = leveled_bookie:book_returnfolder(Bookie1,
{foldobjects_bybucket,
?RIAK_TAG,
"Bucket",
all,
FoldObjectsFun,
false}),
{async, HTreeF2} = leveled_bookie:book_objectfold(Bookie1,
?RIAK_TAG,
"Bucket",
all,
{FoldObjectsFun, []},
false),
KeyHashList2 = HTreeF2(),
{async, HTreeF3} = leveled_bookie:book_returnfolder(Bookie1,
{foldobjects_byindex,
?RIAK_TAG,
"Bucket",
{"idx1_bin",
"#", "|"},
FoldObjectsFun,
false}),
{async, HTreeF3} = leveled_bookie:book_objectfold(Bookie1,
?RIAK_TAG,
"Bucket",
{"idx1_bin", "#", "|"},
{FoldObjectsFun, []},
false),
KeyHashList3 = HTreeF3(),
true = 9901 == length(KeyHashList1), % also includes the test object
true = 9900 == length(KeyHashList2),
@ -173,14 +174,12 @@ small_load_with2i(_Config) ->
true = Total2 == Total1,
FoldBucketsFun = fun(B, Acc) -> sets:add_element(B, Acc) end,
% this should find Bucket and Bucket1 - as we can now find string-based
% buckets using bucket_list - i.e. it isn't just binary buckets now
BucketListQuery = {bucket_list,
?RIAK_TAG,
{FoldBucketsFun, sets:new()}},
{async, BL} = leveled_bookie:book_returnfolder(Bookie2, BucketListQuery),
{async, BL} = leveled_bookie:book_bucketlist(Bookie2, ?RIAK_TAG, {FoldBucketsFun, sets:new()}, all),
true = sets:size(BL()) == 2,
ok = leveled_bookie:book_close(Bookie2),
testutil:reset_filestructure().
@ -511,16 +510,20 @@ multibucket_fold(_Config) ->
IndexGen,
<<"Bucket4">>),
testutil:riakload(Bookie1, ObjL4),
Q1 = {foldheads_bybucket,
?RIAK_TAG,
[<<"Bucket1">>, <<"Bucket4">>], bucket_list,
fun(B, K, _PO, Acc) ->
FF = fun(B, K, _PO, Acc) ->
[{B, K}|Acc]
end,
false,
true,
false},
{async, R1} = leveled_bookie:book_returnfolder(Bookie1, Q1),
end,
FoldAccT = {FF, []},
{async, R1} = leveled_bookie:book_headfold(Bookie1,
?RIAK_TAG,
{bucket_list, [<<"Bucket1">>, <<"Bucket4">>]},
FoldAccT,
false,
true,
false),
O1 = length(R1()),
io:format("Result R1 of length ~w~n", [O1]),
@ -544,10 +547,6 @@ multibucket_fold(_Config) ->
ok = leveled_bookie:book_close(Bookie1),
testutil:reset_filestructure().
rotating_objects(_Config) ->
RootPath = testutil:reset_filestructure(),
ok = testutil:rotating_object_check(RootPath, "Bucket1", 10),
@ -558,8 +557,50 @@ rotating_objects(_Config) ->
ok = testutil:rotating_object_check(RootPath, "Bucket6", 9600),
testutil:reset_filestructure().
foldobjects_bybucket_range(_Config) ->
RootPath = testutil:reset_filestructure(),
{ok, Bookie1} = leveled_bookie:book_start(RootPath,
2000,
50000000,
testutil:sync_strategy()),
ObjectGen = testutil:get_compressiblevalue_andinteger(),
IndexGen = fun() -> [] end,
ObjL1 = testutil:generate_objects(1300,
{fixed_binary, 1},
[],
ObjectGen,
IndexGen,
<<"Bucket1">>),
testutil:riakload(Bookie1, ObjL1),
FoldKeysFun = fun(_B, K,_V, Acc) ->
[ K |Acc]
end,
StartKey = testutil:fixed_bin_key(123),
EndKey = testutil:fixed_bin_key(779),
{async, Folder} = leveled_bookie:book_objectfold(Bookie1,
?RIAK_TAG,
<<"Bucket1">>,
{StartKey, EndKey}, {FoldKeysFun, []},
true
),
ResLen = length(Folder()),
io:format("Length of Result of folder ~w~n", [ResLen]),
true = 657 == ResLen,
{async, AllFolder} = leveled_bookie:book_objectfold(Bookie1,
?RIAK_TAG,
<<"Bucket1">>,
all,
{FoldKeysFun, []},
true
),
AllResLen = length(AllFolder()),
io:format("Length of Result of all keys folder ~w~n", [AllResLen]),
true = 1300 == AllResLen,
ok = leveled_bookie:book_close(Bookie1),
testutil:reset_filestructure().

11
test/end_to_end/recovery_SUITE.erl
View file

@ -174,11 +174,12 @@ aae_missingjournal(_Config) ->
fun(B, K, _V, Acc) -> [{B, K}|Acc] end,
{async, AllHeadF1} =
leveled_bookie:book_returnfolder(Bookie1,
{foldheads_allkeys,
?RIAK_TAG,
FoldHeadsFun,
true, true, false}),
leveled_bookie:book_headfold(Bookie1,
?RIAK_TAG,
{FoldHeadsFun, []},
true,
true,
false),
HeadL1 = length(AllHeadF1()),
io:format("Fold head returned ~w objects~n", [HeadL1]),

49
test/end_to_end/riak_SUITE.erl
View file

@ -425,26 +425,12 @@ handoff(_Config) ->
end,
% Handoff the data from the first store to the other three stores
HandoffFolder2 =
{foldobjects_allkeys,
?RIAK_TAG,
{FoldObjectsFun(Bookie2), ok},
false,
key_order},
HandoffFolder3 =
{foldobjects_allkeys,
?RIAK_TAG,
{FoldObjectsFun(Bookie3), ok},
true,
sqn_order},
HandoffFolder4 =
{foldobjects_allkeys,
?RIAK_TAG,
{FoldObjectsFun(Bookie4), ok},
true,
sqn_order},
{async, Handoff2} =
leveled_bookie:book_returnfolder(Bookie1, HandoffFolder2),
leveled_bookie:book_objectfold(Bookie1,
?RIAK_TAG,
{FoldObjectsFun(Bookie2), ok},
false,
key_order),
SW2 = os:timestamp(),
ok = Handoff2(),
Time_HO2 = timer:now_diff(os:timestamp(), SW2)/1000,
@ -452,14 +438,23 @@ handoff(_Config) ->
[Time_HO2]),
SW3 = os:timestamp(),
{async, Handoff3} =
leveled_bookie:book_returnfolder(Bookie1, HandoffFolder3),
leveled_bookie:book_objectfold(Bookie1,
?RIAK_TAG,
{FoldObjectsFun(Bookie3), ok},
true,
sqn_order),
ok = Handoff3(),
Time_HO3 = timer:now_diff(os:timestamp(), SW3)/1000,
io:format("Handoff to Book3 in sqn_order took ~w milliseconds ~n",
[Time_HO3]),
SW4 = os:timestamp(),
{async, Handoff4} =
leveled_bookie:book_returnfolder(Bookie1, HandoffFolder4),
leveled_bookie:book_objectfold(Bookie1,
?RIAK_TAG,
{FoldObjectsFun(Bookie4), ok},
true,
sqn_order),
ok = Handoff4(),
Time_HO4 = timer:now_diff(os:timestamp(), SW4)/1000,
io:format("Handoff to Book4 in sqn_order took ~w milliseconds ~n",
@ -529,9 +524,12 @@ dollar_key_index(_Config) ->
StartKey = testutil:fixed_bin_key(123),
EndKey = testutil:fixed_bin_key(779),
Query = {keylist, ?RIAK_TAG, <<"Bucket1">>, {StartKey, EndKey}, {FoldKeysFun, []}},
{async, Folder} = leveled_bookie:book_returnfolder(Bookie1, Query),
{async, Folder} = leveled_bookie:book_keylist(Bookie1,
?RIAK_TAG,
<<"Bucket1">>,
{StartKey, EndKey}, {FoldKeysFun, []}
),
ResLen = length(Folder()),
io:format("Length of Result of folder ~w~n", [ResLen]),
true = 657 == ResLen,
@ -575,10 +573,9 @@ dollar_bucket_index(_Config) ->
FoldKeysFun = fun(B, K, Acc) ->
[{B, K}|Acc]
end,
FoldAccT = {FoldKeysFun, []},
Query = {keylist, ?RIAK_TAG, <<"Bucket2">>, {FoldKeysFun, []}},
{async, Folder} = leveled_bookie:book_returnfolder(Bookie1, Query),
{async, Folder} = leveled_bookie:book_keylist(Bookie1, ?RIAK_TAG, <<"Bucket2">>, FoldAccT),
ResLen = length(Folder()),
io:format("Length of Result of folder ~w~n", [ResLen]),

18
test/end_to_end/tictac_SUITE.erl
View file

@ -147,14 +147,16 @@ many_put_compare(_Config) ->
leveled_tictac:add_kv(Acc, Key, Value, ExtractClockFun)
end,
FoldQ0 = {foldheads_bybucket,
o_rkv,
"Bucket",
all,
{FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)},
false, true, false},
FoldAccT = {FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)},
{async, TreeAObjFolder0} =
leveled_bookie:book_returnfolder(Bookie2, FoldQ0),
leveled_bookie:book_headfold(Bookie2,
o_rkv,
{range, "Bucket", all},
FoldAccT,
false,
true,
false),
SWB0Obj = os:timestamp(),
TreeAObj0 = TreeAObjFolder0(),
io:format("Build tictac tree via object fold with no "++
@ -1338,4 +1340,4 @@ get_segment(K, SegmentCount) ->
end,
{SegmentID, ExtraHash} = leveled_codec:segment_hash(BinKey),
SegHash = (ExtraHash band 65535) bsl 16 + SegmentID,
leveled_tictac:get_segment(SegHash, SegmentCount).
leveled_tictac:get_segment(SegHash, SegmentCount).