umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Merge branch 'master' into mas-riakaae-impl-2

Browse source
This commit is contained in:
Martin Sumner 2017-09-28 13:23:29 +01:00
commit 61724cfedb
8 changed files with 290 additions and 134 deletions
Download patch file Download diff file Expand all files Collapse all files

49
test/end_to_end/basic_SUITE.erl
View file

@ -580,22 +580,59 @@ space_clear_ondelete(_Config) ->
io:format("This should cause a final ledger merge event~n"),
io:format("Will require the penciller to resolve the issue of creating" ++
" an empty file as all keys compact on merge~n"),
timer:sleep(12000),
CheckFun =
fun(X, FileCount) ->
case FileCount of
0 ->
0;
_ ->
timer:sleep(X),
{ok, NewFC} =
file:list_dir(RootPath ++ "/ledger/ledger_files"),
io:format("Looping with ledger file count ~w~n",
[length(NewFC)]),
length(strip_nonsst(NewFC))
end
end,
FC = lists:foldl(CheckFun, infinity, [2000, 3000, 5000, 8000]),
ok = leveled_bookie:book_close(Book3),
case FC of
0 ->
ok;
_ ->
{ok, Book4} = leveled_bookie:book_start(StartOpts1),
lists:foldl(CheckFun, infinity, [2000, 3000, 5000, 8000]),
leveled_bookie:book_close(Book4)
end,
{ok, FNsD_L} = file:list_dir(RootPath ++ "/ledger/ledger_files"),
io:format("FNsD - Bookie has ~w ledger files " ++
"after second close~n", [length(FNsD_L)]),
"after second close~n", [length(strip_nonsst(FNsD_L))]),
lists:foreach(fun(FN) ->
io:format("FNsD - Ledger file is ~s~n", [FN])
end,
FNsD_L),
true = PointB_Journals < length(FNsA_J),
true = length(FNsD_L) < length(FNsA_L),
true = length(FNsD_L) < length(FNsB_L),
true = length(FNsD_L) < length(FNsC_L),
true = length(FNsD_L) == 0.
true = length(strip_nonsst(FNsD_L)) < length(strip_nonsst(FNsA_L)),
true = length(strip_nonsst(FNsD_L)) < length(strip_nonsst(FNsB_L)),
true = length(strip_nonsst(FNsD_L)) < length(strip_nonsst(FNsC_L)),
true = length(strip_nonsst(FNsD_L)) == 0.
strip_nonsst(FileList) ->
SSTOnlyFun =
fun(FN, Acc) ->
case filename:extension(FN) of
".sst" ->
[FN|Acc];
_ ->
Acc
end
end,
lists:foldl(SSTOnlyFun, [], FileList).
is_empty_test(_Config) ->
RootPath = testutil:reset_filestructure(),

57
test/end_to_end/tictac_SUITE.erl
View file

@ -114,19 +114,16 @@ many_put_compare(_Config) ->
% Now run the same query by putting the tree-building responsibility onto
% the fold_objects_fun
ApplyHash =
fun(HashFun) ->
fun(_Key, Value) ->
{proxy_object, HeadBin, _Size, _FetchFun} = binary_to_term(Value),
<<?MAGIC:8/integer, ?V1_VERS:8/integer, VclockLen:32/integer,
Rest/binary>> = HeadBin,
<<VclockBin:VclockLen/binary, _NotNeeded/binary>> = Rest,
HashFun(lists:sort(binary_to_term(VclockBin)))
end
end,
ExtractClockFun =
fun(Key, Value) ->
{proxy_object, HeadBin, _Size, _FetchFun} = binary_to_term(Value),
<<?MAGIC:8/integer, ?V1_VERS:8/integer, VclockLen:32/integer,
VclockBin:VclockLen/binary, _Rest/binary>> = HeadBin,
{Key, lists:sort(binary_to_term(VclockBin))}
end,
FoldObjectsFun =
fun(_Bucket, Key, Value, Acc) ->
leveled_tictac:add_kv(Acc, Key, Value, ApplyHash(fun erlang:phash2/1))
leveled_tictac:add_kv(Acc, Key, Value, ExtractClockFun, false)
end,
FoldQ0 = {foldheads_bybucket,
@ -157,22 +154,25 @@ many_put_compare(_Config) ->
[timer:now_diff(os:timestamp(), SWB1Obj)]),
true = length(leveled_tictac:find_dirtyleaves(TreeA, TreeAObj1)) == 0,
% AAE trees within riak are based on a sha of the vector clock. So to
% compare with an AAE tree we need to compare outputs when we're hashing
% a hash
AltHashFun =
fun(Term) ->
erlang:phash2(crypto:hash(sha, term_to_binary(Term)))
% For an exportable comparison, want hash to be based on something not
% coupled to erlang language - so use exportable query
AltExtractFun =
fun(K, V) ->
{proxy_object, HeadBin, _Size, _FetchFun} = binary_to_term(V),
<<?MAGIC:8/integer, ?V1_VERS:8/integer, VclockLen:32/integer,
VclockBin:VclockLen/binary, _Rest/binary>> = HeadBin,
{term_to_binary(K), VclockBin}
end,
AltFoldObjectsFun =
fun(_Bucket, Key, Value, Acc) ->
leveled_tictac:add_kv(Acc, Key, Value, ApplyHash(AltHashFun))
leveled_tictac:add_kv(Acc, Key, Value, AltExtractFun, true)
end,
AltFoldQ0 = {foldheads_bybucket,
o_rkv,
"Bucket",
{AltFoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)},
false, true},
o_rkv,
"Bucket",
{AltFoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)},
false,
true},
{async, TreeAAltObjFolder0} =
leveled_bookie:book_returnfolder(Bookie2, AltFoldQ0),
SWB2Obj = os:timestamp(),
@ -187,15 +187,19 @@ many_put_compare(_Config) ->
io:format("Build tictac tree via object fold with no "++
"presence check and 200K objects and alt hash in ~w~n",
[timer:now_diff(os:timestamp(), SWB3Obj)]),
true =
length(leveled_tictac:find_dirtyleaves(TreeBAltObj, TreeAAltObj)) == 1,
DL_ExportFold =
length(leveled_tictac:find_dirtyleaves(TreeBAltObj, TreeAAltObj)),
io:format("Found dirty leaves with exportable comparison of ~w~n",
[DL_ExportFold]),
true = DL_ExportFold == 1,
%% Finding differing keys
FoldKeysFun =
fun(SegListToFind) ->
fun(_B, K, Acc) ->
Seg = leveled_tictac:get_segment(K, SegmentCount),
Seg =
leveled_tictac:get_segment(erlang:phash2(K), SegmentCount),
case lists:member(Seg, SegListToFind) of
true ->
[K|Acc];
@ -469,7 +473,8 @@ index_compare(_Config) ->
FoldKeysIndexQFun =
fun(_Bucket, {Term, Key}, Acc) ->
Seg = leveled_tictac:get_segment(Key, SegmentCount),
Seg =
leveled_tictac:get_segment(erlang:phash2(Key), SegmentCount),
case lists:member(Seg, DL3_0) of
true ->
[{Term, Key}|Acc];