umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Add unit test to prove binary/copy issue

Browse source 
Need to understand why the binary:copy is necessary - unit test now shows this.
This commit is contained in:
Martin Sumner 2019-07-22 10:35:55 +01:00
parent c9c577259e
commit 5bef21d971
1 changed files with 49 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

54
src/leveled_sst.erl
View file

@ -2128,7 +2128,7 @@ crc_check_slot(FullBin) ->
CRC32H:32/integer,
Rest/binary>> = FullBin,
PosBL0 = min(PosBL, byte_size(FullBin) - 12),
% If the position has been bit-flipped to beyond the maximum paossible
% If the position has been bit-flipped to beyond the maximum possible
% length, use the maximum possible length
<<Header:PosBL0/binary, Blocks/binary>> = Rest,
case {hmac(Header), hmac(PosBL0)} of
@ -2630,16 +2630,20 @@ generate_randomkeys(_Seqn, 0, Acc, _BucketLow, _BucketHigh) ->
generate_randomkeys(Seqn, Count, Acc, BucketLow, BRange) ->
BRand = leveled_rand:uniform(BRange),
BNumber =
lists:flatten(io_lib:format("K~4..0B", [BucketLow + BRand])),
lists:flatten(io_lib:format("B~6..0B", [BucketLow + BRand])),
KNumber =
lists:flatten(io_lib:format("K~6..0B", [leveled_rand:uniform(1000)])),
lists:flatten(io_lib:format("K~8..0B", [leveled_rand:uniform(1000000)])),
LK = leveled_codec:to_ledgerkey("Bucket" ++ BNumber, "Key" ++ KNumber, o),
Chunk = leveled_rand:rand_bytes(64),
{_B, _K, MV, _H, _LMs} =
leveled_codec:generate_ledgerkv(LK, Seqn, Chunk, 64, infinity),
MD = element(4, MV),
?assertMatch(undefined, element(3, MD)),
MD0 = [{magic_md, [<<0:32/integer>>, base64:encode(Chunk)]}],
MV0 = setelement(4, MV, setelement(3, MD, MD0)),
generate_randomkeys(Seqn + 1,
Count - 1,
[{LK, MV}|Acc],
[{LK, MV0}|Acc],
BucketLow,
BRange).
@ -2661,6 +2665,7 @@ generate_indexkey(Term, Count) ->
Count,
infinity).
form_slot_test() ->
% If a skip key happens, mustn't switch to loookup by accident as could be
% over the expected size
@ -3243,13 +3248,17 @@ simple_persisted_test_bothformats() ->
simple_persisted_tester(fun testsst_new/6).
simple_persisted_tester(SSTNewFun) ->
Level = 3,
{RP, Filename} = {?TEST_AREA, "simple_test"},
KVList0 = generate_randomkeys(1, ?LOOK_SLOTSIZE * 32, 1, 20),
KVList1 = lists:ukeysort(1, KVList0),
[{FirstKey, _FV}|_Rest] = KVList1,
{LastKey, _LV} = lists:last(KVList1),
{ok, Pid, {FirstKey, LastKey}, _Bloom} =
SSTNewFun(RP, Filename, 1, KVList1, length(KVList1), native),
SSTNewFun(RP, Filename, Level, KVList1, length(KVList1), native),
B0 = check_binary_references(Pid),
SW0 = os:timestamp(),
lists:foreach(fun({K, V}) ->
?assertMatch({K, V}, sst_get(Pid, K))
@ -3346,9 +3355,44 @@ simple_persisted_tester(SSTNewFun) ->
FetchedListB4 = lists:foldl(FoldFun, [], FetchListB4),
?assertMatch([{Eight000Key, _v800}], FetchedListB4),
B1 = check_binary_references(Pid),
ok = sst_close(Pid),
io:format(user, "Reopen SST file~n", []),
OptsSST = #sst_options{press_method=native,
log_options=leveled_log:get_opts()},
{ok, OpenP, {FirstKey, LastKey}, _Bloom} =
sst_open(RP, Filename ++ ".sst", OptsSST, Level),
B2 = check_binary_references(OpenP),
lists:foreach(fun({K, V}) ->
?assertMatch({K, V}, sst_get(OpenP, K)),
?assertMatch({K, V}, sst_get(OpenP, K))
end,
KVList1),
garbage_collect(OpenP),
B3 = check_binary_references(OpenP),
?assertMatch(0, B2), % Opens with an empty cache
?assertMatch(true, B3 > B2), % Now has headers in cache
?assertMatch(false, B3 > B0 * 2),
% Not significantly bigger than when created new
?assertMatch(false, B3 > B1 * 2),
% Not significantly bigger than when created new
ok = sst_close(OpenP),
ok = file:delete(filename:join(RP, Filename ++ ".sst")).
check_binary_references(Pid) ->
garbage_collect(Pid),
{binary, BinList} = process_info(Pid, binary),
TotalBinMem =
lists:foldl(fun({_R, BM, _RC}, Acc) -> Acc + BM end, 0, BinList),
io:format(user, "Total binary memory ~w~n", [TotalBinMem]),
TotalBinMem.
key_dominates_test() ->
KV1 = {{o, "Bucket", "Key1", null}, {5, {active, infinity}, 0, []}},
KV2 = {{o, "Bucket", "Key3", null}, {6, {active, infinity}, 0, []}},