umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Improved unit test of CRC chekcing in bloom filter

Browse source 
Confirm the impact of bit-flipping in the bloom filter
This commit is contained in:
martinsumner 2016-10-21 16:08:41 +01:00
parent 3710d09fbf
commit 0a2053b557
3 changed files with 68 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

15
src/leveled_codec.erl
View file

@ -40,6 +40,7 @@
is_active/2,
endkey_passed/2,
key_dominates/2,
maybe_reap_expiredkey/2,
print_key/1,
to_ledgerkey/3,
to_ledgerkey/5,
@ -86,6 +87,20 @@ key_dominates(LeftKey, RightKey) ->
right_hand_dominant
end.
maybe_reap_expiredkey(KV, IsBasement) ->
Status = strip_to_statusonly(KV),
maybe_reap(Status, IsBasement).
maybe_reap({_, infinity}, _) ->
false; % key is not set to expire
maybe_reap({_, TS}, {basement, CurrTS}) when CurrTS > TS ->
true; % basement and ready to expire
maybe_reap(tomb, {basement, _CurrTS}) ->
true; % always expire in basement
maybe_reap(_, _) ->
false.
is_active(Key, Value) ->
case strip_to_statusonly({Key, Value}) of
{active, infinity} ->

76
src/leveled_sft.erl
View file

@ -1053,16 +1053,14 @@ key_dominates(KL1, KL2, Level) ->
Level).
key_dominates_expanded([H1|T1], [], Level) ->
St1 = leveled_codec:strip_to_statusonly(H1),
case maybe_reap_expiredkey(St1, Level) of
case leveled_codec:maybe_reap_expiredkey(H1, Level) of
true ->
{skipped_key, maybe_expand_pointer(T1), []};
false ->
{{next_key, H1}, maybe_expand_pointer(T1), []}
end;
key_dominates_expanded([], [H2|T2], Level) ->
St2 = leveled_codec:strip_to_statusonly(H2),
case maybe_reap_expiredkey(St2, Level) of
case leveled_codec:maybe_reap_expiredkey(H2, Level) of
true ->
{skipped_key, [], maybe_expand_pointer(T2)};
false ->
@ -1071,37 +1069,26 @@ key_dominates_expanded([], [H2|T2], Level) ->
key_dominates_expanded([H1|T1], [H2|T2], Level) ->
case leveled_codec:key_dominates(H1, H2) of
left_hand_first ->
St1 = leveled_codec:strip_to_statusonly(H1),
case maybe_reap_expiredkey(St1, Level) of
case leveled_codec:maybe_reap_expiredkey(H1, Level) of
true ->
{skipped_key, maybe_expand_pointer(T1), [H2|T2]};
false ->
{{next_key, H1}, maybe_expand_pointer(T1), [H2|T2]}
end;
left_hand_dominant ->
{skipped_key, [H1|T1], maybe_expand_pointer(T2)};
right_hand_dominant ->
{skipped_key, maybe_expand_pointer(T1), [H2|T2]};
right_hand_first ->
St2 = leveled_codec:strip_to_statusonly(H2),
case maybe_reap_expiredkey(St2, Level) of
case leveled_codec:maybe_reap_expiredkey(H2, Level) of
true ->
{skipped_key, [H1|T1], maybe_expand_pointer(T2)};
false ->
{{next_key, H2}, [H1|T1], maybe_expand_pointer(T2)}
end
end;
left_hand_dominant ->
{skipped_key, [H1|T1], maybe_expand_pointer(T2)};
right_hand_dominant ->
{skipped_key, maybe_expand_pointer(T1), [H2|T2]}
end.
maybe_reap_expiredkey({_, infinity}, _) ->
false; % key is not set to expire
maybe_reap_expiredkey({_, TS}, {basement, CurrTS}) when CurrTS > TS ->
true; % basement and ready to expire
maybe_reap_expiredkey(tomb, {basement, _CurrTS}) ->
true; % always expire in basement
maybe_reap_expiredkey(_, _) ->
false.
%% When a list is provided it may include a pointer to gain another batch of
%% entries from the same file, or a new batch of entries from another file
%%
@ -1550,8 +1537,49 @@ merge_seglists_test() ->
R8 = check_for_segments(SegBin, [0,900], false),
?assertMatch(R8, {maybe_present, [0]}),
R9 = check_for_segments(SegBin, [1024*1024 - 1], false),
?assertMatch(R9, not_present).
?assertMatch(R9, not_present),
io:format("Try corrupted bloom filter with flipped bit in " ++
"penultimate delta~n"),
ExpectedDeltasFlippedBit = <<0:1, 0:13, 0:2,
0:1, 50:13, 1:2,
0:1, 25:13, 2:2,
0:1, 25:13, 0:2,
0:1, 100:13, 0:2,
0:1, 0:13, 1:2,
2:2, 1709:13, 2:2>>,
SegBin1 = <<ExpectedTopHashes/bitstring,
7:16/integer,
ExpectedDeltasFlippedBit/bitstring,
0:7/integer>>,
?assertMatch(error_so_maybe_present,
check_for_segments(SegBin1, [900], true)),
?assertMatch(error_so_maybe_present,
check_for_segments(SegBin1, [200], true)),
?assertMatch(error_so_maybe_present,
check_for_segments(SegBin1, [0,900], true)),
?assertMatch(error_so_maybe_present,
check_for_segments(SegBin1, [1024*1024 - 1], true)),
% This match is before the flipped bit, so still works without CRC check
?assertMatch({maybe_present, [0]},
check_for_segments(SegBin1, [0,900], false)),
io:format("Try corrupted bloom filter with flipped bit in " ++
"final block's top hash~n"),
ExpectedTopHashesFlippedBit = <<200:20, 200:20, 10000:20, 1:20>>,
SegBin2 = <<ExpectedTopHashesFlippedBit/bitstring,
7:16/integer,
ExpectedDeltas/bitstring,
0:7/integer>>,
?assertMatch(error_so_maybe_present,
check_for_segments(SegBin2, [900], true)),
?assertMatch(error_so_maybe_present,
check_for_segments(SegBin2, [200], true)),
?assertMatch(error_so_maybe_present,
check_for_segments(SegBin2, [0,900], true)),
?assertMatch(error_so_maybe_present,
check_for_segments(SegBin2, [1024*1024 - 1], true)),
% This match is before the flipped bit, so still works without CRC check
?assertMatch({maybe_present, [0]},
check_for_segments(SegBin2, [0,900], false)).
createslot_stage1_test() ->
{KeyList1, KeyList2} = sample_keylist(),

2
test/end_to_end/iterator_SUITE.erl
View file

@ -43,7 +43,7 @@ simple_load_with2i(_Config) ->
simple_querycount(_Config) ->
RootPath = testutil:reset_filestructure(),
{ok, Book1} = leveled_bookie:book_start(RootPath, 4000, 50000000),
{ok, Book1} = leveled_bookie:book_start(RootPath, 2500, 50000000),
{TestObject, TestSpec} = testutil:generate_testobject("Bucket",
"Key1",
"Value1",