From 0a2053b557707c36c18358ea6603f708a2500ab3 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 21 Oct 2016 16:08:41 +0100 Subject: [PATCH] Improved unit test of CRC chekcing in bloom filter Confirm the impact of bit-flipping in the bloom filter --- src/leveled_codec.erl | 15 ++++++ src/leveled_sft.erl | 76 ++++++++++++++++++++---------- test/end_to_end/iterator_SUITE.erl | 2 +- 3 files changed, 68 insertions(+), 25 deletions(-) diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index 875fb88..535ab78 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -40,6 +40,7 @@ is_active/2, endkey_passed/2, key_dominates/2, + maybe_reap_expiredkey/2, print_key/1, to_ledgerkey/3, to_ledgerkey/5, @@ -86,6 +87,20 @@ key_dominates(LeftKey, RightKey) -> right_hand_dominant end. + +maybe_reap_expiredkey(KV, IsBasement) -> + Status = strip_to_statusonly(KV), + maybe_reap(Status, IsBasement). + +maybe_reap({_, infinity}, _) -> + false; % key is not set to expire +maybe_reap({_, TS}, {basement, CurrTS}) when CurrTS > TS -> + true; % basement and ready to expire +maybe_reap(tomb, {basement, _CurrTS}) -> + true; % always expire in basement +maybe_reap(_, _) -> + false. + is_active(Key, Value) -> case strip_to_statusonly({Key, Value}) of {active, infinity} -> diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 60054a1..b88b254 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -1053,16 +1053,14 @@ key_dominates(KL1, KL2, Level) -> Level). key_dominates_expanded([H1|T1], [], Level) -> - St1 = leveled_codec:strip_to_statusonly(H1), - case maybe_reap_expiredkey(St1, Level) of + case leveled_codec:maybe_reap_expiredkey(H1, Level) of true -> {skipped_key, maybe_expand_pointer(T1), []}; false -> {{next_key, H1}, maybe_expand_pointer(T1), []} end; key_dominates_expanded([], [H2|T2], Level) -> - St2 = leveled_codec:strip_to_statusonly(H2), - case maybe_reap_expiredkey(St2, Level) of + case leveled_codec:maybe_reap_expiredkey(H2, Level) of true -> {skipped_key, [], maybe_expand_pointer(T2)}; false -> @@ -1071,37 +1069,26 @@ key_dominates_expanded([], [H2|T2], Level) -> key_dominates_expanded([H1|T1], [H2|T2], Level) -> case leveled_codec:key_dominates(H1, H2) of left_hand_first -> - St1 = leveled_codec:strip_to_statusonly(H1), - case maybe_reap_expiredkey(St1, Level) of + case leveled_codec:maybe_reap_expiredkey(H1, Level) of true -> {skipped_key, maybe_expand_pointer(T1), [H2|T2]}; false -> {{next_key, H1}, maybe_expand_pointer(T1), [H2|T2]} end; - left_hand_dominant -> - {skipped_key, [H1|T1], maybe_expand_pointer(T2)}; - right_hand_dominant -> - {skipped_key, maybe_expand_pointer(T1), [H2|T2]}; right_hand_first -> - St2 = leveled_codec:strip_to_statusonly(H2), - case maybe_reap_expiredkey(St2, Level) of + case leveled_codec:maybe_reap_expiredkey(H2, Level) of true -> {skipped_key, [H1|T1], maybe_expand_pointer(T2)}; false -> {{next_key, H2}, [H1|T1], maybe_expand_pointer(T2)} - end + end; + left_hand_dominant -> + {skipped_key, [H1|T1], maybe_expand_pointer(T2)}; + right_hand_dominant -> + {skipped_key, maybe_expand_pointer(T1), [H2|T2]} end. -maybe_reap_expiredkey({_, infinity}, _) -> - false; % key is not set to expire -maybe_reap_expiredkey({_, TS}, {basement, CurrTS}) when CurrTS > TS -> - true; % basement and ready to expire -maybe_reap_expiredkey(tomb, {basement, _CurrTS}) -> - true; % always expire in basement -maybe_reap_expiredkey(_, _) -> - false. - %% When a list is provided it may include a pointer to gain another batch of %% entries from the same file, or a new batch of entries from another file %% @@ -1550,8 +1537,49 @@ merge_seglists_test() -> R8 = check_for_segments(SegBin, [0,900], false), ?assertMatch(R8, {maybe_present, [0]}), R9 = check_for_segments(SegBin, [1024*1024 - 1], false), - ?assertMatch(R9, not_present). - + ?assertMatch(R9, not_present), + io:format("Try corrupted bloom filter with flipped bit in " ++ + "penultimate delta~n"), + ExpectedDeltasFlippedBit = <<0:1, 0:13, 0:2, + 0:1, 50:13, 1:2, + 0:1, 25:13, 2:2, + 0:1, 25:13, 0:2, + 0:1, 100:13, 0:2, + 0:1, 0:13, 1:2, + 2:2, 1709:13, 2:2>>, + SegBin1 = <>, + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin1, [900], true)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin1, [200], true)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin1, [0,900], true)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin1, [1024*1024 - 1], true)), + % This match is before the flipped bit, so still works without CRC check + ?assertMatch({maybe_present, [0]}, + check_for_segments(SegBin1, [0,900], false)), + io:format("Try corrupted bloom filter with flipped bit in " ++ + "final block's top hash~n"), + ExpectedTopHashesFlippedBit = <<200:20, 200:20, 10000:20, 1:20>>, + SegBin2 = <>, + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin2, [900], true)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin2, [200], true)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin2, [0,900], true)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin2, [1024*1024 - 1], true)), + % This match is before the flipped bit, so still works without CRC check + ?assertMatch({maybe_present, [0]}, + check_for_segments(SegBin2, [0,900], false)). createslot_stage1_test() -> {KeyList1, KeyList2} = sample_keylist(), diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index 8fb7505..f00571e 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -43,7 +43,7 @@ simple_load_with2i(_Config) -> simple_querycount(_Config) -> RootPath = testutil:reset_filestructure(), - {ok, Book1} = leveled_bookie:book_start(RootPath, 4000, 50000000), + {ok, Book1} = leveled_bookie:book_start(RootPath, 2500, 50000000), {TestObject, TestSpec} = testutil:generate_testobject("Bucket", "Key1", "Value1",