From f3e962c43ac46170fa26a27ef4697d11e1cd31ce Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 13 Mar 2017 12:16:36 +0000 Subject: [PATCH 1/2] Add level to SST slow fetch log --- src/leveled_log.erl | 3 ++- src/leveled_penciller.erl | 16 ++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/leveled_log.erl b/src/leveled_log.erl index c89fbea..7f42bf3 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -166,7 +166,8 @@ {"PC015", {info, "File created"}}, {"PC016", - {info, "Slow fetch from SFT ~w of ~w microseconds with result ~w"}}, + {info, "Slow fetch from SFT ~w of ~w microseconds at level ~w " + ++ "with result ~w"}}, {"PC017", {info, "Notified clerk of manifest change"}}, {"PC018", diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 2f5c1a5..110889a 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -853,7 +853,7 @@ fetch_mem(Key, Hash, Manifest, L0Cache, L0Index) -> L0Check = leveled_pmem:check_levelzero(Key, Hash, PosList, L0Cache), case L0Check of {false, not_found} -> - fetch(Key, Hash, Manifest, 0, fun timed_sst_get/3); + fetch(Key, Hash, Manifest, 0, fun timed_sst_get/4); {true, KV} -> {KV, 0} end. @@ -865,7 +865,7 @@ fetch(Key, Hash, Manifest, Level, FetchFun) -> false -> fetch(Key, Hash, Manifest, Level + 1, FetchFun); FP -> - case FetchFun(FP, Key, Hash) of + case FetchFun(FP, Key, Hash, Level) of not_present -> fetch(Key, Hash, Manifest, Level + 1, FetchFun); ObjectFound -> @@ -873,21 +873,21 @@ fetch(Key, Hash, Manifest, Level, FetchFun) -> end end. -timed_sst_get(PID, Key, Hash) -> +timed_sst_get(PID, Key, Hash, Level) -> SW = os:timestamp(), R = leveled_sst:sst_get(PID, Key, Hash), T0 = timer:now_diff(os:timestamp(), SW), - log_slowfetch(T0, R, PID, ?SLOW_FETCH). + log_slowfetch(T0, R, PID, Level, ?SLOW_FETCH). -log_slowfetch(T0, R, PID, FetchTolerance) -> +log_slowfetch(T0, R, PID, Level, FetchTolerance) -> case {T0, R} of {T, R} when T < FetchTolerance -> R; {T, not_present} -> - leveled_log:log("PC016", [PID, T, not_present]), + leveled_log:log("PC016", [PID, T, Level, not_present]), not_present; {T, R} -> - leveled_log:log("PC016", [PID, T, found]), + leveled_log:log("PC016", [PID, T, Level, found]), R end. @@ -1498,7 +1498,7 @@ create_file_test() -> ?assertMatch("hello", binary_to_term(Bin)). slow_fetch_test() -> - ?assertMatch(not_present, log_slowfetch(2, not_present, "fake", 1)). + ?assertMatch(not_present, log_slowfetch(2, not_present, "fake", 0, 1)). checkready(Pid) -> try From c787e0cd784eac80d5d6440304f0760c6bbbddaa Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 13 Mar 2017 14:32:46 +0000 Subject: [PATCH 2/2] Handle corrupted Ledger Key when applying recovery strategy Otherwise may blow up in journal_compaction_bustedjournal test --- src/leveled_codec.erl | 61 +++++++++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index ad25c58..a01f008 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -198,32 +198,41 @@ compact_inkerkvc({_InkerKey, crc_wonky, false}, _Strategy) -> compact_inkerkvc({{_SQN, ?INKT_TOMB, _LK}, _V, _CrcCheck}, _Strategy) -> skip; compact_inkerkvc({{SQN, ?INKT_KEYD, LK}, V, CrcCheck}, Strategy) -> - {Tag, _, _, _} = LK, - {Tag, TagStrat} = lists:keyfind(Tag, 1, Strategy), - case TagStrat of + case get_tagstrategy(LK, Strategy) of + skip -> + skip; retain -> {retain, {{SQN, ?INKT_KEYD, LK}, V, CrcCheck}}; TagStrat -> {TagStrat, null} end; compact_inkerkvc({{SQN, ?INKT_STND, LK}, V, CrcCheck}, Strategy) -> - {Tag, _, _, _} = LK, - case lists:keyfind(Tag, 1, Strategy) of - {Tag, TagStrat} -> - case TagStrat of - retain -> - {_V, KeyDeltas} = split_inkvalue(V), - {TagStrat, {{SQN, ?INKT_KEYD, LK}, {null, KeyDeltas}, CrcCheck}}; - TagStrat -> - {TagStrat, null} - end; - false -> - leveled_log:log("IC012", [Tag, Strategy]), - skip + case get_tagstrategy(LK, Strategy) of + skip -> + skip; + retain -> + {_V, KeyDeltas} = split_inkvalue(V), + {retain, {{SQN, ?INKT_KEYD, LK}, {null, KeyDeltas}, CrcCheck}}; + TagStrat -> + {TagStrat, null} end; compact_inkerkvc(_KVC, _Strategy) -> skip. +get_tagstrategy(LK, Strategy) -> + case LK of + {Tag, _, _, _} -> + case lists:keyfind(Tag, 1, Strategy) of + {Tag, TagStrat} -> + TagStrat; + false -> + leveled_log:log("IC012", [Tag, Strategy]), + skip + end; + _ -> + skip + end. + split_inkvalue(VBin) -> case is_binary(VBin) of true -> @@ -429,6 +438,26 @@ endkey_passed_test() -> ?assertMatch(true, endkey_passed(TestKey, K2)). +corrupted_ledgerkey_test() -> + % When testing for compacted journal which has been corrupted, there may + % be a corruptes ledger key. Always skip these keys + % Key has become a 3-tuple not a 4-tuple + TagStrat1 = compact_inkerkvc({{1, + ?INKT_STND, + {?STD_TAG, "B1", "K1andSK"}}, + {}, + true}, + [{?STD_TAG, retain}]), + ?assertMatch(skip, TagStrat1), + TagStrat2 = compact_inkerkvc({{1, + ?INKT_KEYD, + {?STD_TAG, "B1", "K1andSK"}}, + {}, + true}, + [{?STD_TAG, retain}]), + ?assertMatch(skip, TagStrat2). + + %% Test below proved that the overhead of performing hashes was trivial %% Maybe 5 microseconds per hash