Merge remote-tracking branch 'refs/remotes/origin/mas-sstblock-i42' into mas-sstblockv2-i42

src/leveled_codec.erl

@@ -207,32 +207,41 @@ compact_inkerkvc({_InkerKey, crc_wonky, false}, _Strategy) ->
 compact_inkerkvc({{_SQN, ?INKT_TOMB, _LK}, _V, _CrcCheck}, _Strategy) ->
     skip;
 compact_inkerkvc({{SQN, ?INKT_KEYD, LK}, V, CrcCheck}, Strategy) ->
-    {Tag, _, _, _} = LK,
+    case get_tagstrategy(LK, Strategy) of
-    {Tag, TagStrat} = lists:keyfind(Tag, 1, Strategy),
+        skip ->
-    case TagStrat of
+            skip;
         retain ->
             {retain, {{SQN, ?INKT_KEYD, LK}, V, CrcCheck}};
         TagStrat ->
             {TagStrat, null}
     end;
 compact_inkerkvc({{SQN, ?INKT_STND, LK}, V, CrcCheck}, Strategy) ->
-    {Tag, _, _, _} = LK,
+    case get_tagstrategy(LK, Strategy) of
-    case lists:keyfind(Tag, 1, Strategy) of
+        skip ->
-        {Tag, TagStrat} ->
+            skip;
-            case TagStrat of
+        retain ->
-                retain ->
+            {_V, KeyDeltas} = split_inkvalue(V),    
-                    {_V, KeyDeltas} = split_inkvalue(V),    
+            {retain, {{SQN, ?INKT_KEYD, LK}, {null, KeyDeltas}, CrcCheck}};
-                    {TagStrat, {{SQN, ?INKT_KEYD, LK}, {null, KeyDeltas}, CrcCheck}}; 
+        TagStrat ->
-                TagStrat ->
+            {TagStrat, null}
-                    {TagStrat, null}
-            end;
-        false ->
-            leveled_log:log("IC012", [Tag, Strategy]),
-            skip
     end;
 compact_inkerkvc(_KVC, _Strategy) ->
     skip.
 
+get_tagstrategy(LK, Strategy) ->
+    case LK of
+        {Tag, _, _, _} ->
+            case lists:keyfind(Tag, 1, Strategy) of
+                {Tag, TagStrat} ->
+                    TagStrat;
+                false ->
+                    leveled_log:log("IC012", [Tag, Strategy]),
+                    skip
+            end;
+        _ ->
+            skip
+    end.
+
 split_inkvalue(VBin) ->
     case is_binary(VBin) of
             true ->
@@ -438,6 +447,26 @@ endkey_passed_test() ->
     ?assertMatch(true, endkey_passed(TestKey, K2)).
 
 
+corrupted_ledgerkey_test() ->
+    % When testing for compacted journal which has been corrupted, there may
+    % be a corruptes ledger key.  Always skip these keys
+    % Key has become a 3-tuple not a 4-tuple
+    TagStrat1 = compact_inkerkvc({{1,
+                                        ?INKT_STND,
+                                        {?STD_TAG, "B1", "K1andSK"}},
+                                    {},
+                                    true},
+                                    [{?STD_TAG, retain}]),
+    ?assertMatch(skip, TagStrat1),
+    TagStrat2 = compact_inkerkvc({{1,
+                                        ?INKT_KEYD,
+                                        {?STD_TAG, "B1", "K1andSK"}},
+                                    {},
+                                    true},
+                                    [{?STD_TAG, retain}]),
+    ?assertMatch(skip, TagStrat2).
+    
+
 %% Test below proved that the overhead of performing hashes was trivial
 %% Maybe 5 microseconds per hash
 

src/leveled_log.erl

@@ -166,7 +166,8 @@
     {"PC015",
         {info, "File created"}},
     {"PC016",
-        {info, "Slow fetch from SFT ~w of ~w microseconds with result ~w"}},
+        {info, "Slow fetch from SFT ~w of ~w microseconds at level ~w "
+                    ++ "with result ~w"}},
     {"PC017",
         {info, "Notified clerk of manifest change"}},
     {"PC018",

src/leveled_penciller.erl

@@ -853,7 +853,7 @@ fetch_mem(Key, Hash, Manifest, L0Cache, L0Index) ->
     L0Check = leveled_pmem:check_levelzero(Key, Hash, PosList, L0Cache),
     case L0Check of
         {false, not_found} ->
-            fetch(Key, Hash, Manifest, 0, fun timed_sst_get/3);
+            fetch(Key, Hash, Manifest, 0, fun timed_sst_get/4);
         {true, KV} ->
             {KV, 0}
     end.
@@ -865,7 +865,7 @@ fetch(Key, Hash, Manifest, Level, FetchFun) ->
         false ->
             fetch(Key, Hash, Manifest, Level + 1, FetchFun);
         FP ->
-            case FetchFun(FP, Key, Hash) of
+            case FetchFun(FP, Key, Hash, Level) of
                 not_present ->
                     fetch(Key, Hash, Manifest, Level + 1, FetchFun);
                 ObjectFound ->
@@ -873,21 +873,21 @@ fetch(Key, Hash, Manifest, Level, FetchFun) ->
             end
     end.
     
-timed_sst_get(PID, Key, Hash) ->
+timed_sst_get(PID, Key, Hash, Level) ->
     SW = os:timestamp(),
     R = leveled_sst:sst_get(PID, Key, Hash),
     T0 = timer:now_diff(os:timestamp(), SW),
-    log_slowfetch(T0, R, PID, ?SLOW_FETCH).
+    log_slowfetch(T0, R, PID, Level, ?SLOW_FETCH).
 
-log_slowfetch(T0, R, PID, FetchTolerance) ->
+log_slowfetch(T0, R, PID, Level, FetchTolerance) ->
     case {T0, R} of
         {T, R} when T < FetchTolerance ->
             R;
         {T, not_present} ->
-            leveled_log:log("PC016", [PID, T, not_present]),
+            leveled_log:log("PC016", [PID, T, Level, not_present]),
             not_present;
         {T, R} ->
-            leveled_log:log("PC016", [PID, T, found]),
+            leveled_log:log("PC016", [PID, T, Level, found]),
             R
     end.
 
@@ -1498,7 +1498,7 @@ create_file_test() ->
     ?assertMatch("hello", binary_to_term(Bin)).
 
 slow_fetch_test() ->
-    ?assertMatch(not_present, log_slowfetch(2, not_present, "fake", 1)).
+    ?assertMatch(not_present, log_slowfetch(2, not_present, "fake", 0, 1)).
 
 checkready(Pid) ->
     try