Recovery Tests

Some initial entropy tests showing loss of data from a corrupted CDB
file.

src/leveled_bookie.erl

@@ -314,6 +314,8 @@ handle_call({get, Bucket, Key, Tag}, _From, State) ->
                     Active = TS >= leveled_codec:integer_now(),
                     case {Active,
                             fetch_value(LedgerKey, Seqn, State#state.inker)} of
+                        {_, not_present} ->
+                            {reply, not_found, State};
                         {true, Object} ->
                             {reply, {ok, Object}, State};
                         _ ->

src/leveled_cdb.erl

@@ -904,7 +904,7 @@ extract_kvpair(_, [], _) ->
 extract_kvpair(Handle, [Position|Rest], Key) ->
     {ok, _} = file:position(Handle, Position),
     {KeyLength, ValueLength} = read_next_2_integers(Handle),
-    case read_next_term(Handle, KeyLength) of
+    case safe_read_next_term(Handle, KeyLength) of
         Key ->  % If same key as passed in, then found!
             case read_next_term(Handle, ValueLength, crc) of
                 {false, _} -> 
@@ -1094,7 +1094,7 @@ read_next_term(Handle, Length, crc) ->
         CRC ->
             {true, binary_to_term(Bin)};
         _ ->
-            {false, binary_to_term(Bin)}
+            {false, crc_wonky}
     end.
 
 %% Extract value and size from binary containing CRC

src/leveled_inker.erl

@@ -249,7 +249,7 @@ handle_call({fetch, Key, SQN}, _From, State) ->
             {reply, {ok, Value}, State};
         Other ->
             io:format("Unexpected failure to fetch value for" ++
-                        "Key=~w SQN=~w with reason ~w", [Key, SQN, Other]),
+                        "Key=~w SQN=~w with reason ~w~n", [Key, SQN, Other]),
             {reply, not_present, State}
     end;
 handle_call({get, Key, SQN}, _From, State) ->