From 73004328e1bc76b3f4cf504a3dcf20bde46ae02a Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 31 Oct 2016 20:58:19 +0000 Subject: [PATCH] Recovery Tests Some initial entropy tests showing loss of data from a corrupted CDB file. --- src/leveled_bookie.erl | 2 + src/leveled_cdb.erl | 4 +- src/leveled_inker.erl | 2 +- .../{restart_SUITE.erl => recovery_SUITE.erl} | 75 ++++++++++++++++++- 4 files changed, 77 insertions(+), 6 deletions(-) rename test/end_to_end/{restart_SUITE.erl => recovery_SUITE.erl} (60%) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index e67e334..8ab54cf 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -314,6 +314,8 @@ handle_call({get, Bucket, Key, Tag}, _From, State) -> Active = TS >= leveled_codec:integer_now(), case {Active, fetch_value(LedgerKey, Seqn, State#state.inker)} of + {_, not_present} -> + {reply, not_found, State}; {true, Object} -> {reply, {ok, Object}, State}; _ -> diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 29bbbfd..454c4ec 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -904,7 +904,7 @@ extract_kvpair(_, [], _) -> extract_kvpair(Handle, [Position|Rest], Key) -> {ok, _} = file:position(Handle, Position), {KeyLength, ValueLength} = read_next_2_integers(Handle), - case read_next_term(Handle, KeyLength) of + case safe_read_next_term(Handle, KeyLength) of Key -> % If same key as passed in, then found! case read_next_term(Handle, ValueLength, crc) of {false, _} -> @@ -1094,7 +1094,7 @@ read_next_term(Handle, Length, crc) -> CRC -> {true, binary_to_term(Bin)}; _ -> - {false, binary_to_term(Bin)} + {false, crc_wonky} end. %% Extract value and size from binary containing CRC diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index b91976c..3eb8272 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -249,7 +249,7 @@ handle_call({fetch, Key, SQN}, _From, State) -> {reply, {ok, Value}, State}; Other -> io:format("Unexpected failure to fetch value for" ++ - "Key=~w SQN=~w with reason ~w", [Key, SQN, Other]), + "Key=~w SQN=~w with reason ~w~n", [Key, SQN, Other]), {reply, not_present, State} end; handle_call({get, Key, SQN}, _From, State) -> diff --git a/test/end_to_end/restart_SUITE.erl b/test/end_to_end/recovery_SUITE.erl similarity index 60% rename from test/end_to_end/restart_SUITE.erl rename to test/end_to_end/recovery_SUITE.erl index 65efbf7..54b058b 100644 --- a/test/end_to_end/restart_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -1,12 +1,14 @@ --module(restart_SUITE). +-module(recovery_SUITE). -include_lib("common_test/include/ct.hrl"). -include("include/leveled.hrl"). -export([all/0]). --export([retain_strategy/1 +-export([retain_strategy/1, + aae_bustedjournal/1 ]). all() -> [ - retain_strategy + retain_strategy, + aae_bustedjournal ]. retain_strategy(_Config) -> @@ -34,6 +36,73 @@ retain_strategy(_Config) -> +aae_bustedjournal(_Config) -> + RootPath = testutil:reset_filestructure(), + StartOpts = #bookie_options{root_path=RootPath, + max_journalsize=20000000}, + {ok, Bookie1} = leveled_bookie:book_start(StartOpts), + {TestObject, TestSpec} = testutil:generate_testobject(), + ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), + testutil:check_forobject(Bookie1, TestObject), + GenList = [2], + _CLs = testutil:load_objects(20000, GenList, Bookie1, TestObject, + fun testutil:generate_objects/2), + ok = leveled_bookie:book_close(Bookie1), + {ok, FNsA_J} = file:list_dir(RootPath ++ "/journal/journal_files"), + {ok, Regex} = re:compile(".*\.cdb"), + CDBFiles = lists:foldl(fun(FN, Acc) -> case re:run(FN, Regex) of + nomatch -> + Acc; + _ -> + [FN|Acc] + end + end, + [], + FNsA_J), + [HeadF|_Rest] = CDBFiles, + io:format("Selected Journal for corruption of ~s~n", [HeadF]), + {ok, Handle} = file:open(RootPath ++ "/journal/journal_files/" ++ HeadF, + [binary, raw, read, write]), + lists:foreach(fun(X) -> + Position = X * 1000 + 2048, + ok = file:pwrite(Handle, Position, <<0:8/integer>>) + end, + lists:seq(1, 1000)), + ok = file:close(Handle), + {ok, Bookie2} = leveled_bookie:book_start(StartOpts), + + {async, KeyF} = leveled_bookie:book_returnfolder(Bookie2, + {keylist, ?RIAK_TAG}), + KeyList = KeyF(), + 20001 = length(KeyList), + HeadCount = lists:foldl(fun({B, K}, Acc) -> + case leveled_bookie:book_riakhead(Bookie2, + B, + K) of + {ok, _} -> Acc + 1; + not_found -> Acc + end + end, + 0, + KeyList), + 20001 = HeadCount, + GetCount = lists:foldl(fun({B, K}, Acc) -> + case leveled_bookie:book_riakget(Bookie2, + B, + K) of + {ok, _} -> Acc + 1; + not_found -> Acc + end + end, + 0, + KeyList), + true = GetCount > 19000, + true = GetCount < HeadCount, + + ok = leveled_bookie:book_close(Bookie2), + testutil:reset_filestructure(). + + rotating_object_check(BookOpts, B, NumberOfObjects) -> {ok, Book1} = leveled_bookie:book_start(BookOpts), {KSpcL1, V1} = testutil:put_indexed_objects(Book1, B, NumberOfObjects),