From 9a0082db4eb88fbb9068ecb01d45e1aeb0ab7971 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 3 Oct 2018 18:29:20 +0100 Subject: [PATCH 1/2] Test to expose startup issue See #194 --- test/end_to_end/basic_SUITE.erl | 62 ++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 4234aaf..968b56a 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -12,21 +12,23 @@ is_empty_test/1, many_put_fetch_switchcompression/1, bigjournal_littlejournal/1, - safereaderror_startup/1 + safereaderror_startup/1, + remove_journal_test/1 ]). all() -> [ - simple_put_fetch_head_delete, - many_put_fetch_head, - journal_compaction, - fetchput_snapshot, - load_and_count, - load_and_count_withdelete, - space_clear_ondelete, - is_empty_test, - many_put_fetch_switchcompression, - bigjournal_littlejournal, - safereaderror_startup + % simple_put_fetch_head_delete, + % many_put_fetch_head, + % journal_compaction, + % fetchput_snapshot, + % load_and_count, + % load_and_count_withdelete, + % space_clear_ondelete, + % is_empty_test, + % many_put_fetch_switchcompression, + % bigjournal_littlejournal, + % safereaderror_startup, + remove_journal_test ]. @@ -831,6 +833,42 @@ is_empty_test(_Config) -> ok = leveled_bookie:book_close(Bookie1). +remove_journal_test(_Config) -> + RootPath = testutil:reset_filestructure(), + StartOpts1 = [{root_path, RootPath}, + {max_pencillercachesize, 16000}, + {sync_strategy, testutil:sync_strategy()}, + {compression_point, on_compact}], + {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), + GenList = [1, 20001, 40001, 60001], + CLs = testutil:load_objects(20000, GenList, Bookie1, no_check, + fun testutil:generate_smallobjects/2), + CheckList1 = lists:sublist(lists:nth(1, CLs), 100, 1000), + CheckList2 = lists:sublist(lists:nth(2, CLs), 100, 1000), + CheckList3 = lists:sublist(lists:nth(3, CLs), 100, 1000), + CheckList4 = lists:sublist(lists:nth(4, CLs), 100, 1000), + testutil:check_forlist(Bookie1, CheckList1), + testutil:check_forlist(Bookie1, CheckList2), + testutil:check_forlist(Bookie1, CheckList3), + testutil:check_forlist(Bookie1, CheckList4), + + ok = leveled_bookie:book_close(Bookie1), + leveled_inker:clean_testdir(RootPath ++ "/journal"), + {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), + + % If we're not careful here new data will be added, and we + % won't be able to read it + [NewCheckList] = + testutil:load_objects(1000, [80001], Bookie2, no_check, + fun testutil:generate_smallobjects/2), + + ok = leveled_bookie:book_close(Bookie2), + {ok, Bookie3} = leveled_bookie:book_start(StartOpts1), + testutil:check_forlist(Bookie3, NewCheckList), + ok = leveled_bookie:book_destroy(Bookie3). + + + many_put_fetch_switchcompression(_Config) -> RootPath = testutil:reset_filestructure(), StartOpts1 = [{root_path, RootPath}, From 0bbaf1f25eea834063d3356e95f96250f57c608e Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 3 Oct 2018 18:38:56 +0100 Subject: [PATCH 2/2] Handle missing journal files without data loss Make sure data added after the journal has been lost will not be lost (even though lost data may stay lost) --- src/leveled_bookie.erl | 1 + src/leveled_inker.erl | 15 +++++++++++++++ src/leveled_log.erl | 3 +++ test/end_to_end/basic_SUITE.erl | 22 +++++++++++----------- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index b160db6..39384d9 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -1419,6 +1419,7 @@ startup(InkerOpts, PencillerOpts, State) -> LedgerSQN + 1, get_loadfun(State), Penciller), + ok = leveled_inker:ink_checksqn(Inker, LedgerSQN), {Inker, Penciller}. diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 0412b52..b1dec77 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -115,6 +115,7 @@ ink_doom/1, ink_roll/1, ink_backup/2, + ink_checksqn/2, build_dummy_journal/0, clean_testdir/1, filepath/2, @@ -432,6 +433,12 @@ ink_updatemanifest(Pid, ManifestSnippet, DeletedFiles) -> ink_printmanifest(Pid) -> gen_server:call(Pid, print_manifest, infinity). +-spec ink_checksqn(pid(), integer()) -> ok. +%% @doc +%% Check that the Inker doesn't have a SQN behind that of the Ledger +ink_checksqn(Pid, LedgerSQN) -> + gen_server:call(Pid, {check_sqn, LedgerSQN}). + %%%============================================================================ %%% gen_server callbacks %%%============================================================================ @@ -633,6 +640,14 @@ handle_call({backup, BackupPath}, _from, State) length(BackupManifest)], SW), {reply, ok, State}; +handle_call({check_sqn, LedgerSQN}, _From, State) -> + case State#state.journal_sqn of + JSQN when JSQN < LedgerSQN -> + leveled_log:log("I0025", [JSQN, LedgerSQN]), + {reply, ok, State#state{journal_sqn = LedgerSQN}}; + _JSQN -> + {reply, ok, State} + end; handle_call(close, _From, State) -> case State#state.is_snapshot of true -> diff --git a/src/leveled_log.erl b/src/leveled_log.erl index b0eb9a9..d48c597 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -290,6 +290,9 @@ {info, "Backup commencing into folder with ~w existing files"}}, {"I0024", {info, "Prompted roll at NewSQN=~w"}}, + {"I0025", + {warn, "Journal SQN of ~w is below Ledger SQN of ~w " ++ + "anti-entropy will be required"}}, {"IC001", {info, "Closed for reason ~w so maybe leaving garbage"}}, diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 968b56a..cd29e60 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -17,17 +17,17 @@ ]). all() -> [ - % simple_put_fetch_head_delete, - % many_put_fetch_head, - % journal_compaction, - % fetchput_snapshot, - % load_and_count, - % load_and_count_withdelete, - % space_clear_ondelete, - % is_empty_test, - % many_put_fetch_switchcompression, - % bigjournal_littlejournal, - % safereaderror_startup, + simple_put_fetch_head_delete, + many_put_fetch_head, + journal_compaction, + fetchput_snapshot, + load_and_count, + load_and_count_withdelete, + space_clear_ondelete, + is_empty_test, + many_put_fetch_switchcompression, + bigjournal_littlejournal, + safereaderror_startup, remove_journal_test ].