From 0f7e4213713079d7e5f3f91f271c5122e43db21a Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 21 Nov 2016 12:34:40 +0000 Subject: [PATCH] Add destruction Allow a store to be cleared out and destroyed --- src/leveled_bookie.erl | 22 +++++++++++++++++++--- src/leveled_inker.erl | 13 ++++++++++++- src/leveled_log.erl | 6 ++++++ src/leveled_penciller.erl | 12 ++++++++++-- test/end_to_end/basic_SUITE.erl | 6 ++---- 5 files changed, 49 insertions(+), 10 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 40050b3..ff2382e 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -132,7 +132,8 @@ book_snapshotledger/3, book_compactjournal/2, book_islastcompactionpending/1, - book_close/1]). + book_close/1, + book_destroy/1]). -export([get_opt/2, get_opt/3]). @@ -214,6 +215,9 @@ book_islastcompactionpending(Pid) -> book_close(Pid) -> gen_server:call(Pid, close, infinity). +book_destroy(Pid) -> + gen_server:call(Pid, destroy, infinity). + %%%============================================================================ %%% gen_server callbacks %%%============================================================================ @@ -392,7 +396,8 @@ handle_call({compact_journal, Timeout}, _From, State) -> handle_call(confirm_compact, _From, State) -> {reply, leveled_inker:ink_compactionpending(State#state.inker), State}; handle_call(close, _From, State) -> - {stop, normal, ok, State}. + {stop, normal, ok, State}; + {stop, destroy, ok, State}. handle_cast(_Msg, State) -> {noreply, State}. @@ -400,6 +405,13 @@ handle_cast(_Msg, State) -> handle_info(_Info, State) -> {noreply, State}. +terminate(destroy, State) -> + leveled_log:log("B0011", []), + {ok, InkPathList} = leveled_inker:ink_doom(State#state.inker), + {ok, PCLPathList} = leveled_penciller:pcl_doom(State#state.penciller), + lists:foreach(fun(DirPath) -> delete_path(DirPath) end, InkPathList), + lists:foreach(fun(DirPath) -> delete_path(DirPath) end, PCLPathList), + ok; terminate(Reason, State) -> leveled_log:log("B0003", [Reason]), ok = leveled_inker:ink_close(State#state.inker), @@ -477,7 +489,6 @@ get_nextbucket(NextBucket, Tag, LedgerSnapshot, BKList) -> NB -> leveled_log:log("B0010",[NB]), [] - end. @@ -922,6 +933,11 @@ get_opt(Key, Opts, Default) -> Value end. +delete_path(DirPath) -> + ok = filelib:ensure_dir(DirPath), + {ok, Files} = file:list_dir(DirPath), + [file:delete(filename:join([DirPath, File])) || File <- Files], + file:del_dir(DirPath). %%%============================================================================ %%% Test diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index d731b89..acaad5f 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -108,6 +108,7 @@ ink_updatemanifest/3, ink_print_manifest/1, ink_close/1, + ink_doom/1, build_dummy_journal/0, simple_manifest_reader/2, clean_testdir/1, @@ -172,6 +173,9 @@ ink_confirmdelete(Pid, ManSQN) -> ink_close(Pid) -> gen_server:call(Pid, close, infinity). +ink_doom(Pid) -> + gen_server:call(Pid, doom, 60000). + ink_loadpcl(Pid, MinSQN, FilterFun, Penciller) -> gen_server:call(Pid, {load_pcl, MinSQN, FilterFun, Penciller}, infinity). @@ -324,7 +328,14 @@ handle_call(compaction_complete, _From, State) -> handle_call(compaction_pending, _From, State) -> {reply, State#state.compaction_pending, State}; handle_call(close, _From, State) -> - {stop, normal, ok, State}. + {stop, normal, ok, State}; +handle_call(doom, _From, State) -> + FPs = [filepath(State#state.root_path, journal_dir), + filepath(State#state.root_path, manifest_dir), + filepath(State#state.root_path, journal_compact_dir), + filepath(State#state.root_path, journal_waste_dir)], + leveled_log:log("I0018", []), + {stop, normal, {ok, FPs}, State}. handle_cast({release_snapshot, Snapshot}, State) -> Rs = lists:keydelete(Snapshot, 1, State#state.registered_snapshots), diff --git a/src/leveled_log.erl b/src/leveled_log.erl index d33928d..9a9c668 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -38,6 +38,8 @@ {info, "Bucket list finds Bucket ~w"}}, {"B0010", {info, "Bucket list finds non-binary Bucket ~w"}}, + {"B0011", + {warn, "Call to destroy the store and so all files to be removed"}}, {"P0001", {info, "Ledger snapshot ~w registered"}}, @@ -100,6 +102,8 @@ {info, "Adding cleared file ~s to deletion list"}}, {"P0029", {info, "L0 completion confirmed and will transition to not pending"}}, + {"P0030", + {warn, "We're doomed - intention recorded to destroy all files"}}, {"PC001", {info, "Penciller's clerk ~w started with owner ~w"}}, @@ -168,6 +172,8 @@ {info, "Writing new version of manifest for manifestSQN=~w"}}, {"I0017", {info, "At SQN=~w journal has filename ~s"}}, + {"I0018", + {warn, "We're doomed - intention recorded to destroy all files"}}, {"IC001", {info, "Closed for reason ~w so maybe leaving garbage"}}, diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 87dcbdc..1c24470 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -176,6 +176,7 @@ pcl_confirml0complete/4, pcl_confirmdelete/2, pcl_close/1, + pcl_doom/1, pcl_registersnapshot/2, pcl_releasesnapshot/2, pcl_loadsnapshot/2, @@ -289,6 +290,8 @@ pcl_loadsnapshot(Pid, Increment) -> pcl_close(Pid) -> gen_server:call(Pid, close, 60000). +pcl_doom(Pid) -> + gen_server:call(Pid, doom, 60000). %%%============================================================================ %%% gen_server callbacks @@ -405,8 +408,12 @@ handle_call({load_snapshot, BookieIncrTree}, _From, State) -> handle_call({fetch_levelzero, Slot}, _From, State) -> {reply, lists:nth(Slot, State#state.levelzero_cache), State}; handle_call(close, _From, State) -> - {stop, normal, ok, State}. - + {stop, normal, ok, State}; +handle_call(doom, _From, State) -> + leveled_log:log("P0030", []), + ManifestFP = State#state.root_path ++ "/" ++ ?MANIFEST_FP ++ "/", + FilesFP = State#state.root_path ++ "/" ++ ?FILES_FP ++ "/", + {stop, normal, {ok, [ManifestFP, FilesFP]}, State}. handle_cast({manifest_change, WI}, State) -> {ok, UpdState} = commit_manifest_change(WI, State), @@ -519,6 +526,7 @@ code_change(_OldVsn, State, _Extra) -> %%% Internal functions %%%============================================================================ + start_from_file(PCLopts) -> RootPath = PCLopts#penciller_options.root_path, MaxTableSize = case PCLopts#penciller_options.max_inmemory_tablesize of diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 0ce533d..5dab283 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -62,8 +62,7 @@ simple_put_fetch_head_delete(_Config) -> ok = leveled_bookie:book_close(Bookie3), {ok, Bookie4} = leveled_bookie:book_start(StartOpts2), not_found = leveled_bookie:book_get(Bookie4, "Bucket1", "Key2"), - ok = leveled_bookie:book_close(Bookie4), - testutil:reset_filestructure(). + ok = leveled_bookie:book_destroy(Bookie4). many_put_fetch_head(_Config) -> RootPath = testutil:reset_filestructure(), @@ -98,8 +97,7 @@ many_put_fetch_head(_Config) -> {ok, Bookie3} = leveled_bookie:book_start(StartOpts2), testutil:check_forlist(Bookie3, ChkList2A), testutil:check_forobject(Bookie3, TestObject), - ok = leveled_bookie:book_close(Bookie3), - testutil:reset_filestructure(). + ok = leveled_bookie:book_destroy(Bookie3). journal_compaction(_Config) -> RootPath = testutil:reset_filestructure(),