From 91f751ddc6ba3d8205f6475ea08ba23c42d55255 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 7 Sep 2018 14:21:01 +0100 Subject: [PATCH] Garbage collect backups If ther are backups made to the same folder, need to remove any files from that folder that are not included in this backup. Some initial testing, needs more. --- src/leveled_bookie.erl | 4 +++- src/leveled_inker.erl | 40 +++++++++++++++++++++++++++++++--------- src/leveled_log.erl | 8 +++++++- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index c25d4f1..c377393 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -1169,7 +1169,9 @@ handle_call(hot_backup, _From, State) when State#state.head_only == false -> end end, InkerOpts = - #inker_options{start_snapshot=true, source_inker=State#state.inker}, + #inker_options{start_snapshot = true, + source_inker = State#state.inker, + bookies_pid = self()}, {ok, Snapshot} = leveled_inker:ink_snapstart(InkerOpts), {reply, {async, BackupFun(Snapshot)}, State}; handle_call(close, _From, State) -> diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 959c80b..0412b52 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -301,7 +301,7 @@ ink_doom(Pid) -> %% %% The InitAccFun should return an initial batch accumulator for each subfold. %% It is a 2-arity function that takes a filename and a MinSQN as an input -%% potentially to be use din logging +%% potentially to be used in logging %% %% The BatchFun is a two arity function that should take as inputs: %% An overall accumulator @@ -563,6 +563,7 @@ handle_call(roll, _From, State) -> {reply, ok, State}; _ -> NewSQN = State#state.journal_sqn + 1, + SWroll = os:timestamp(), {NewJournalP, Manifest1, NewManSQN} = roll_active(State#state.active_journaldb, State#state.manifest, @@ -570,6 +571,7 @@ handle_call(roll, _From, State) -> State#state.cdb_options, State#state.root_path, State#state.manifest_sqn), + leveled_log:log_timer("I0024", [NewSQN], SWroll), {reply, ok, State#state{journal_sqn = NewSQN, manifest = Manifest1, manifest_sqn = NewManSQN, @@ -580,11 +582,14 @@ handle_call({backup, BackupPath}, _from, State) SW = os:timestamp(), BackupJFP = filepath(filename:join(BackupPath, ?JOURNAL_FP), journal_dir), ok = filelib:ensure_dir(BackupJFP), + {ok, CurrentFNs} = file:list_dir(BackupJFP), + leveled_log:log("I0023", [length(CurrentFNs)]), BackupFun = - fun({SQN, FN, PidR, LastKey}, Acc) -> + fun({SQN, FN, PidR, LastKey}, {ManAcc, FTRAcc}) -> case SQN < State#state.journal_sqn of true -> BaseFN = filename:basename(FN), + ExtendedBaseFN = BaseFN ++ "." ++ ?JOURNAL_FILEX, BackupName = filename:join(BackupJFP, BaseFN), true = leveled_cdb:finished_rolling(PidR), case file:make_link(FN ++ "." ++ ?JOURNAL_FILEX, @@ -594,16 +599,32 @@ handle_call({backup, BackupPath}, _from, State) {error, eexist} -> ok end, - [{SQN, BackupName, PidR, LastKey}|Acc]; + {[{SQN, BackupName, PidR, LastKey}|ManAcc], + [ExtendedBaseFN|FTRAcc]}; false -> leveled_log:log("I0021", [FN, SQN, State#state.journal_sqn]), - Acc + {ManAcc, FTRAcc} end end, - BackupManifest = + {BackupManifest, FilesToRetain} = lists:foldr(BackupFun, - [], - leveled_imanifest:to_list(State#state.manifest)), + {[], []}, + leveled_imanifest:to_list(State#state.manifest)), + + FilesToRemove = lists:subtract(CurrentFNs, FilesToRetain), + RemoveFun = + fun(RFN) -> + leveled_log:log("I0022", [RFN]), + RemoveFile = filename:join(BackupJFP, RFN), + case filelib:is_file(RemoveFile) + and not filelib:is_dir(RemoveFile) of + true -> + ok = file:delete(RemoveFile); + false -> + ok + end + end, + lists:foreach(RemoveFun, FilesToRemove), leveled_imanifest:writer(leveled_imanifest:from_list(BackupManifest), State#state.manifest_sqn, filename:join(BackupPath, ?JOURNAL_FP)), @@ -793,6 +814,7 @@ put_object(LedgerKey, Object, KeyChanges, State) -> State#state{journal_sqn=NewSQN}, byte_size(JournalBin)}; roll -> + SWroll = os:timestamp(), {NewJournalP, Manifest1, NewManSQN} = roll_active(ActiveJournal, State#state.manifest, @@ -800,6 +822,7 @@ put_object(LedgerKey, Object, KeyChanges, State) -> State#state.cdb_options, State#state.root_path, State#state.manifest_sqn), + leveled_log:log_timer("I0008", [], SWroll), ok = leveled_cdb:cdb_put(NewJournalP, JournalKey, JournalBin), @@ -837,7 +860,6 @@ get_object(LedgerKey, SQN, Manifest, ToIgnoreKeyChanges) -> %% Roll the active journal, and start a new active journal, updating the %% manifest roll_active(ActiveJournal, Manifest, NewSQN, CDBopts, RootPath, ManifestSQN) -> - SWroll = os:timestamp(), LastKey = leveled_cdb:cdb_lastkey(ActiveJournal), ok = leveled_cdb:cdb_roll(ActiveJournal), Manifest0 = @@ -847,7 +869,7 @@ roll_active(ActiveJournal, Manifest, NewSQN, CDBopts, RootPath, ManifestSQN) -> {_, _, NewJournalP, _} = ManEntry, Manifest1 = leveled_imanifest:add_entry(Manifest0, ManEntry, true), ok = leveled_imanifest:writer(Manifest1, ManifestSQN + 1, RootPath), - leveled_log:log_timer("I0008", [], SWroll), + {NewJournalP, Manifest1, ManifestSQN + 1}. -spec key_check(leveled_codec:ledger_key(), diff --git a/src/leveled_log.erl b/src/leveled_log.erl index 41fcdfb..b00c3a5 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -284,7 +284,13 @@ {info, "Journal backup completed to path=~s with file_count=~w"}}, {"I0021", {info, "Ingoring filename=~s with SQN=~w and JournalSQN=~w"}}, - + {"I0022", + {info, "Removing filename=~s from backup folder as not in backup"}}, + {"I0023", + {info, "Backup commencing into folder with ~w existing files"}}, + {"I0024", + {info, "Prompted roll at NewSQN=~w"}}, + {"IC001", {info, "Closed for reason ~w so maybe leaving garbage"}}, {"IC002",