From 0838ff34e5f3cad8c52ab1f7c2ef4d2a9769c286 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 6 Sep 2018 17:50:30 +0100 Subject: [PATCH 1/8] Initial hot_backup The idea being that a consistent inker manifest and set of journal files is guaranteed - with hard links to the actual manifest files. --- include/leveled.hrl | 3 + src/leveled_bookie.erl | 33 ++++++- src/leveled_cdb.erl | 10 ++ src/leveled_imanifest.erl | 15 ++- src/leveled_inker.erl | 146 +++++++++++++++++++++++------ src/leveled_log.erl | 4 + test/end_to_end/recovery_SUITE.erl | 37 +++++++- 7 files changed, 211 insertions(+), 37 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index 64f0dfe..dd0b90b 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -1,3 +1,6 @@ +% File paths +-define(JOURNAL_FP, "journal"). +-define(LEDGER_FP, "ledger"). %% Tag to be used on standard Riak KV objects -define(RIAK_TAG, o_rkv). diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 901f6b8..c612485 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -63,6 +63,7 @@ book_compactjournal/2, book_islastcompactionpending/1, book_trimjournal/1, + book_hotbackup/1, book_close/1, book_destroy/1, book_isempty/2]). @@ -76,8 +77,6 @@ -include_lib("eunit/include/eunit.hrl"). -define(CACHE_SIZE, 2500). --define(JOURNAL_FP, "journal"). --define(LEDGER_FP, "ledger"). -define(SNAPSHOT_TIMEOUT, 300000). -define(CACHE_SIZE_JITTER, 25). -define(JOURNAL_SIZE_JITTER, 20). @@ -587,6 +586,19 @@ book_destroy(Pid) -> gen_server:call(Pid, destroy, infinity). +-spec book_hotbackup(pid()) -> {async, fun()}. +%% @doc Backup the Bookie +%% Return a function that will take a backup of a snapshot of the Journal. +%% The function will be 1-arity, and can be passed the absolute folder name +%% to store the backup. +%% +%% Backup files are hard-linked. Does not work in head_only mode +%% +%% TODO: Can extend to head_only mode, and also support another parameter +%% which would backup persisted part of ledger (to make restart faster) +book_hotbackup(Pid) -> + gen_server:call(Pid, hot_backup, infinity). + -spec book_isempty(pid(), leveled_codec:tag()) -> boolean(). %% @doc %% Confirm if the store is empty, or if it contains a Key and Value for a @@ -842,6 +854,19 @@ handle_call(confirm_compact, _From, State) handle_call(trim, _From, State) when State#state.head_only == true -> PSQN = leveled_penciller:pcl_persistedsqn(State#state.penciller), {reply, leveled_inker:ink_trim(State#state.inker, PSQN), State}; +handle_call(hot_backup, _From, State) when State#state.head_only == false -> + ok = leveled_inker:ink_roll(State#state.inker), + BackupFun = + fun(InkerSnapshot) -> + fun(BackupPath) -> + ok = leveled_inker:ink_backup(InkerSnapshot, BackupPath), + ok = leveled_inker:ink_close(InkerSnapshot) + end + end, + InkerOpts = + #inker_options{start_snapshot=true, source_inker=State#state.inker}, + {ok, Snapshot} = leveled_inker:ink_snapstart(InkerOpts), + {reply, {async, BackupFun(Snapshot)}, State}; handle_call(close, _From, State) -> leveled_inker:ink_close(State#state.inker), leveled_penciller:pcl_close(State#state.penciller), @@ -1019,8 +1044,8 @@ set_options(Opts) -> PCLL0CacheSize = proplists:get_value(max_pencillercachesize, Opts), RootPath = proplists:get_value(root_path, Opts), - JournalFP = RootPath ++ "/" ++ ?JOURNAL_FP, - LedgerFP = RootPath ++ "/" ++ ?LEDGER_FP, + JournalFP = filename:join(RootPath, ?JOURNAL_FP), + LedgerFP = filename:join(RootPath, ?LEDGER_FP), ok = filelib:ensure_dir(JournalFP), ok = filelib:ensure_dir(LedgerFP), diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 86f85a6..acbac25 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -100,6 +100,7 @@ cdb_destroy/1, cdb_deletepending/1, cdb_deletepending/3, + cdb_isrolling/1, hashtable_calc/2]). -include_lib("eunit/include/eunit.hrl"). @@ -380,6 +381,13 @@ cdb_filename(Pid) -> cdb_keycheck(Pid, Key) -> gen_fsm:sync_send_event(Pid, {key_check, Key}, infinity). +-spec cdb_isrolling(pid()) -> boolean(). +%% @doc +%% Check to see if a cdb file is still rolling +cdb_isrolling(Pid) -> + gen_fsm:sync_send_all_state_event(Pid, cdb_isrolling, infinity). + + %%%============================================================================ %%% gen_server callbacks %%%============================================================================ @@ -744,6 +752,8 @@ handle_sync_event(cdb_firstkey, _From, StateName, State) -> {reply, FirstKey, StateName, State}; handle_sync_event(cdb_filename, _From, StateName, State) -> {reply, State#state.filename, StateName, State}; +handle_sync_event(cdb_isrolling, _From, StateName, State) -> + {reply, StateName == rolling, StateName, State}; handle_sync_event(cdb_close, _From, delete_pending, State) -> leveled_log:log("CDB05", [State#state.filename, delete_pending, cdb_close]), diff --git a/src/leveled_imanifest.erl b/src/leveled_imanifest.erl index e619ab0..2be891f 100644 --- a/src/leveled_imanifest.erl +++ b/src/leveled_imanifest.erl @@ -161,6 +161,8 @@ reader(SQN, RootPath) -> %% disk writer(Manifest, ManSQN, RootPath) -> ManPath = leveled_inker:filepath(RootPath, manifest_dir), + ok = filelib:ensure_dir(ManPath), + % When writing during backups, may not have been generated NewFN = filename:join(ManPath, integer_to_list(ManSQN) ++ "." ++ ?MANIFEST_FILEX), TmpFN = filename:join(ManPath, @@ -198,10 +200,10 @@ complete_filex() -> ?MANIFEST_FILEX. -%%%============================================================================ -%%% Internal Functions -%%%============================================================================ - +-spec from_list(list()) -> manifest(). +%% @doc +%% Convert from a flat list into a manifest with lookup jumps. +%% The opposite of to_list/1 from_list(Manifest) -> % Manifest should already be sorted with the highest SQN at the head % This will be maintained so that we can fold from the left, and find @@ -209,6 +211,11 @@ from_list(Manifest) -> % reads are more common than stale reads lists:foldr(fun prepend_entry/2, [], Manifest). + +%%%============================================================================ +%%% Internal Functions +%%%============================================================================ + prepend_entry(Entry, AccL) -> {SQN, _FN, _PidR, _LastKey} = Entry, case AccL of diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index a03494d..27794d5 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -113,6 +113,8 @@ ink_printmanifest/1, ink_close/1, ink_doom/1, + ink_roll/1, + ink_backup/2, build_dummy_journal/0, clean_testdir/1, filepath/2, @@ -235,10 +237,10 @@ ink_fetch(Pid, PrimaryKey, SQN) -> ink_keycheck(Pid, PrimaryKey, SQN) -> gen_server:call(Pid, {key_check, PrimaryKey, SQN}, infinity). --spec ink_registersnapshot(pid(), pid()) -> {list(), pid()}. +-spec ink_registersnapshot(pid(), pid()) -> {list(), pid(), integer()}. %% @doc %% Register a snapshot clone for the process, returning the Manifest and the -%% pid of the active journal. +%% pid of the active journal, as well as the JournalSQN. ink_registersnapshot(Pid, Requestor) -> gen_server:call(Pid, {register_snapshot, Requestor}, infinity). @@ -389,6 +391,18 @@ ink_compactionpending(Pid) -> ink_trim(Pid, PersistedSQN) -> gen_server:call(Pid, {trim, PersistedSQN}, infinity). +-spec ink_roll(pid()) -> ok. +%% @doc +%% Roll the active journal +ink_roll(Pid) -> + gen_server:call(Pid, roll, infinity). + +-spec ink_backup(pid(), string()) -> ok. +%% @doc +%% Backup the journal to the specified path +ink_backup(Pid, BackupPath) -> + gen_server:call(Pid, {backup, BackupPath}). + -spec ink_getmanifest(pid()) -> list(). %% @doc %% Allows the clerk to fetch the manifest at the point it starts a compaction @@ -428,11 +442,13 @@ init([InkerOpts]) -> {undefined, true} -> SrcInker = InkerOpts#inker_options.source_inker, {Manifest, - ActiveJournalDB} = ink_registersnapshot(SrcInker, self()), - {ok, #state{manifest=Manifest, - active_journaldb=ActiveJournalDB, - source_inker=SrcInker, - is_snapshot=true}}; + ActiveJournalDB, + JournalSQN} = ink_registersnapshot(SrcInker, self()), + {ok, #state{manifest = Manifest, + active_journaldb = ActiveJournalDB, + source_inker = SrcInker, + journal_sqn = JournalSQN, + is_snapshot = true}}; %% Need to do something about timeout {_RootPath, false} -> start_from_file(InkerOpts) @@ -477,7 +493,8 @@ handle_call({register_snapshot, Requestor}, _From , State) -> State#state.manifest_sqn}|State#state.registered_snapshots], leveled_log:log("I0002", [Requestor, State#state.manifest_sqn]), {reply, {State#state.manifest, - State#state.active_journaldb}, + State#state.active_journaldb, + State#state.journal_sqn}, State#state{registered_snapshots=Rs}}; handle_call({confirm_delete, ManSQN}, _From, State) -> CheckSQNFun = @@ -535,6 +552,56 @@ handle_call(compaction_pending, _From, State) -> handle_call({trim, PersistedSQN}, _From, State) -> ok = leveled_iclerk:clerk_trim(State#state.clerk, self(), PersistedSQN), {reply, ok, State}; +handle_call(roll, _From, State) -> + NewSQN = State#state.journal_sqn + 1, + {NewJournalP, Manifest1, NewManSQN} = + roll_active(State#state.active_journaldb, + State#state.manifest, + NewSQN, + State#state.cdb_options, + State#state.root_path, + State#state.manifest_sqn), + {reply, ok, State#state{journal_sqn = NewSQN, + manifest = Manifest1, + manifest_sqn = NewManSQN, + active_journaldb = NewJournalP}}; +handle_call({backup, BackupPath}, _from, State) + when State#state.is_snapshot == true -> + SW = os:timestamp(), + BackupJFP = filepath(filename:join(BackupPath, ?JOURNAL_FP), journal_dir), + ok = filelib:ensure_dir(BackupJFP), + BackupFun = + fun({SQN, FN, PidR, LastKey}, Acc) -> + case SQN < State#state.journal_sqn of + true -> + BaseFN = filename:basename(FN), + BackupName = filename:join(BackupJFP, BaseFN), + false = when_not_rolling(PidR), + case file:make_link(FN ++ "." ++ ?JOURNAL_FILEX, + BackupName ++ "." ++ ?JOURNAL_FILEX) of + ok -> + ok; + {error, eexist} -> + ok + end, + [{SQN, BackupName, PidR, LastKey}|Acc]; + false -> + leveled_log:log("I0021", [FN, SQN, State#state.journal_sqn]), + Acc + end + end, + BackupManifest = + lists:foldr(BackupFun, + [], + leveled_imanifest:to_list(State#state.manifest)), + leveled_imanifest:writer(leveled_imanifest:from_list(BackupManifest), + State#state.manifest_sqn, + filename:join(BackupPath, ?JOURNAL_FP)), + leveled_log:log_timer("I0020", + [filename:join(BackupPath, ?JOURNAL_FP), + length(BackupManifest)], + SW), + {reply, ok, State}; handle_call(close, _From, State) -> case State#state.is_snapshot of true -> @@ -596,9 +663,9 @@ start_from_file(InkOpts) -> % Determine filepaths RootPath = InkOpts#inker_options.root_path, JournalFP = filepath(RootPath, journal_dir), - filelib:ensure_dir(JournalFP), + ok = filelib:ensure_dir(JournalFP), CompactFP = filepath(RootPath, journal_compact_dir), - filelib:ensure_dir(CompactFP), + ok = filelib:ensure_dir(CompactFP), ManifestFP = filepath(RootPath, manifest_dir), ok = filelib:ensure_dir(ManifestFP), % The IClerk must start files with the compaction file path so that they @@ -645,6 +712,19 @@ start_from_file(InkOpts) -> clerk = Clerk}}. +when_not_rolling(CDB) -> + RollerFun = + fun(Sleep, WasRolling) -> + case WasRolling of + false -> + false; + true -> + timer:sleep(Sleep), + leveled_cdb:cdb_isrolling(CDB) + end + end, + lists:foldl(RollerFun, true, [0, 1000, 10000, 100000]). + -spec shutdown_snapshots(list(tuple())) -> ok. %% @doc %% Shutdown any snapshots before closing the store @@ -710,29 +790,20 @@ put_object(LedgerKey, Object, KeyChanges, State) -> State#state{journal_sqn=NewSQN}, byte_size(JournalBin)}; roll -> - SWroll = os:timestamp(), - LastKey = leveled_cdb:cdb_lastkey(ActiveJournal), - ok = leveled_cdb:cdb_roll(ActiveJournal), - Manifest0 = leveled_imanifest:append_lastkey(State#state.manifest, - ActiveJournal, - LastKey), - CDBopts = State#state.cdb_options, - ManEntry = start_new_activejournal(NewSQN, - State#state.root_path, - CDBopts), - {_, _, NewJournalP, _} = ManEntry, - Manifest1 = leveled_imanifest:add_entry(Manifest0, ManEntry, true), - ok = leveled_imanifest:writer(Manifest1, - State#state.manifest_sqn + 1, - State#state.root_path), + {NewJournalP, Manifest1, NewManSQN} = + roll_active(ActiveJournal, + State#state.manifest, + NewSQN, + State#state.cdb_options, + State#state.root_path, + State#state.manifest_sqn), ok = leveled_cdb:cdb_put(NewJournalP, JournalKey, JournalBin), - leveled_log:log_timer("I0008", [], SWroll), {rolling, State#state{journal_sqn=NewSQN, manifest=Manifest1, - manifest_sqn = State#state.manifest_sqn + 1, + manifest_sqn = NewManSQN, active_journaldb=NewJournalP}, byte_size(JournalBin)} end. @@ -756,6 +827,26 @@ get_object(LedgerKey, SQN, Manifest, ToIgnoreKeyChanges) -> leveled_codec:from_inkerkv(Obj, ToIgnoreKeyChanges). +-spec roll_active(pid(), leveled_imanifest:manifest(), + integer(), #cdb_options{}, string(), integer()) -> + {pid(), leveled_imanifest:manifest(), integer()}. +%% @doc +%% Roll the active journal, and start a new active journal, updating the +%% manifest +roll_active(ActiveJournal, Manifest, NewSQN, CDBopts, RootPath, ManifestSQN) -> + SWroll = os:timestamp(), + LastKey = leveled_cdb:cdb_lastkey(ActiveJournal), + ok = leveled_cdb:cdb_roll(ActiveJournal), + Manifest0 = + leveled_imanifest:append_lastkey(Manifest, ActiveJournal, LastKey), + ManEntry = + start_new_activejournal(NewSQN, RootPath, CDBopts), + {_, _, NewJournalP, _} = ManEntry, + Manifest1 = leveled_imanifest:add_entry(Manifest0, ManEntry, true), + ok = leveled_imanifest:writer(Manifest1, ManifestSQN + 1, RootPath), + leveled_log:log_timer("I0008", [], SWroll), + {NewJournalP, Manifest1, ManifestSQN + 1}. + -spec key_check(leveled_codec:ledger_key(), integer(), leveled_imanifest:manifest()) -> missing|probably. @@ -1014,7 +1105,6 @@ sequencenumbers_fromfilenames(Filenames, Regex, IntName) -> [], Filenames). - filepath(RootPath, journal_dir) -> RootPath ++ "/" ++ ?FILES_FP ++ "/"; filepath(RootPath, manifest_dir) -> diff --git a/src/leveled_log.erl b/src/leveled_log.erl index b52f1cc..bd690f7 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -280,6 +280,10 @@ {"I0019", {info, "After ~w PUTs total prepare time is ~w total cdb time is ~w " ++ "and max prepare time is ~w and max cdb time is ~w"}}, + {"I0020", + {info, "Journal backup completed to path=~s with file_count=~w"}}, + {"I0021", + {info, "Ingoring filename=~s with SQN=~w and JournalSQN=~w"}}, {"IC001", {info, "Closed for reason ~w so maybe leaving garbage"}}, diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index 8c63b04..a8b063a 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -2,7 +2,8 @@ -include_lib("common_test/include/ct.hrl"). -include("include/leveled.hrl"). -export([all/0]). --export([retain_strategy/1, +-export([hot_backup_simple/1, + retain_strategy/1, recovr_strategy/1, aae_missingjournal/1, aae_bustedjournal/1, @@ -10,6 +11,7 @@ ]). all() -> [ + hot_backup_simple, retain_strategy, recovr_strategy, aae_missingjournal, @@ -17,6 +19,39 @@ all() -> [ journal_compaction_bustedjournal ]. + +hot_backup_simple(_Config) -> + % The journal may have a hot backup. This allows for an online Bookie + % to be sent a message to prepare a backup function, which an asynchronous + % worker can then call to generate a backup taken at the point in time + % the original message was processsed. + % + % The basic test is to: + % 1 - load a Bookie, take a backup, delete the original path, restore from + % that path + RootPath = testutil:reset_filestructure(), + BookOpts = [{root_path, RootPath}, + {cache_size, 1000}, + {max_journalsize, 10000000}, + {sync_strategy, testutil:sync_strategy()}], + {ok, Spcl1, LastV1} = rotating_object_check(BookOpts, "Bucket1", 10000), + {ok, Book1} = leveled_bookie:book_start(BookOpts), + {async, BackupFun} = leveled_bookie:book_hotbackup(Book1), + BackupPath = testutil:reset_filestructure("backup0"), + ok = BackupFun(BackupPath), + ok = leveled_bookie:book_close(Book1), + RootPath = testutil:reset_filestructure(), + BookOptsBackup = [{root_path, BackupPath}, + {cache_size, 2000}, + {max_journalsize, 20000000}, + {sync_strategy, testutil:sync_strategy()}], + {ok, BookBackup} = leveled_bookie:book_start(BookOptsBackup), + ok = testutil:check_indexed_objects(BookBackup, "Bucket1", Spcl1, LastV1), + ok = leveled_bookie:book_close(BookBackup), + BackupPath = testutil:reset_filestructure("backup0"). + + + retain_strategy(_Config) -> RootPath = testutil:reset_filestructure(), BookOpts = [{root_path, RootPath}, From daf0a1a607678fc7c8d6c59e188834f3a40fa1e7 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 6 Sep 2018 17:50:59 +0100 Subject: [PATCH 2/8] Accumulate generated objects efficienty --- test/end_to_end/testutil.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index 03f00e8..5059083 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -374,7 +374,7 @@ generate_objects(Count, KeyNumber, ObjL, Value, IndexGen) -> generate_objects(Count, KeyNumber, ObjL, Value, IndexGen, "Bucket"). generate_objects(0, _KeyNumber, ObjL, _Value, _IndexGen, _Bucket) -> - ObjL; + lists:reverse(ObjL); generate_objects(Count, binary_uuid, ObjL, Value, IndexGen, Bucket) -> {Obj1, Spec1} = set_object(list_to_binary(Bucket), list_to_binary(leveled_util:generate_uuid()), @@ -382,7 +382,7 @@ generate_objects(Count, binary_uuid, ObjL, Value, IndexGen, Bucket) -> IndexGen), generate_objects(Count - 1, binary_uuid, - ObjL ++ [{leveled_rand:uniform(), Obj1, Spec1}], + [{leveled_rand:uniform(), Obj1, Spec1}|ObjL], Value, IndexGen, Bucket); @@ -393,7 +393,7 @@ generate_objects(Count, uuid, ObjL, Value, IndexGen, Bucket) -> IndexGen), generate_objects(Count - 1, uuid, - ObjL ++ [{leveled_rand:uniform(), Obj1, Spec1}], + [{leveled_rand:uniform(), Obj1, Spec1}|ObjL], Value, IndexGen, Bucket); @@ -405,7 +405,7 @@ generate_objects(Count, {binary, KeyNumber}, ObjL, Value, IndexGen, Bucket) -> IndexGen), generate_objects(Count - 1, {binary, KeyNumber + 1}, - ObjL ++ [{leveled_rand:uniform(), Obj1, Spec1}], + [{leveled_rand:uniform(), Obj1, Spec1}|ObjL], Value, IndexGen, Bucket); @@ -417,7 +417,7 @@ generate_objects(Count, {fixed_binary, KeyNumber}, ObjL, Value, IndexGen, Bucket IndexGen), generate_objects(Count - 1, {fixed_binary, KeyNumber + 1}, - ObjL ++ [{leveled_rand:uniform(), Obj1, Spec1}], + [{leveled_rand:uniform(), Obj1, Spec1}|ObjL], Value, IndexGen, Bucket); @@ -428,7 +428,7 @@ generate_objects(Count, KeyNumber, ObjL, Value, IndexGen, Bucket) -> IndexGen), generate_objects(Count - 1, KeyNumber + 1, - ObjL ++ [{leveled_rand:uniform(), Obj1, Spec1}], + [{leveled_rand:uniform(), Obj1, Spec1}|ObjL], Value, IndexGen, Bucket). From 9a8ce88ed2d09f58944cd0f7124fab8cec26a997 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 7 Sep 2018 10:24:51 +0100 Subject: [PATCH 3/8] Add double-backup test Added a test with back-to-back backups. This caused issues with the empty CDB file it created (on opening, couldn't cope with last key of empty). So now backup won't roll the active journal if it is empty. --- src/leveled_cdb.erl | 36 +++++++++++++++++++++- src/leveled_imanifest.erl | 9 ++---- src/leveled_inker.erl | 48 ++++++++++++------------------ src/leveled_log.erl | 2 +- test/end_to_end/recovery_SUITE.erl | 28 ++++++++++++++++- 5 files changed, 85 insertions(+), 38 deletions(-) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index acbac25..d10b0a7 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -100,7 +100,9 @@ cdb_destroy/1, cdb_deletepending/1, cdb_deletepending/3, - cdb_isrolling/1, + cdb_isrolling/1]). + +-export([finished_rolling/1, hashtable_calc/2]). -include_lib("eunit/include/eunit.hrl"). @@ -780,6 +782,25 @@ terminate(_Reason, _StateName, _State) -> code_change(_OldVsn, StateName, State, _Extra) -> {ok, StateName, State}. + +%%%============================================================================ +%%% External functions +%%%============================================================================ + + +finished_rolling(CDB) -> + RollerFun = + fun(Sleep, FinishedRolling) -> + case FinishedRolling of + true -> + true; + false -> + timer:sleep(Sleep), + not leveled_cdb:cdb_isrolling(CDB) + end + end, + lists:foldl(RollerFun, false, [0, 1000, 10000, 100000]). + %%%============================================================================ %%% Internal functions %%%============================================================================ @@ -2127,6 +2148,19 @@ emptyvalue_fromdict_test() -> ?assertMatch(KVP, D_Result), ok = file:delete("../test/from_dict_test_ev.cdb"). + +empty_roll_test() -> + file:delete("../test/empty_roll.cdb"), + file:delete("../test/empty_roll.pnd"), + {ok, P1} = cdb_open_writer("../test/empty_roll.pnd", + #cdb_options{binary_mode=true}), + ok = cdb_roll(P1), + true = finished_rolling(P1), + {ok, P2} = cdb_open_reader("../test/empty_roll.cdb", + #cdb_options{binary_mode=true}), + ok = cdb_close(P2), + ok = file:delete("../test/empty_roll.cdb"). + find_lastkey_test() -> file:delete("../test/lastkey.pnd"), {ok, P1} = cdb_open_writer("../test/lastkey.pnd", diff --git a/src/leveled_imanifest.erl b/src/leveled_imanifest.erl index 2be891f..8dca448 100644 --- a/src/leveled_imanifest.erl +++ b/src/leveled_imanifest.erl @@ -168,12 +168,9 @@ writer(Manifest, ManSQN, RootPath) -> TmpFN = filename:join(ManPath, integer_to_list(ManSQN) ++ "." ++ ?PENDING_FILEX), MBin = term_to_binary(to_list(Manifest), [compressed]), - case filelib:is_file(NewFN) of - false -> - leveled_log:log("I0016", [ManSQN]), - ok = file:write_file(TmpFN, MBin), - ok = file:rename(TmpFN, NewFN) - end, + leveled_log:log("I0016", [ManSQN]), + ok = file:write_file(TmpFN, MBin), + ok = file:rename(TmpFN, NewFN), GC_SQN = ManSQN - ?MANIFESTS_TO_RETAIN, GC_Man = filename:join(ManPath, integer_to_list(GC_SQN) ++ "." ++ ?MANIFEST_FILEX), diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 27794d5..04418a6 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -553,18 +553,23 @@ handle_call({trim, PersistedSQN}, _From, State) -> ok = leveled_iclerk:clerk_trim(State#state.clerk, self(), PersistedSQN), {reply, ok, State}; handle_call(roll, _From, State) -> - NewSQN = State#state.journal_sqn + 1, - {NewJournalP, Manifest1, NewManSQN} = - roll_active(State#state.active_journaldb, - State#state.manifest, - NewSQN, - State#state.cdb_options, - State#state.root_path, - State#state.manifest_sqn), - {reply, ok, State#state{journal_sqn = NewSQN, - manifest = Manifest1, - manifest_sqn = NewManSQN, - active_journaldb = NewJournalP}}; + case leveled_cdb:cdb_lastkey(State#state.active_journaldb) of + empty -> + {reply, ok, State}; + _ -> + NewSQN = State#state.journal_sqn + 1, + {NewJournalP, Manifest1, NewManSQN} = + roll_active(State#state.active_journaldb, + State#state.manifest, + NewSQN, + State#state.cdb_options, + State#state.root_path, + State#state.manifest_sqn), + {reply, ok, State#state{journal_sqn = NewSQN, + manifest = Manifest1, + manifest_sqn = NewManSQN, + active_journaldb = NewJournalP}} + end; handle_call({backup, BackupPath}, _from, State) when State#state.is_snapshot == true -> SW = os:timestamp(), @@ -576,7 +581,7 @@ handle_call({backup, BackupPath}, _from, State) true -> BaseFN = filename:basename(FN), BackupName = filename:join(BackupJFP, BaseFN), - false = when_not_rolling(PidR), + true = leveled_cdb:finished_rolling(PidR), case file:make_link(FN ++ "." ++ ?JOURNAL_FILEX, BackupName ++ "." ++ ?JOURNAL_FILEX) of ok -> @@ -712,19 +717,6 @@ start_from_file(InkOpts) -> clerk = Clerk}}. -when_not_rolling(CDB) -> - RollerFun = - fun(Sleep, WasRolling) -> - case WasRolling of - false -> - false; - true -> - timer:sleep(Sleep), - leveled_cdb:cdb_isrolling(CDB) - end - end, - lists:foldl(RollerFun, true, [0, 1000, 10000, 100000]). - -spec shutdown_snapshots(list(tuple())) -> ok. %% @doc %% Shutdown any snapshots before closing the store @@ -983,9 +975,7 @@ open_all_manifest(Man0, RootPath, CDBOpts) -> NewManEntry = start_new_activejournal(LastSQN + 1, RootPath, CDBOpts), - leveled_imanifest:add_entry(ManToHead, - NewManEntry, - true); + leveled_imanifest:add_entry(ManToHead, NewManEntry, true); false -> {ok, HeadW} = leveled_cdb:cdb_open_writer(PendingHeadFN, CDBOpts), diff --git a/src/leveled_log.erl b/src/leveled_log.erl index bd690f7..41fcdfb 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -284,7 +284,7 @@ {info, "Journal backup completed to path=~s with file_count=~w"}}, {"I0021", {info, "Ingoring filename=~s with SQN=~w and JournalSQN=~w"}}, - + {"IC001", {info, "Closed for reason ~w so maybe leaving garbage"}}, {"IC002", diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index a8b063a..19109aa 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -3,6 +3,7 @@ -include("include/leveled.hrl"). -export([all/0]). -export([hot_backup_simple/1, + hot_backup_double/1, retain_strategy/1, recovr_strategy/1, aae_missingjournal/1, @@ -12,6 +13,7 @@ all() -> [ hot_backup_simple, + hot_backup_double, retain_strategy, recovr_strategy, aae_missingjournal, @@ -50,7 +52,31 @@ hot_backup_simple(_Config) -> ok = leveled_bookie:book_close(BookBackup), BackupPath = testutil:reset_filestructure("backup0"). - +hot_backup_double(_Config) -> + % As with simple test, but check that calling for backup twice doesn't have + % any side effects + RootPath = testutil:reset_filestructure(), + BookOpts = [{root_path, RootPath}, + {cache_size, 1000}, + {max_journalsize, 10000000}, + {sync_strategy, testutil:sync_strategy()}], + {ok, Spcl1, LastV1} = rotating_object_check(BookOpts, "Bucket1", 4000), + {ok, Book1} = leveled_bookie:book_start(BookOpts), + {async, BackupFun1} = leveled_bookie:book_hotbackup(Book1), + BackupPath = testutil:reset_filestructure("backup0"), + ok = BackupFun1(BackupPath), + {async, BackupFun2} = leveled_bookie:book_hotbackup(Book1), + ok = BackupFun2(BackupPath), + ok = leveled_bookie:book_close(Book1), + RootPath = testutil:reset_filestructure(), + BookOptsBackup = [{root_path, BackupPath}, + {cache_size, 2000}, + {max_journalsize, 20000000}, + {sync_strategy, testutil:sync_strategy()}], + {ok, BookBackup} = leveled_bookie:book_start(BookOptsBackup), + ok = testutil:check_indexed_objects(BookBackup, "Bucket1", Spcl1, LastV1), + ok = leveled_bookie:book_close(BookBackup), + BackupPath = testutil:reset_filestructure("backup0"). retain_strategy(_Config) -> RootPath = testutil:reset_filestructure(), From 91f751ddc6ba3d8205f6475ea08ba23c42d55255 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 7 Sep 2018 14:21:01 +0100 Subject: [PATCH 4/8] Garbage collect backups If ther are backups made to the same folder, need to remove any files from that folder that are not included in this backup. Some initial testing, needs more. --- src/leveled_bookie.erl | 4 +++- src/leveled_inker.erl | 40 +++++++++++++++++++++++++++++++--------- src/leveled_log.erl | 8 +++++++- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index c25d4f1..c377393 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -1169,7 +1169,9 @@ handle_call(hot_backup, _From, State) when State#state.head_only == false -> end end, InkerOpts = - #inker_options{start_snapshot=true, source_inker=State#state.inker}, + #inker_options{start_snapshot = true, + source_inker = State#state.inker, + bookies_pid = self()}, {ok, Snapshot} = leveled_inker:ink_snapstart(InkerOpts), {reply, {async, BackupFun(Snapshot)}, State}; handle_call(close, _From, State) -> diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 959c80b..0412b52 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -301,7 +301,7 @@ ink_doom(Pid) -> %% %% The InitAccFun should return an initial batch accumulator for each subfold. %% It is a 2-arity function that takes a filename and a MinSQN as an input -%% potentially to be use din logging +%% potentially to be used in logging %% %% The BatchFun is a two arity function that should take as inputs: %% An overall accumulator @@ -563,6 +563,7 @@ handle_call(roll, _From, State) -> {reply, ok, State}; _ -> NewSQN = State#state.journal_sqn + 1, + SWroll = os:timestamp(), {NewJournalP, Manifest1, NewManSQN} = roll_active(State#state.active_journaldb, State#state.manifest, @@ -570,6 +571,7 @@ handle_call(roll, _From, State) -> State#state.cdb_options, State#state.root_path, State#state.manifest_sqn), + leveled_log:log_timer("I0024", [NewSQN], SWroll), {reply, ok, State#state{journal_sqn = NewSQN, manifest = Manifest1, manifest_sqn = NewManSQN, @@ -580,11 +582,14 @@ handle_call({backup, BackupPath}, _from, State) SW = os:timestamp(), BackupJFP = filepath(filename:join(BackupPath, ?JOURNAL_FP), journal_dir), ok = filelib:ensure_dir(BackupJFP), + {ok, CurrentFNs} = file:list_dir(BackupJFP), + leveled_log:log("I0023", [length(CurrentFNs)]), BackupFun = - fun({SQN, FN, PidR, LastKey}, Acc) -> + fun({SQN, FN, PidR, LastKey}, {ManAcc, FTRAcc}) -> case SQN < State#state.journal_sqn of true -> BaseFN = filename:basename(FN), + ExtendedBaseFN = BaseFN ++ "." ++ ?JOURNAL_FILEX, BackupName = filename:join(BackupJFP, BaseFN), true = leveled_cdb:finished_rolling(PidR), case file:make_link(FN ++ "." ++ ?JOURNAL_FILEX, @@ -594,16 +599,32 @@ handle_call({backup, BackupPath}, _from, State) {error, eexist} -> ok end, - [{SQN, BackupName, PidR, LastKey}|Acc]; + {[{SQN, BackupName, PidR, LastKey}|ManAcc], + [ExtendedBaseFN|FTRAcc]}; false -> leveled_log:log("I0021", [FN, SQN, State#state.journal_sqn]), - Acc + {ManAcc, FTRAcc} end end, - BackupManifest = + {BackupManifest, FilesToRetain} = lists:foldr(BackupFun, - [], - leveled_imanifest:to_list(State#state.manifest)), + {[], []}, + leveled_imanifest:to_list(State#state.manifest)), + + FilesToRemove = lists:subtract(CurrentFNs, FilesToRetain), + RemoveFun = + fun(RFN) -> + leveled_log:log("I0022", [RFN]), + RemoveFile = filename:join(BackupJFP, RFN), + case filelib:is_file(RemoveFile) + and not filelib:is_dir(RemoveFile) of + true -> + ok = file:delete(RemoveFile); + false -> + ok + end + end, + lists:foreach(RemoveFun, FilesToRemove), leveled_imanifest:writer(leveled_imanifest:from_list(BackupManifest), State#state.manifest_sqn, filename:join(BackupPath, ?JOURNAL_FP)), @@ -793,6 +814,7 @@ put_object(LedgerKey, Object, KeyChanges, State) -> State#state{journal_sqn=NewSQN}, byte_size(JournalBin)}; roll -> + SWroll = os:timestamp(), {NewJournalP, Manifest1, NewManSQN} = roll_active(ActiveJournal, State#state.manifest, @@ -800,6 +822,7 @@ put_object(LedgerKey, Object, KeyChanges, State) -> State#state.cdb_options, State#state.root_path, State#state.manifest_sqn), + leveled_log:log_timer("I0008", [], SWroll), ok = leveled_cdb:cdb_put(NewJournalP, JournalKey, JournalBin), @@ -837,7 +860,6 @@ get_object(LedgerKey, SQN, Manifest, ToIgnoreKeyChanges) -> %% Roll the active journal, and start a new active journal, updating the %% manifest roll_active(ActiveJournal, Manifest, NewSQN, CDBopts, RootPath, ManifestSQN) -> - SWroll = os:timestamp(), LastKey = leveled_cdb:cdb_lastkey(ActiveJournal), ok = leveled_cdb:cdb_roll(ActiveJournal), Manifest0 = @@ -847,7 +869,7 @@ roll_active(ActiveJournal, Manifest, NewSQN, CDBopts, RootPath, ManifestSQN) -> {_, _, NewJournalP, _} = ManEntry, Manifest1 = leveled_imanifest:add_entry(Manifest0, ManEntry, true), ok = leveled_imanifest:writer(Manifest1, ManifestSQN + 1, RootPath), - leveled_log:log_timer("I0008", [], SWroll), + {NewJournalP, Manifest1, ManifestSQN + 1}. -spec key_check(leveled_codec:ledger_key(), diff --git a/src/leveled_log.erl b/src/leveled_log.erl index 41fcdfb..b00c3a5 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -284,7 +284,13 @@ {info, "Journal backup completed to path=~s with file_count=~w"}}, {"I0021", {info, "Ingoring filename=~s with SQN=~w and JournalSQN=~w"}}, - + {"I0022", + {info, "Removing filename=~s from backup folder as not in backup"}}, + {"I0023", + {info, "Backup commencing into folder with ~w existing files"}}, + {"I0024", + {info, "Prompted roll at NewSQN=~w"}}, + {"IC001", {info, "Closed for reason ~w so maybe leaving garbage"}}, {"IC002", From bbb667b09fab0e8f2e7b646de3601b70ecfc5b80 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 7 Sep 2018 17:24:27 +0100 Subject: [PATCH 5/8] Add test of repeated backups Confirm that file counts in the backup folder rise and fall as expected --- test/end_to_end/recovery_SUITE.erl | 91 ++++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 18 deletions(-) diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index 50ab518..cf6e9ef 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -3,6 +3,7 @@ -include("include/leveled.hrl"). -export([all/0]). -export([hot_backup_simple/1, + hot_backup_changes/1, hot_backup_double/1, retain_strategy/1, recovr_strategy/1, @@ -13,6 +14,7 @@ all() -> [ hot_backup_simple, + hot_backup_changes, hot_backup_double, retain_strategy, recovr_strategy, @@ -78,6 +80,57 @@ hot_backup_double(_Config) -> ok = leveled_bookie:book_close(BookBackup), BackupPath = testutil:reset_filestructure("backup0"). +hot_backup_changes(_Config) -> + RootPath = testutil:reset_filestructure(), + BackupPath = testutil:reset_filestructure("backup0"), + BookOpts = [{root_path, RootPath}, + {cache_size, 1000}, + {max_journalsize, 10000000}, + {sync_strategy, testutil:sync_strategy()}], + B = "Bucket0", + + {ok, Book1} = leveled_bookie:book_start(BookOpts), + {KSpcL1, _V1} = testutil:put_indexed_objects(Book1, B, 20000), + + {async, BackupFun1} = leveled_bookie:book_hotbackup(Book1), + ok = BackupFun1(BackupPath), + {ok, FileList1} = + file:list_dir(filename:join(BackupPath, "journal/journal_files/")), + + {KSpcL2, V2} = testutil:put_altered_indexed_objects(Book1, B, KSpcL1), + + {async, BackupFun2} = leveled_bookie:book_hotbackup(Book1), + ok = BackupFun2(BackupPath), + {ok, FileList2} = + file:list_dir(filename:join(BackupPath, "journal/journal_files/")), + + ok = testutil:check_indexed_objects(Book1, B, KSpcL2, V2), + compact_and_wait(Book1), + + {async, BackupFun3} = leveled_bookie:book_hotbackup(Book1), + ok = BackupFun3(BackupPath), + {ok, FileList3} = + file:list_dir(filename:join(BackupPath, "journal/journal_files/")), + + true = length(FileList2) > length(FileList1), + true = length(FileList2) > length(FileList3), + + ok = leveled_bookie:book_close(Book1), + + RootPath = testutil:reset_filestructure(), + BookOptsBackup = [{root_path, BackupPath}, + {cache_size, 2000}, + {max_journalsize, 20000000}, + {sync_strategy, testutil:sync_strategy()}], + {ok, BookBackup} = leveled_bookie:book_start(BookOptsBackup), + + ok = testutil:check_indexed_objects(BookBackup, B, KSpcL2, V2), + + testutil:reset_filestructure("backup0"), + testutil:reset_filestructure(). + + + retain_strategy(_Config) -> RootPath = testutil:reset_filestructure(), BookOpts = [{root_path, RootPath}, @@ -480,22 +533,7 @@ rotating_object_check(BookOpts, B, NumberOfObjects) -> KSpcL1 ++ KSpcL2 ++ KSpcL3 ++ KSpcL4, V4), - ok = leveled_bookie:book_compactjournal(Book2, 30000), - F = fun leveled_bookie:book_islastcompactionpending/1, - lists:foldl(fun(X, Pending) -> - case Pending of - false -> - false; - true -> - io:format("Loop ~w waiting for journal " - ++ "compaction to complete~n", [X]), - timer:sleep(20000), - F(Book2) - end end, - true, - lists:seq(1, 15)), - io:format("Waiting for journal deletes~n"), - timer:sleep(20000), + compact_and_wait(Book2), io:format("Checking index following compaction~n"), ok = testutil:check_indexed_objects(Book2, @@ -505,8 +543,25 @@ rotating_object_check(BookOpts, B, NumberOfObjects) -> ok = leveled_bookie:book_close(Book2), {ok, KSpcL1 ++ KSpcL2 ++ KSpcL3 ++ KSpcL4, V4}. - - + +compact_and_wait(Book) -> + ok = leveled_bookie:book_compactjournal(Book, 30000), + F = fun leveled_bookie:book_islastcompactionpending/1, + lists:foldl(fun(X, Pending) -> + case Pending of + false -> + false; + true -> + io:format("Loop ~w waiting for journal " + ++ "compaction to complete~n", [X]), + timer:sleep(20000), + F(Book) + end end, + true, + lists:seq(1, 15)), + io:format("Waiting for journal deletes~n"), + timer:sleep(20000). + restart_from_blankledger(BookOpts, B_SpcL) -> leveled_penciller:clean_testdir(proplists:get_value(root_path, BookOpts) ++ "/ledger"), From 0cfdcb1976ad36469ca4c9149f52ae44d28f66ba Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 7 Sep 2018 21:03:09 +0100 Subject: [PATCH 6/8] Remove double test should now be covered by the _changes test --- test/end_to_end/recovery_SUITE.erl | 36 ++++++------------------------ 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index cf6e9ef..d7b1df0 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -4,7 +4,6 @@ -export([all/0]). -export([hot_backup_simple/1, hot_backup_changes/1, - hot_backup_double/1, retain_strategy/1, recovr_strategy/1, aae_missingjournal/1, @@ -15,7 +14,6 @@ all() -> [ hot_backup_simple, hot_backup_changes, - hot_backup_double, retain_strategy, recovr_strategy, aae_missingjournal, @@ -34,6 +32,7 @@ hot_backup_simple(_Config) -> % 1 - load a Bookie, take a backup, delete the original path, restore from % that path RootPath = testutil:reset_filestructure(), + BackupPath = testutil:reset_filestructure("backup0"), BookOpts = [{root_path, RootPath}, {cache_size, 1000}, {max_journalsize, 10000000}, @@ -41,7 +40,6 @@ hot_backup_simple(_Config) -> {ok, Spcl1, LastV1} = rotating_object_check(BookOpts, "Bucket1", 10000), {ok, Book1} = leveled_bookie:book_start(BookOpts), {async, BackupFun} = leveled_bookie:book_hotbackup(Book1), - BackupPath = testutil:reset_filestructure("backup0"), ok = BackupFun(BackupPath), ok = leveled_bookie:book_close(Book1), RootPath = testutil:reset_filestructure(), @@ -54,32 +52,6 @@ hot_backup_simple(_Config) -> ok = leveled_bookie:book_close(BookBackup), BackupPath = testutil:reset_filestructure("backup0"). -hot_backup_double(_Config) -> - % As with simple test, but check that calling for backup twice doesn't have - % any side effects - RootPath = testutil:reset_filestructure(), - BookOpts = [{root_path, RootPath}, - {cache_size, 1000}, - {max_journalsize, 10000000}, - {sync_strategy, testutil:sync_strategy()}], - {ok, Spcl1, LastV1} = rotating_object_check(BookOpts, "Bucket1", 4000), - {ok, Book1} = leveled_bookie:book_start(BookOpts), - {async, BackupFun1} = leveled_bookie:book_hotbackup(Book1), - BackupPath = testutil:reset_filestructure("backup0"), - ok = BackupFun1(BackupPath), - {async, BackupFun2} = leveled_bookie:book_hotbackup(Book1), - ok = BackupFun2(BackupPath), - ok = leveled_bookie:book_close(Book1), - RootPath = testutil:reset_filestructure(), - BookOptsBackup = [{root_path, BackupPath}, - {cache_size, 2000}, - {max_journalsize, 20000000}, - {sync_strategy, testutil:sync_strategy()}], - {ok, BookBackup} = leveled_bookie:book_start(BookOptsBackup), - ok = testutil:check_indexed_objects(BookBackup, "Bucket1", Spcl1, LastV1), - ok = leveled_bookie:book_close(BookBackup), - BackupPath = testutil:reset_filestructure("backup0"). - hot_backup_changes(_Config) -> RootPath = testutil:reset_filestructure(), BackupPath = testutil:reset_filestructure("backup0"), @@ -111,9 +83,15 @@ hot_backup_changes(_Config) -> ok = BackupFun3(BackupPath), {ok, FileList3} = file:list_dir(filename:join(BackupPath, "journal/journal_files/")), + % Confirm null impact of backing up twice in a row + {async, BackupFun4} = leveled_bookie:book_hotbackup(Book1), + ok = BackupFun4(BackupPath), + {ok, FileList4} = + file:list_dir(filename:join(BackupPath, "journal/journal_files/")), true = length(FileList2) > length(FileList1), true = length(FileList2) > length(FileList3), + true = length(FileList3) == length(FileList4), ok = leveled_bookie:book_close(Book1), From cd7e64712b111b0ca1d894a7296dbc64b6341c07 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Sun, 9 Sep 2018 18:17:19 +0100 Subject: [PATCH 7/8] Add description --- docs/DESIGN.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 23142a5..4440dd6 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -120,6 +120,16 @@ Three potential recovery strategies are supported to provide some flexibility fo - recalc (not yet implemented) - the compaction rules assume that on recovery the key changes will be recalculated by comparing the change with the current database state. In recovery the key changes will be recalculated by comparing the change with the current database state. +### Hot Backups + +A request can be made to backup a leveled instance, where there is OS support for hard links of files in Erlang. The backup request will return an `{async, BackupFun}` response, and calling `BackupFun(BackupPath)` will cause a backup to be taken into the given path. If a backup already exists in that path, then the Backup will be updated. + +Backups are taken of the Journal only, as the Ledger can be recreated on startup from empty using the KeyChanges in the Journal (backups are not currently an option in `head_only` mode). + +The backup uses hard-links, so at the point the backup is taken, there will be a minimal change to the on-disk footprint of the store. However, as journal compaction is run, the hard-links will prevent space from getting released by the dropping of replaced journal files - so backups will cause the size of the store to grow faster than it would otherwise do. It is an operator responsibility to garbage collect old backups, to prevent this growth from being an issue. + +As backups depend on hard-links, they cannot be taken with a `BackupPath` on a different file system to the standard data path. The move a backup across to a different file system, standard tools should be used such as rsync. The leveled backups should be relatively friendly for rsync-like delta-based backup approaches due to significantly lower write amplification when compared to other LSM stores (e.g. leveldb). + ## Head only Leveled can be started in `head_only` mode. This is a special mode which dispenses with the long-term role of the Journal in retaining data. This is a mode to be used in *special circumstances* when values are small, and Key/Value pairs are added in batches. From 6a87f398f2017909ab6c8717c8c3334dff9bbe11 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Sun, 9 Sep 2018 18:19:25 +0100 Subject: [PATCH 8/8] Reduce object count in simple backup This test is almost entirely covered by `hot_backup_changes` now. --- test/end_to_end/recovery_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index d7b1df0..0f2f81a 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -37,7 +37,7 @@ hot_backup_simple(_Config) -> {cache_size, 1000}, {max_journalsize, 10000000}, {sync_strategy, testutil:sync_strategy()}], - {ok, Spcl1, LastV1} = rotating_object_check(BookOpts, "Bucket1", 10000), + {ok, Spcl1, LastV1} = rotating_object_check(BookOpts, "Bucket1", 3200), {ok, Book1} = leveled_bookie:book_start(BookOpts), {async, BackupFun} = leveled_bookie:book_hotbackup(Book1), ok = BackupFun(BackupPath),