Merge pull request #250 from martinsumner/mas-i249-iclerkshutdown
Mas i249 iclerkshutdown
This commit is contained in:
commit
2c1503b6b5
5 changed files with 102 additions and 116 deletions
|
@ -1005,7 +1005,7 @@ book_snapshot(Pid, SnapType, Query, LongRunning) ->
|
||||||
gen_server:call(Pid, {snapshot, SnapType, Query, LongRunning}, infinity).
|
gen_server:call(Pid, {snapshot, SnapType, Query, LongRunning}, infinity).
|
||||||
|
|
||||||
|
|
||||||
-spec book_compactjournal(pid(), integer()) -> ok.
|
-spec book_compactjournal(pid(), integer()) -> ok|busy.
|
||||||
-spec book_eqccompactjournal(pid(), integer()) -> {ok, pid()}.
|
-spec book_eqccompactjournal(pid(), integer()) -> {ok, pid()}.
|
||||||
-spec book_islastcompactionpending(pid()) -> boolean().
|
-spec book_islastcompactionpending(pid()) -> boolean().
|
||||||
-spec book_trimjournal(pid()) -> ok.
|
-spec book_trimjournal(pid()) -> ok.
|
||||||
|
@ -1016,11 +1016,12 @@ book_snapshot(Pid, SnapType, Query, LongRunning) ->
|
||||||
%% in Riak it will be triggered by a vnode callback.
|
%% in Riak it will be triggered by a vnode callback.
|
||||||
|
|
||||||
book_eqccompactjournal(Pid, Timeout) ->
|
book_eqccompactjournal(Pid, Timeout) ->
|
||||||
gen_server:call(Pid, {compact_journal, Timeout}, infinity).
|
{_R, P} = gen_server:call(Pid, {compact_journal, Timeout}, infinity),
|
||||||
|
{ok, P}.
|
||||||
|
|
||||||
book_compactjournal(Pid, Timeout) ->
|
book_compactjournal(Pid, Timeout) ->
|
||||||
{ok, _P} = gen_server:call(Pid, {compact_journal, Timeout}, infinity),
|
{R, _P} = gen_server:call(Pid, {compact_journal, Timeout}, infinity),
|
||||||
ok.
|
R.
|
||||||
|
|
||||||
%% @doc Check on progress of the last compaction
|
%% @doc Check on progress of the last compaction
|
||||||
|
|
||||||
|
|
|
@ -82,9 +82,10 @@
|
||||||
code_change/3]).
|
code_change/3]).
|
||||||
|
|
||||||
-export([clerk_new/1,
|
-export([clerk_new/1,
|
||||||
clerk_compact/7,
|
clerk_compact/6,
|
||||||
clerk_hashtablecalc/3,
|
clerk_hashtablecalc/3,
|
||||||
clerk_trim/3,
|
clerk_trim/3,
|
||||||
|
clerk_promptdeletions/3,
|
||||||
clerk_stop/1,
|
clerk_stop/1,
|
||||||
clerk_loglevel/2,
|
clerk_loglevel/2,
|
||||||
clerk_addlogs/2,
|
clerk_addlogs/2,
|
||||||
|
@ -148,25 +149,30 @@ clerk_new(InkerClerkOpts) ->
|
||||||
|
|
||||||
-spec clerk_compact(pid(), pid(),
|
-spec clerk_compact(pid(), pid(),
|
||||||
fun(), fun(), fun(),
|
fun(), fun(), fun(),
|
||||||
pid(), integer()) -> ok.
|
list()) -> ok.
|
||||||
%% @doc
|
%% @doc
|
||||||
%% Trigger a compaction for this clerk if the threshold of data recovery has
|
%% Trigger a compaction for this clerk if the threshold of data recovery has
|
||||||
%% been met
|
%% been met
|
||||||
clerk_compact(Pid, Checker, InitiateFun, CloseFun, FilterFun, Inker, TimeO) ->
|
clerk_compact(Pid, Checker, InitiateFun, CloseFun, FilterFun, Manifest) ->
|
||||||
gen_server:cast(Pid,
|
gen_server:cast(Pid,
|
||||||
{compact,
|
{compact,
|
||||||
Checker,
|
Checker,
|
||||||
InitiateFun,
|
InitiateFun,
|
||||||
CloseFun,
|
CloseFun,
|
||||||
FilterFun,
|
FilterFun,
|
||||||
Inker,
|
Manifest}).
|
||||||
TimeO}).
|
|
||||||
|
|
||||||
-spec clerk_trim(pid(), pid(), integer()) -> ok.
|
-spec clerk_trim(pid(), integer(), list()) -> ok.
|
||||||
%% @doc
|
%% @doc
|
||||||
%% Trim the Inker back to the persisted SQN
|
%% Trim the Inker back to the persisted SQN
|
||||||
clerk_trim(Pid, Inker, PersistedSQN) ->
|
clerk_trim(Pid, PersistedSQN, ManifestAsList) ->
|
||||||
gen_server:cast(Pid, {trim, Inker, PersistedSQN}).
|
gen_server:cast(Pid, {trim, PersistedSQN, ManifestAsList}).
|
||||||
|
|
||||||
|
-spec clerk_promptdeletions(pid(), pos_integer(), list()) -> ok.
|
||||||
|
%% @doc
|
||||||
|
%%
|
||||||
|
clerk_promptdeletions(Pid, ManifestSQN, DeletedFiles) ->
|
||||||
|
gen_server:cast(Pid, {prompt_deletions, ManifestSQN, DeletedFiles}).
|
||||||
|
|
||||||
-spec clerk_hashtablecalc(ets:tid(), integer(), pid()) -> ok.
|
-spec clerk_hashtablecalc(ets:tid(), integer(), pid()) -> ok.
|
||||||
%% @doc
|
%% @doc
|
||||||
|
@ -182,8 +188,7 @@ clerk_hashtablecalc(HashTree, StartPos, CDBpid) ->
|
||||||
%% @doc
|
%% @doc
|
||||||
%% Stop the clerk
|
%% Stop the clerk
|
||||||
clerk_stop(Pid) ->
|
clerk_stop(Pid) ->
|
||||||
unlink(Pid),
|
gen_server:call(Pid, stop, 60000).
|
||||||
gen_server:cast(Pid, stop).
|
|
||||||
|
|
||||||
-spec clerk_loglevel(pid(), leveled_log:log_level()) -> ok.
|
-spec clerk_loglevel(pid(), leveled_log:log_level()) -> ok.
|
||||||
%% @doc
|
%% @doc
|
||||||
|
@ -248,10 +253,10 @@ init([LogOpts, IClerkOpts]) ->
|
||||||
compression_method =
|
compression_method =
|
||||||
IClerkOpts#iclerk_options.compression_method}}.
|
IClerkOpts#iclerk_options.compression_method}}.
|
||||||
|
|
||||||
handle_call(_Msg, _From, State) ->
|
handle_call(stop, _From, State) ->
|
||||||
{reply, not_supported, State}.
|
{stop, normal, ok, State}.
|
||||||
|
|
||||||
handle_cast({compact, Checker, InitiateFun, CloseFun, FilterFun, Inker, _TO},
|
handle_cast({compact, Checker, InitiateFun, CloseFun, FilterFun, Manifest0},
|
||||||
State) ->
|
State) ->
|
||||||
% Empty the waste folder
|
% Empty the waste folder
|
||||||
clear_waste(State),
|
clear_waste(State),
|
||||||
|
@ -261,7 +266,8 @@ handle_cast({compact, Checker, InitiateFun, CloseFun, FilterFun, Inker, _TO},
|
||||||
|
|
||||||
% Need to fetch manifest at start rather than have it be passed in
|
% Need to fetch manifest at start rather than have it be passed in
|
||||||
% Don't want to process a queued call waiting on an old manifest
|
% Don't want to process a queued call waiting on an old manifest
|
||||||
[_Active|Manifest] = leveled_inker:ink_getmanifest(Inker),
|
[_Active|Manifest] = Manifest0,
|
||||||
|
Inker = State#state.inker,
|
||||||
MaxRunLength = State#state.max_run_length,
|
MaxRunLength = State#state.max_run_length,
|
||||||
{FilterServer, MaxSQN} = InitiateFun(Checker),
|
{FilterServer, MaxSQN} = InitiateFun(Checker),
|
||||||
CDBopts = State#state.cdb_options,
|
CDBopts = State#state.cdb_options,
|
||||||
|
@ -292,24 +298,29 @@ handle_cast({compact, Checker, InitiateFun, CloseFun, FilterFun, Inker, _TO},
|
||||||
end,
|
end,
|
||||||
BestRun1),
|
BestRun1),
|
||||||
leveled_log:log("IC002", [length(FilesToDelete)]),
|
leveled_log:log("IC002", [length(FilesToDelete)]),
|
||||||
case is_process_alive(Inker) of
|
ok = leveled_inker:ink_clerkcomplete(Inker,
|
||||||
true ->
|
ManifestSlice,
|
||||||
update_inker(Inker,
|
FilesToDelete),
|
||||||
ManifestSlice,
|
ok = CloseFun(FilterServer),
|
||||||
FilesToDelete),
|
{noreply, State};
|
||||||
ok = CloseFun(FilterServer),
|
|
||||||
{noreply, State}
|
|
||||||
end;
|
|
||||||
false ->
|
false ->
|
||||||
ok = leveled_inker:ink_compactioncomplete(Inker),
|
ok = leveled_inker:ink_clerkcomplete(Inker, [], []),
|
||||||
ok = CloseFun(FilterServer),
|
ok = CloseFun(FilterServer),
|
||||||
{noreply, State}
|
{noreply, State}
|
||||||
end;
|
end;
|
||||||
handle_cast({trim, Inker, PersistedSQN}, State) ->
|
handle_cast({trim, PersistedSQN, ManifestAsList}, State) ->
|
||||||
ManifestAsList = leveled_inker:ink_getmanifest(Inker),
|
|
||||||
FilesToDelete =
|
FilesToDelete =
|
||||||
leveled_imanifest:find_persistedentries(PersistedSQN, ManifestAsList),
|
leveled_imanifest:find_persistedentries(PersistedSQN, ManifestAsList),
|
||||||
ok = update_inker(Inker, [], FilesToDelete),
|
leveled_log:log("IC007", []),
|
||||||
|
ok = leveled_inker:ink_clerkcomplete(State#state.inker, [], FilesToDelete),
|
||||||
|
{noreply, State};
|
||||||
|
handle_cast({prompt_deletions, ManifestSQN, FilesToDelete}, State) ->
|
||||||
|
lists:foreach(fun({_SQN, _FN, J2D, _LK}) ->
|
||||||
|
leveled_cdb:cdb_deletepending(J2D,
|
||||||
|
ManifestSQN,
|
||||||
|
State#state.inker)
|
||||||
|
end,
|
||||||
|
FilesToDelete),
|
||||||
{noreply, State};
|
{noreply, State};
|
||||||
handle_cast({hashtable_calc, HashTree, StartPos, CDBpid}, State) ->
|
handle_cast({hashtable_calc, HashTree, StartPos, CDBpid}, State) ->
|
||||||
{IndexList, HashTreeBin} = leveled_cdb:hashtable_calc(HashTree, StartPos),
|
{IndexList, HashTreeBin} = leveled_cdb:hashtable_calc(HashTree, StartPos),
|
||||||
|
@ -329,9 +340,7 @@ handle_cast({remove_logs, ForcedLogs}, State) ->
|
||||||
ok = leveled_log:remove_forcedlogs(ForcedLogs),
|
ok = leveled_log:remove_forcedlogs(ForcedLogs),
|
||||||
CDBopts = State#state.cdb_options,
|
CDBopts = State#state.cdb_options,
|
||||||
CDBopts0 = CDBopts#cdb_options{log_options = leveled_log:get_opts()},
|
CDBopts0 = CDBopts#cdb_options{log_options = leveled_log:get_opts()},
|
||||||
{noreply, State#state{cdb_options = CDBopts0}};
|
{noreply, State#state{cdb_options = CDBopts0}}.
|
||||||
handle_cast(stop, State) ->
|
|
||||||
{stop, normal, State}.
|
|
||||||
|
|
||||||
handle_info(_Info, State) ->
|
handle_info(_Info, State) ->
|
||||||
{noreply, State}.
|
{noreply, State}.
|
||||||
|
@ -614,20 +623,6 @@ sort_run(RunOfFiles) ->
|
||||||
Cand1#candidate.low_sqn =< Cand2#candidate.low_sqn end,
|
Cand1#candidate.low_sqn =< Cand2#candidate.low_sqn end,
|
||||||
lists:sort(CompareFun, RunOfFiles).
|
lists:sort(CompareFun, RunOfFiles).
|
||||||
|
|
||||||
update_inker(Inker, ManifestSlice, FilesToDelete) ->
|
|
||||||
{ok, ManSQN} = leveled_inker:ink_updatemanifest(Inker,
|
|
||||||
ManifestSlice,
|
|
||||||
FilesToDelete),
|
|
||||||
ok = leveled_inker:ink_compactioncomplete(Inker),
|
|
||||||
leveled_log:log("IC007", []),
|
|
||||||
lists:foreach(fun({_SQN, _FN, J2D, _LK}) ->
|
|
||||||
leveled_cdb:cdb_deletepending(J2D,
|
|
||||||
ManSQN,
|
|
||||||
Inker)
|
|
||||||
end,
|
|
||||||
FilesToDelete),
|
|
||||||
ok.
|
|
||||||
|
|
||||||
compact_files(BestRun, CDBopts, FilterFun, FilterServer,
|
compact_files(BestRun, CDBopts, FilterFun, FilterServer,
|
||||||
MaxSQN, RStrategy, PressMethod) ->
|
MaxSQN, RStrategy, PressMethod) ->
|
||||||
BatchesOfPositions = get_all_positions(BestRun, []),
|
BatchesOfPositions = get_all_positions(BestRun, []),
|
||||||
|
@ -1148,7 +1143,6 @@ size_score_test() ->
|
||||||
coverage_cheat_test() ->
|
coverage_cheat_test() ->
|
||||||
{noreply, _State0} = handle_info(timeout, #state{}),
|
{noreply, _State0} = handle_info(timeout, #state{}),
|
||||||
{ok, _State1} = code_change(null, #state{}, null),
|
{ok, _State1} = code_change(null, #state{}, null),
|
||||||
{reply, not_supported, _State2} = handle_call(null, null, #state{}),
|
|
||||||
terminate(error, #state{}).
|
terminate(error, #state{}).
|
||||||
|
|
||||||
-endif.
|
-endif.
|
||||||
|
|
|
@ -105,11 +105,10 @@
|
||||||
ink_registersnapshot/2,
|
ink_registersnapshot/2,
|
||||||
ink_confirmdelete/2,
|
ink_confirmdelete/2,
|
||||||
ink_compactjournal/3,
|
ink_compactjournal/3,
|
||||||
ink_compactioncomplete/1,
|
ink_clerkcomplete/3,
|
||||||
ink_compactionpending/1,
|
ink_compactionpending/1,
|
||||||
ink_trim/2,
|
ink_trim/2,
|
||||||
ink_getmanifest/1,
|
ink_getmanifest/1,
|
||||||
ink_updatemanifest/3,
|
|
||||||
ink_printmanifest/1,
|
ink_printmanifest/1,
|
||||||
ink_close/1,
|
ink_close/1,
|
||||||
ink_doom/1,
|
ink_doom/1,
|
||||||
|
@ -142,7 +141,7 @@
|
||||||
journal_sqn = 0 :: integer(),
|
journal_sqn = 0 :: integer(),
|
||||||
active_journaldb :: pid() | undefined,
|
active_journaldb :: pid() | undefined,
|
||||||
pending_removals = [] :: list(),
|
pending_removals = [] :: list(),
|
||||||
registered_snapshots = [] :: list(),
|
registered_snapshots = [] :: list(registered_snapshot()),
|
||||||
root_path :: string() | undefined,
|
root_path :: string() | undefined,
|
||||||
cdb_options :: #cdb_options{} | undefined,
|
cdb_options :: #cdb_options{} | undefined,
|
||||||
clerk :: pid() | undefined,
|
clerk :: pid() | undefined,
|
||||||
|
@ -157,7 +156,7 @@
|
||||||
|
|
||||||
-type inker_options() :: #inker_options{}.
|
-type inker_options() :: #inker_options{}.
|
||||||
-type ink_state() :: #state{}.
|
-type ink_state() :: #state{}.
|
||||||
|
-type registered_snapshot() :: {pid(), os:timestamp(), integer()}.
|
||||||
|
|
||||||
%%%============================================================================
|
%%%============================================================================
|
||||||
%%% API
|
%%% API
|
||||||
|
@ -348,7 +347,7 @@ ink_loadpcl(Pid, MinSQN, FilterFun, Penciller) ->
|
||||||
as_ink},
|
as_ink},
|
||||||
infinity).
|
infinity).
|
||||||
|
|
||||||
-spec ink_compactjournal(pid(), pid(), integer()) -> {ok, pid()}.
|
-spec ink_compactjournal(pid(), pid(), integer()) -> {ok|busy, pid()}.
|
||||||
%% @doc
|
%% @doc
|
||||||
%% Trigger a compaction event. the compaction event will use a sqn check
|
%% Trigger a compaction event. the compaction event will use a sqn check
|
||||||
%% against the Ledger to see if a value can be compacted - if the penciller
|
%% against the Ledger to see if a value can be compacted - if the penciller
|
||||||
|
@ -359,7 +358,7 @@ ink_loadpcl(Pid, MinSQN, FilterFun, Penciller) ->
|
||||||
%% that any value that was written more recently than the last flush to disk
|
%% that any value that was written more recently than the last flush to disk
|
||||||
%% of the Ledger will not be considered for compaction (as this may be
|
%% of the Ledger will not be considered for compaction (as this may be
|
||||||
%% required to reload the Ledger on startup).
|
%% required to reload the Ledger on startup).
|
||||||
ink_compactjournal(Pid, Bookie, Timeout) ->
|
ink_compactjournal(Pid, Bookie, _Timeout) ->
|
||||||
CheckerInitiateFun = fun initiate_penciller_snapshot/1,
|
CheckerInitiateFun = fun initiate_penciller_snapshot/1,
|
||||||
CheckerCloseFun = fun leveled_penciller:pcl_close/1,
|
CheckerCloseFun = fun leveled_penciller:pcl_close/1,
|
||||||
CheckerFilterFun =
|
CheckerFilterFun =
|
||||||
|
@ -369,28 +368,26 @@ ink_compactjournal(Pid, Bookie, Timeout) ->
|
||||||
Bookie,
|
Bookie,
|
||||||
CheckerInitiateFun,
|
CheckerInitiateFun,
|
||||||
CheckerCloseFun,
|
CheckerCloseFun,
|
||||||
CheckerFilterFun,
|
CheckerFilterFun},
|
||||||
Timeout},
|
|
||||||
infinity).
|
infinity).
|
||||||
|
|
||||||
%% Allows the Checker to be overriden in test, use something other than a
|
%% Allows the Checker to be overriden in test, use something other than a
|
||||||
%% penciller
|
%% penciller
|
||||||
ink_compactjournal(Pid, Checker, InitiateFun, CloseFun, FilterFun, Timeout) ->
|
ink_compactjournal(Pid, Checker, InitiateFun, CloseFun, FilterFun, _Timeout) ->
|
||||||
gen_server:call(Pid,
|
gen_server:call(Pid,
|
||||||
{compact,
|
{compact,
|
||||||
Checker,
|
Checker,
|
||||||
InitiateFun,
|
InitiateFun,
|
||||||
CloseFun,
|
CloseFun,
|
||||||
FilterFun,
|
FilterFun},
|
||||||
Timeout},
|
|
||||||
infinity).
|
infinity).
|
||||||
|
|
||||||
-spec ink_compactioncomplete(pid()) -> ok.
|
-spec ink_clerkcomplete(pid(), list(), list()) -> ok.
|
||||||
%% @doc
|
%% @doc
|
||||||
%% Used by a clerk to state that a compaction process is over, only change
|
%% Used by a clerk to state that a compaction process is over, only change
|
||||||
%% is to unlock the Inker for further compactions.
|
%% is to unlock the Inker for further compactions.
|
||||||
ink_compactioncomplete(Pid) ->
|
ink_clerkcomplete(Pid, ManifestSnippet, FilesToDelete) ->
|
||||||
gen_server:call(Pid, compaction_complete, infinity).
|
gen_server:cast(Pid, {clerk_complete, ManifestSnippet, FilesToDelete}).
|
||||||
|
|
||||||
-spec ink_compactionpending(pid()) -> boolean().
|
-spec ink_compactionpending(pid()) -> boolean().
|
||||||
%% @doc
|
%% @doc
|
||||||
|
@ -425,21 +422,6 @@ ink_backup(Pid, BackupPath) ->
|
||||||
ink_getmanifest(Pid) ->
|
ink_getmanifest(Pid) ->
|
||||||
gen_server:call(Pid, get_manifest, infinity).
|
gen_server:call(Pid, get_manifest, infinity).
|
||||||
|
|
||||||
-spec ink_updatemanifest(pid(), list(), list()) -> {ok, integer()}.
|
|
||||||
%% @doc
|
|
||||||
%% Add a section of new entries into the manifest, and drop a bunch of deleted
|
|
||||||
%% files out of the manifest. Used to update the manifest after a compaction
|
|
||||||
%% job.
|
|
||||||
%%
|
|
||||||
%% Returns {ok, ManSQN} with the ManSQN being the sequence number of the
|
|
||||||
%% updated manifest
|
|
||||||
ink_updatemanifest(Pid, ManifestSnippet, DeletedFiles) ->
|
|
||||||
gen_server:call(Pid,
|
|
||||||
{update_manifest,
|
|
||||||
ManifestSnippet,
|
|
||||||
DeletedFiles},
|
|
||||||
infinity).
|
|
||||||
|
|
||||||
-spec ink_printmanifest(pid()) -> ok.
|
-spec ink_printmanifest(pid()) -> ok.
|
||||||
%% @doc
|
%% @doc
|
||||||
%% Used in tests to print out the manifest
|
%% Used in tests to print out the manifest
|
||||||
|
@ -574,27 +556,6 @@ handle_call({confirm_delete, ManSQN}, _From, State) ->
|
||||||
State#state{registered_snapshots = RegisteredSnapshots0}};
|
State#state{registered_snapshots = RegisteredSnapshots0}};
|
||||||
handle_call(get_manifest, _From, State) ->
|
handle_call(get_manifest, _From, State) ->
|
||||||
{reply, leveled_imanifest:to_list(State#state.manifest), State};
|
{reply, leveled_imanifest:to_list(State#state.manifest), State};
|
||||||
handle_call({update_manifest,
|
|
||||||
ManifestSnippet,
|
|
||||||
DeletedFiles}, _From, State) ->
|
|
||||||
DropFun =
|
|
||||||
fun(E, Acc) ->
|
|
||||||
leveled_imanifest:remove_entry(Acc, E)
|
|
||||||
end,
|
|
||||||
Man0 = lists:foldl(DropFun, State#state.manifest, DeletedFiles),
|
|
||||||
AddFun =
|
|
||||||
fun(E, Acc) ->
|
|
||||||
leveled_imanifest:add_entry(Acc, E, false)
|
|
||||||
end,
|
|
||||||
Man1 = lists:foldl(AddFun, Man0, ManifestSnippet),
|
|
||||||
NewManifestSQN = State#state.manifest_sqn + 1,
|
|
||||||
leveled_imanifest:printer(Man1),
|
|
||||||
leveled_imanifest:writer(Man1, NewManifestSQN, State#state.root_path),
|
|
||||||
{reply,
|
|
||||||
{ok, NewManifestSQN},
|
|
||||||
State#state{manifest=Man1,
|
|
||||||
manifest_sqn=NewManifestSQN,
|
|
||||||
pending_removals=DeletedFiles}};
|
|
||||||
handle_call(print_manifest, _From, State) ->
|
handle_call(print_manifest, _From, State) ->
|
||||||
leveled_imanifest:printer(State#state.manifest),
|
leveled_imanifest:printer(State#state.manifest),
|
||||||
{reply, ok, State};
|
{reply, ok, State};
|
||||||
|
@ -602,23 +563,27 @@ handle_call({compact,
|
||||||
Checker,
|
Checker,
|
||||||
InitiateFun,
|
InitiateFun,
|
||||||
CloseFun,
|
CloseFun,
|
||||||
FilterFun,
|
FilterFun},
|
||||||
Timeout},
|
|
||||||
_From, State) ->
|
_From, State) ->
|
||||||
leveled_iclerk:clerk_compact(State#state.clerk,
|
Clerk = State#state.clerk,
|
||||||
Checker,
|
case State#state.compaction_pending of
|
||||||
InitiateFun,
|
true ->
|
||||||
CloseFun,
|
{reply, {busy, Clerk}, State};
|
||||||
FilterFun,
|
false ->
|
||||||
self(),
|
Manifest = leveled_imanifest:to_list(State#state.manifest),
|
||||||
Timeout),
|
leveled_iclerk:clerk_compact(State#state.clerk,
|
||||||
{reply, {ok, State#state.clerk}, State#state{compaction_pending=true}};
|
Checker,
|
||||||
handle_call(compaction_complete, _From, State) ->
|
InitiateFun,
|
||||||
{reply, ok, State#state{compaction_pending=false}};
|
CloseFun,
|
||||||
|
FilterFun,
|
||||||
|
Manifest),
|
||||||
|
{reply, {ok, Clerk}, State#state{compaction_pending=true}}
|
||||||
|
end;
|
||||||
handle_call(compaction_pending, _From, State) ->
|
handle_call(compaction_pending, _From, State) ->
|
||||||
{reply, State#state.compaction_pending, State};
|
{reply, State#state.compaction_pending, State};
|
||||||
handle_call({trim, PersistedSQN}, _From, State) ->
|
handle_call({trim, PersistedSQN}, _From, State) ->
|
||||||
ok = leveled_iclerk:clerk_trim(State#state.clerk, self(), PersistedSQN),
|
Manifest = leveled_imanifest:to_list(State#state.manifest),
|
||||||
|
ok = leveled_iclerk:clerk_trim(State#state.clerk, PersistedSQN, Manifest),
|
||||||
{reply, ok, State};
|
{reply, ok, State};
|
||||||
handle_call(roll, _From, State) ->
|
handle_call(roll, _From, State) ->
|
||||||
case leveled_cdb:cdb_lastkey(State#state.active_journaldb) of
|
case leveled_cdb:cdb_lastkey(State#state.active_journaldb) of
|
||||||
|
@ -712,7 +677,7 @@ handle_call(close, _From, State) ->
|
||||||
leveled_log:log("I0005", [close]),
|
leveled_log:log("I0005", [close]),
|
||||||
leveled_log:log("I0006", [State#state.journal_sqn,
|
leveled_log:log("I0006", [State#state.journal_sqn,
|
||||||
State#state.manifest_sqn]),
|
State#state.manifest_sqn]),
|
||||||
leveled_iclerk:clerk_stop(State#state.clerk),
|
ok = leveled_iclerk:clerk_stop(State#state.clerk),
|
||||||
shutdown_snapshots(State#state.registered_snapshots),
|
shutdown_snapshots(State#state.registered_snapshots),
|
||||||
shutdown_manifest(State#state.manifest)
|
shutdown_manifest(State#state.manifest)
|
||||||
end,
|
end,
|
||||||
|
@ -727,12 +692,33 @@ handle_call(doom, _From, State) ->
|
||||||
leveled_log:log("I0005", [doom]),
|
leveled_log:log("I0005", [doom]),
|
||||||
leveled_log:log("I0006", [State#state.journal_sqn,
|
leveled_log:log("I0006", [State#state.journal_sqn,
|
||||||
State#state.manifest_sqn]),
|
State#state.manifest_sqn]),
|
||||||
leveled_iclerk:clerk_stop(State#state.clerk),
|
ok = leveled_iclerk:clerk_stop(State#state.clerk),
|
||||||
shutdown_snapshots(State#state.registered_snapshots),
|
shutdown_snapshots(State#state.registered_snapshots),
|
||||||
shutdown_manifest(State#state.manifest),
|
shutdown_manifest(State#state.manifest),
|
||||||
|
|
||||||
{stop, normal, {ok, FPs}, State}.
|
{stop, normal, {ok, FPs}, State}.
|
||||||
|
|
||||||
|
|
||||||
|
handle_cast({clerk_complete, ManifestSnippet, FilesToDelete}, State) ->
|
||||||
|
DropFun =
|
||||||
|
fun(E, Acc) ->
|
||||||
|
leveled_imanifest:remove_entry(Acc, E)
|
||||||
|
end,
|
||||||
|
Man0 = lists:foldl(DropFun, State#state.manifest, FilesToDelete),
|
||||||
|
AddFun =
|
||||||
|
fun(E, Acc) ->
|
||||||
|
leveled_imanifest:add_entry(Acc, E, false)
|
||||||
|
end,
|
||||||
|
Man1 = lists:foldl(AddFun, Man0, ManifestSnippet),
|
||||||
|
NewManifestSQN = State#state.manifest_sqn + 1,
|
||||||
|
leveled_imanifest:printer(Man1),
|
||||||
|
leveled_imanifest:writer(Man1, NewManifestSQN, State#state.root_path),
|
||||||
|
ok = leveled_iclerk:clerk_promptdeletions(State#state.clerk,
|
||||||
|
NewManifestSQN,
|
||||||
|
FilesToDelete),
|
||||||
|
{noreply, State#state{manifest=Man1,
|
||||||
|
manifest_sqn=NewManifestSQN,
|
||||||
|
pending_removals=FilesToDelete,
|
||||||
|
compaction_pending=false}};
|
||||||
handle_cast({release_snapshot, Snapshot}, State) ->
|
handle_cast({release_snapshot, Snapshot}, State) ->
|
||||||
Rs = lists:keydelete(Snapshot, 1, State#state.registered_snapshots),
|
Rs = lists:keydelete(Snapshot, 1, State#state.registered_snapshots),
|
||||||
leveled_log:log("I0003", [Snapshot]),
|
leveled_log:log("I0003", [Snapshot]),
|
||||||
|
@ -843,11 +829,12 @@ start_from_file(InkOpts) ->
|
||||||
clerk = Clerk}}.
|
clerk = Clerk}}.
|
||||||
|
|
||||||
|
|
||||||
-spec shutdown_snapshots(list(tuple())) -> ok.
|
-spec shutdown_snapshots(list(registered_snapshot())) -> ok.
|
||||||
%% @doc
|
%% @doc
|
||||||
%% Shutdown any snapshots before closing the store
|
%% Shutdown any snapshots before closing the store
|
||||||
shutdown_snapshots(Snapshots) ->
|
shutdown_snapshots(Snapshots) ->
|
||||||
lists:foreach(fun({Snap, _TS, _SQN}) -> ok = ink_close(Snap) end, Snapshots).
|
lists:foreach(fun({Snap, _TS, _SQN}) -> ok = ink_close(Snap) end,
|
||||||
|
Snapshots).
|
||||||
|
|
||||||
-spec shutdown_manifest(leveled_imanifest:manifest()) -> ok.
|
-spec shutdown_manifest(leveled_imanifest:manifest()) -> ok.
|
||||||
%% @doc
|
%% @doc
|
||||||
|
|
|
@ -300,6 +300,7 @@ journal_compaction_tester(Restart, WRP) ->
|
||||||
{sync_strategy, testutil:sync_strategy()}],
|
{sync_strategy, testutil:sync_strategy()}],
|
||||||
{ok, Bookie3} = leveled_bookie:book_start(StartOpts2),
|
{ok, Bookie3} = leveled_bookie:book_start(StartOpts2),
|
||||||
ok = leveled_bookie:book_compactjournal(Bookie3, 30000),
|
ok = leveled_bookie:book_compactjournal(Bookie3, 30000),
|
||||||
|
busy = leveled_bookie:book_compactjournal(Bookie3, 30000),
|
||||||
testutil:wait_for_compaction(Bookie3),
|
testutil:wait_for_compaction(Bookie3),
|
||||||
ok = leveled_bookie:book_close(Bookie3),
|
ok = leveled_bookie:book_close(Bookie3),
|
||||||
|
|
||||||
|
|
|
@ -724,6 +724,9 @@ basic_headonly_test(ObjectCount, RemoveCount, HeadOnly) ->
|
||||||
|
|
||||||
{ok, FinalFNs} = file:list_dir(JFP),
|
{ok, FinalFNs} = file:list_dir(JFP),
|
||||||
|
|
||||||
|
ok = leveled_bookie:book_trimjournal(Bookie1),
|
||||||
|
% CCheck a second trim is still OK
|
||||||
|
|
||||||
[{add, SegmentID0, Bucket0, Key0, Hash0}|_Rest] = ObjectSpecL,
|
[{add, SegmentID0, Bucket0, Key0, Hash0}|_Rest] = ObjectSpecL,
|
||||||
case HeadOnly of
|
case HeadOnly of
|
||||||
with_lookup ->
|
with_lookup ->
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue