umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Release penciller snapshot after journal compaction

Browse source 
As otherwise memory consumption beocmes an issue, as they will take an
hour to timeout naturally.
This commit is contained in:
martinsumner 2017-05-26 10:51:30 +01:00
parent 5c5fa8ee8d
commit c664176247
2 changed files with 16 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

11
src/leveled_iclerk.erl
View file

@ -80,7 +80,7 @@
handle_info/2, handle_info/2,
terminate/2, terminate/2,
clerk_new/1, clerk_new/1,
clerk_compact/6, clerk_compact/7,
clerk_hashtablecalc/3, clerk_hashtablecalc/3,
clerk_stop/1, clerk_stop/1,
code_change/3]). code_change/3]).
@ -124,14 +124,15 @@
clerk_new(InkerClerkOpts) -> clerk_new(InkerClerkOpts) ->
gen_server:start(?MODULE, [InkerClerkOpts], []). gen_server:start(?MODULE, [InkerClerkOpts], []).
clerk_compact(Pid, Checker, InitiateFun, FilterFun, Inker, Timeout) -> clerk_compact(Pid, Checker, InitiateFun, CloseFun, FilterFun, Inker, TimeO) ->
gen_server:cast(Pid, gen_server:cast(Pid,
{compact, {compact,
Checker, Checker,
InitiateFun, InitiateFun,
CloseFun,
FilterFun, FilterFun,
Inker, Inker,
Timeout}). TimeO}).
clerk_hashtablecalc(HashTree, StartPos, CDBpid) -> clerk_hashtablecalc(HashTree, StartPos, CDBpid) ->
{ok, Clerk} = gen_server:start(?MODULE, [#iclerk_options{}], []), {ok, Clerk} = gen_server:start(?MODULE, [#iclerk_options{}], []),
@ -171,7 +172,7 @@ init([IClerkOpts]) ->
handle_call(_Msg, _From, State) -> handle_call(_Msg, _From, State) ->
{reply, not_supported, State}. {reply, not_supported, State}.
handle_cast({compact, Checker, InitiateFun, FilterFun, Inker, _Timeout}, handle_cast({compact, Checker, InitiateFun, CloseFun, FilterFun, Inker, _TO},
State) -> State) ->
% Empty the waste folder % Empty the waste folder
clear_waste(State), clear_waste(State),
@ -207,11 +208,13 @@ handle_cast({compact, Checker, InitiateFun, FilterFun, Inker, _Timeout},
update_inker(Inker, update_inker(Inker,
ManifestSlice, ManifestSlice,
FilesToDelete), FilesToDelete),
ok = CloseFun(FilterServer),
{noreply, State} {noreply, State}
end; end;
Score -> Score ->
leveled_log:log("IC003", [Score]), leveled_log:log("IC003", [Score]),
ok = leveled_inker:ink_compactioncomplete(Inker), ok = leveled_inker:ink_compactioncomplete(Inker),
ok = CloseFun(FilterServer),
{noreply, State} {noreply, State}
end; end;
handle_cast({hashtable_calc, HashTree, StartPos, CDBpid}, State) -> handle_cast({hashtable_calc, HashTree, StartPos, CDBpid}, State) ->

11
src/leveled_inker.erl
View file

@ -178,22 +178,25 @@ ink_loadpcl(Pid, MinSQN, FilterFun, Penciller) ->
ink_compactjournal(Pid, Bookie, Timeout) -> ink_compactjournal(Pid, Bookie, Timeout) ->
CheckerInitiateFun = fun initiate_penciller_snapshot/1, CheckerInitiateFun = fun initiate_penciller_snapshot/1,
CheckerCloseFun = fun leveled_penciller:pcl_close/1,
CheckerFilterFun = fun leveled_penciller:pcl_checksequencenumber/3, CheckerFilterFun = fun leveled_penciller:pcl_checksequencenumber/3,
gen_server:call(Pid, gen_server:call(Pid,
{compact, {compact,
Bookie, Bookie,
CheckerInitiateFun, CheckerInitiateFun,
CheckerCloseFun,
CheckerFilterFun, CheckerFilterFun,
Timeout}, Timeout},
infinity). infinity).
%% Allows the Checker to be overriden in test, use something other than a %% Allows the Checker to be overriden in test, use something other than a
%% penciller %% penciller
ink_compactjournal(Pid, Checker, InitiateFun, FilterFun, Timeout) -> ink_compactjournal(Pid, Checker, InitiateFun, CloseFun, FilterFun, Timeout) ->
gen_server:call(Pid, gen_server:call(Pid,
{compact, {compact,
Checker, Checker,
InitiateFun, InitiateFun,
CloseFun,
FilterFun, FilterFun,
Timeout}, Timeout},
infinity). infinity).
@ -308,12 +311,14 @@ handle_call(print_manifest, _From, State) ->
handle_call({compact, handle_call({compact,
Checker, Checker,
InitiateFun, InitiateFun,
CloseFun,
FilterFun, FilterFun,
Timeout}, Timeout},
_From, State) -> _From, State) ->
leveled_iclerk:clerk_compact(State#state.clerk, leveled_iclerk:clerk_compact(State#state.clerk,
Checker, Checker,
InitiateFun, InitiateFun,
CloseFun,
FilterFun, FilterFun,
self(), self(),
Timeout), Timeout),
@ -720,7 +725,6 @@ initiate_penciller_snapshot(Bookie) ->
MaxSQN = leveled_penciller:pcl_getstartupsequencenumber(LedgerSnap), MaxSQN = leveled_penciller:pcl_getstartupsequencenumber(LedgerSnap),
{LedgerSnap, MaxSQN}. {LedgerSnap, MaxSQN}.
%%%============================================================================ %%%============================================================================
%%% Test %%% Test
%%%============================================================================ %%%============================================================================
@ -878,6 +882,7 @@ compact_journal_test() ->
ok = ink_compactjournal(Ink1, ok = ink_compactjournal(Ink1,
Checker, Checker,
fun(X) -> {X, 55} end, fun(X) -> {X, 55} end,
fun(_F) -> ok end,
fun(L, K, SQN) -> lists:member({SQN, K}, L) end, fun(L, K, SQN) -> lists:member({SQN, K}, L) end,
5000), 5000),
timer:sleep(1000), timer:sleep(1000),
@ -887,6 +892,7 @@ compact_journal_test() ->
ok = ink_compactjournal(Ink1, ok = ink_compactjournal(Ink1,
Checker2, Checker2,
fun(X) -> {X, 55} end, fun(X) -> {X, 55} end,
fun(_F) -> ok end,
fun(L, K, SQN) -> lists:member({SQN, K}, L) end, fun(L, K, SQN) -> lists:member({SQN, K}, L) end,
5000), 5000),
timer:sleep(1000), timer:sleep(1000),
@ -918,6 +924,7 @@ empty_manifest_test() ->
ok = ink_compactjournal(Ink1, ok = ink_compactjournal(Ink1,
[], [],
fun(X) -> {X, 55} end, fun(X) -> {X, 55} end,
fun(_F) -> ok end,
CheckFun, CheckFun,
5000), 5000),
timer:sleep(1000), timer:sleep(1000),