From 42c4100c2d3f91e78dc7ab4962e4caa1806825e7 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 28 Mar 2019 13:23:37 +0000 Subject: [PATCH 1/3] Add GC are initialisation in OTP R16 (and perhaps other OTP releases) there is a failure to fully garbage collect leveled_sst files after thya have initialised. They sppear to maintain a 4MB "hangover" from the initialisation process. This can be removed by manually calling garbage_collect. So we do this now on all new non-L0 files. A L0 file will be short-lived or switched - short-lived and it doesn't matter, switched and this is already GC'd. --- src/leveled_sst.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index d9accee..16861af 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -512,12 +512,14 @@ starting({sst_new, leveled_log:log_timer("SST08", [ActualFilename, Level, Summary#summary.max_sqn], SW), + erlang:send_after(?STARTUP_TIMEOUT, self(), timeout), + % always want to have an opportunity to GC - so force the timeout to + % occur whether or not there is an intervening message {reply, {ok, {Summary#summary.first_key, Summary#summary.last_key}, Bloom}, reader, UpdState#state{blockindex_cache = BlockIndex, - starting_pid = StartingPID}, - ?STARTUP_TIMEOUT}; + starting_pid = StartingPID}}; starting({sst_newlevelzero, RootPath, Filename, Penciller, MaxSQN, OptsSST, IdxModDate}, _From, State) -> @@ -715,6 +717,7 @@ reader(switch_levels, State) -> reader(timeout, State) -> case is_process_alive(State#state.starting_pid) of true -> + erlang:garbage_collect(self()), {next_state, reader, State}; false -> {stop, normal, State} From dfa857469534844d056513cd241ae6f78bf52927 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 28 Mar 2019 17:46:08 +0000 Subject: [PATCH 2/3] Use correct send So it actually works --- src/leveled_sst.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index 16861af..91559ce 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -512,7 +512,7 @@ starting({sst_new, leveled_log:log_timer("SST08", [ActualFilename, Level, Summary#summary.max_sqn], SW), - erlang:send_after(?STARTUP_TIMEOUT, self(), timeout), + gen_fsm:send_event(self(), timeout), % always want to have an opportunity to GC - so force the timeout to % occur whether or not there is an intervening message {reply, From 2f3d2a634c15c90deb2bcf9417b561046c7ec431 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 28 Mar 2019 21:01:01 +0000 Subject: [PATCH 3/3] Correct the tidyup after startup Use send_after/3 and unit test to confirm this works as expected --- src/leveled_sst.erl | 61 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index 91559ce..2168bb9 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -512,7 +512,7 @@ starting({sst_new, leveled_log:log_timer("SST08", [ActualFilename, Level, Summary#summary.max_sqn], SW), - gen_fsm:send_event(self(), timeout), + erlang:send_after(?STARTUP_TIMEOUT, self(), tidyup_after_startup), % always want to have an opportunity to GC - so force the timeout to % occur whether or not there is an intervening message {reply, @@ -713,15 +713,7 @@ reader(close, _From, State) -> reader(switch_levels, State) -> erlang:garbage_collect(self()), - {next_state, reader, State}; -reader(timeout, State) -> - case is_process_alive(State#state.starting_pid) of - true -> - erlang:garbage_collect(self()), - {next_state, reader, State}; - false -> - {stop, normal, State} - end. + {next_state, reader, State}. delete_pending({get_kv, LedgerKey, Hash}, _From, State) -> @@ -784,8 +776,14 @@ handle_sync_event(_Msg, _From, StateName, State) -> handle_event(_Msg, StateName, State) -> {next_state, StateName, State}. -handle_info(_Msg, StateName, State) -> - {next_state, StateName, State}. +handle_info(tidyup_after_startup, StateName, State) -> + case is_process_alive(State#state.starting_pid) of + true -> + erlang:garbage_collect(self()), + {next_state, StateName, State}; + false -> + {stop, normal, State} + end. terminate(normal, delete_pending, _State) -> ok; @@ -3355,9 +3353,6 @@ key_dominates_test() -> nonsense_coverage_test() -> {ok, Pid} = gen_fsm:start_link(?MODULE, [], []), ok = gen_fsm:send_all_state_event(Pid, nonsense), - ?assertMatch({next_state, reader, #state{}}, handle_info(nonsense, - reader, - #state{})), ?assertMatch({ok, reader, #state{}}, code_change(nonsense, reader, #state{}, @@ -3435,4 +3430,40 @@ stopstart_test() -> % fetcher on new level zero files working in a loop ok = sst_close(Pid). +stop_whenstarter_stopped_test_() -> + {timeout, 60, fun() -> stop_whenstarter_stopped_testto() end}. + +stop_whenstarter_stopped_testto() -> + RP = spawn(fun receive_fun/0), + spawn(fun() -> start_sst_fun(RP) end), + TestFun = + fun(X, Acc) -> + case Acc of + false -> false; + true -> + timer:sleep(X), + is_process_alive(RP) + end + end, + ?assertMatch(false, lists:foldl(TestFun, true, [10000, 2000, 2000, 2000])). + + +receive_fun() -> + receive + {sst_pid, SST_P} -> + timer:sleep(?STARTUP_TIMEOUT + 1000), + ?assertMatch(false, is_process_alive(SST_P)) + end. + +start_sst_fun(ProcessToInform) -> + N = 3000, + KVL1 = lists:ukeysort(1, generate_randomkeys(N + 1, N, 1, 20)), + OptsSST = + #sst_options{press_method=native, + log_options=leveled_log:get_opts()}, + {ok, P1, {_FK1, _LK1}, _Bloom1} = + sst_new("test/test_area/", "level1_src", 1, KVL1, 6000, OptsSST), + ProcessToInform ! {sst_pid, P1}. + + -endif.