From 672cfd4fcdb6b471ef60c2f930da9f18e82b53f1 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 11 Dec 2018 21:59:57 +0000 Subject: [PATCH] Allow for run-time changes to log_level and forced_logs Will not lead to immediate run time changes in SST or CDB logs. These log settings will only change once the new files are re-written. To completely change the log level - a restart of the store is necessary with new startup options. --- src/leveled_bookie.erl | 59 ++++++++++++++++++++++++++++++--- src/leveled_iclerk.erl | 43 ++++++++++++++++++++++-- src/leveled_inker.erl | 47 ++++++++++++++++++++++++-- src/leveled_log.erl | 11 ++++-- src/leveled_pclerk.erl | 40 ++++++++++++++++++++-- src/leveled_penciller.erl | 47 ++++++++++++++++++++++++-- test/end_to_end/basic_SUITE.erl | 8 +++++ 7 files changed, 239 insertions(+), 16 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 89067ea..9cfba69 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -67,7 +67,11 @@ book_hotbackup/1, book_close/1, book_destroy/1, - book_isempty/2]). + book_isempty/2, + book_logsettings/1, + book_loglevel/2, + book_addlogs/2, + book_removelogs/2]). %% folding API -export([ @@ -1047,6 +1051,31 @@ book_isempty(Pid, Tag) -> {async, Runner} = book_bucketlist(Pid, Tag, FoldAccT, first), Runner(). +-spec book_logsettings(pid()) -> {leveled_log:log_level(), list(string())}. +%% @doc +%% Retrieve the current log settings +book_logsettings(Pid) -> + gen_server:call(Pid, log_settings, infinity). + +-spec book_loglevel(pid(), leveled_log:log_level()) -> ok. +%% @doc +%% Change the log level of the store +book_loglevel(Pid, LogLevel) -> + gen_server:cast(Pid, {log_level, LogLevel}). + +-spec book_addlogs(pid(), list(string())) -> ok. +%% @doc +%% Add to the list of forced logs, a list of more forced logs +book_addlogs(Pid, ForcedLogs) -> + gen_server:cast(Pid, {add_logs, ForcedLogs}). + +-spec book_removelogs(pid(), list(string())) -> ok. +%% @doc +%% Remove from the list of forced logs, a list of forced logs +book_removelogs(Pid, ForcedLogs) -> + gen_server:cast(Pid, {remove_logs, ForcedLogs}). + + %%%============================================================================ %%% gen_server callbacks %%%============================================================================ @@ -1302,6 +1331,8 @@ handle_call({snapshot, SnapType, Query, LongRunning}, _From, State) -> % e.g. many minutes) Reply = snapshot_store(State, SnapType, Query, LongRunning), {reply, Reply, State}; +handle_call(log_settings, _From, State) -> + {reply, leveled_log:return_settings(), State}; handle_call({return_runner, QueryType}, _From, State) -> SW = os:timestamp(), Runner = get_runner(State, QueryType), @@ -1352,9 +1383,30 @@ handle_call(destroy, _From, State=#state{is_snapshot=Snp}) when Snp == false -> handle_call(Msg, _From, State) -> {reply, {unsupported_message, element(1, Msg)}, State}. -handle_cast(_Msg, State) -> + +handle_cast({log_level, LogLevel}, State) -> + PCL = State#state.penciller, + INK = State#state.inker, + ok = leveled_penciller:pcl_loglevel(PCL, LogLevel), + ok = leveled_inker:ink_loglevel(INK, LogLevel), + ok = leveled_log:set_loglevel(LogLevel), + {noreply, State}; +handle_cast({add_logs, ForcedLogs}, State) -> + PCL = State#state.penciller, + INK = State#state.inker, + ok = leveled_penciller:pcl_addlogs(PCL, ForcedLogs), + ok = leveled_inker:ink_addlogs(INK, ForcedLogs), + ok = leveled_log:add_forcedlogs(ForcedLogs), + {noreply, State}; +handle_cast({remove_logs, ForcedLogs}, State) -> + PCL = State#state.penciller, + INK = State#state.inker, + ok = leveled_penciller:pcl_removelogs(PCL, ForcedLogs), + ok = leveled_inker:ink_removelogs(INK, ForcedLogs), + ok = leveled_log:remove_forcedlogs(ForcedLogs), {noreply, State}. + handle_info(_Info, State) -> {noreply, State}. @@ -2879,8 +2931,7 @@ longrunning_test() -> coverage_cheat_test() -> {noreply, _State0} = handle_info(timeout, #state{}), - {ok, _State1} = code_change(null, #state{}, null), - {noreply, _State2} = handle_cast(null, #state{}). + {ok, _State1} = code_change(null, #state{}, null). erase_journal_test() -> RootPath = reset_filestructure(), diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index e474364..167c63d 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -79,12 +79,16 @@ handle_cast/2, handle_info/2, terminate/2, - clerk_new/1, + code_change/3]). + +-export([clerk_new/1, clerk_compact/7, clerk_hashtablecalc/3, clerk_trim/3, clerk_stop/1, - code_change/3]). + clerk_loglevel/2, + clerk_addlogs/2, + clerk_removelogs/2]). -export([schedule_compaction/3]). @@ -104,7 +108,7 @@ -record(state, {inker :: pid() | undefined, max_run_length :: integer() | undefined, - cdb_options, + cdb_options = #cdb_options{} :: #cdb_options{}, waste_retention_period :: integer() | undefined, waste_path :: string() | undefined, reload_strategy = ?DEFAULT_RELOAD_STRATEGY :: list(), @@ -180,6 +184,24 @@ clerk_hashtablecalc(HashTree, StartPos, CDBpid) -> clerk_stop(Pid) -> gen_server:cast(Pid, stop). +-spec clerk_loglevel(pid(), leveled_log:log_level()) -> ok. +%% @doc +%% Change the log level of the Journal +clerk_loglevel(Pid, LogLevel) -> + gen_server:cast(Pid, {log_level, LogLevel}). + +-spec clerk_addlogs(pid(), list(string())) -> ok. +%% @doc +%% Add to the list of forced logs, a list of more forced logs +clerk_addlogs(Pid, ForcedLogs) -> + gen_server:cast(Pid, {add_logs, ForcedLogs}). + +-spec clerk_removelogs(pid(), list(string())) -> ok. +%% @doc +%% Remove from the list of forced logs, a list of forced logs +clerk_removelogs(Pid, ForcedLogs) -> + gen_server:cast(Pid, {remove_logs, ForcedLogs}). + %%%============================================================================ %%% gen_server callbacks %%%============================================================================ @@ -292,6 +314,21 @@ handle_cast({hashtable_calc, HashTree, StartPos, CDBpid}, State) -> {IndexList, HashTreeBin} = leveled_cdb:hashtable_calc(HashTree, StartPos), ok = leveled_cdb:cdb_returnhashtable(CDBpid, IndexList, HashTreeBin), {stop, normal, State}; +handle_cast({log_level, LogLevel}, State) -> + ok = leveled_log:set_loglevel(LogLevel), + CDBopts = State#state.cdb_options, + CDBopts0 = CDBopts#cdb_options{log_options = leveled_log:get_opts()}, + {noreply, State#state{cdb_options = CDBopts0}}; +handle_cast({add_logs, ForcedLogs}, State) -> + ok = leveled_log:add_forcedlogs(ForcedLogs), + CDBopts = State#state.cdb_options, + CDBopts0 = CDBopts#cdb_options{log_options = leveled_log:get_opts()}, + {noreply, State#state{cdb_options = CDBopts0}}; +handle_cast({remove_logs, ForcedLogs}, State) -> + ok = leveled_log:remove_forcedlogs(ForcedLogs), + CDBopts = State#state.cdb_options, + CDBopts0 = CDBopts#cdb_options{log_options = leveled_log:get_opts()}, + {noreply, State#state{cdb_options = CDBopts0}}; handle_cast(stop, State) -> {stop, normal, State}. diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 3ecae53..01571f6 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -116,7 +116,11 @@ ink_roll/1, ink_backup/2, ink_checksqn/2, - build_dummy_journal/0, + ink_loglevel/2, + ink_addlogs/2, + ink_removelogs/2]). + +-export([build_dummy_journal/0, clean_testdir/1, filepath/2, filepath/3]). @@ -447,6 +451,24 @@ ink_printmanifest(Pid) -> ink_checksqn(Pid, LedgerSQN) -> gen_server:call(Pid, {check_sqn, LedgerSQN}). +-spec ink_loglevel(pid(), leveled_log:log_level()) -> ok. +%% @doc +%% Change the log level of the Journal +ink_loglevel(Pid, LogLevel) -> + gen_server:cast(Pid, {log_level, LogLevel}). + +-spec ink_addlogs(pid(), list(string())) -> ok. +%% @doc +%% Add to the list of forced logs, a list of more forced logs +ink_addlogs(Pid, ForcedLogs) -> + gen_server:cast(Pid, {add_logs, ForcedLogs}). + +-spec ink_removelogs(pid(), list(string())) -> ok. +%% @doc +%% Remove from the list of forced logs, a list of forced logs +ink_removelogs(Pid, ForcedLogs) -> + gen_server:cast(Pid, {remove_logs, ForcedLogs}). + %%%============================================================================ %%% gen_server callbacks %%%============================================================================ @@ -698,7 +720,28 @@ handle_cast({release_snapshot, Snapshot}, State) -> Rs = lists:keydelete(Snapshot, 1, State#state.registered_snapshots), leveled_log:log("I0003", [Snapshot]), leveled_log:log("I0004", [length(Rs)]), - {noreply, State#state{registered_snapshots=Rs}}. + {noreply, State#state{registered_snapshots=Rs}}; +handle_cast({log_level, LogLevel}, State) -> + INC = State#state.clerk, + ok = leveled_iclerk:clerk_loglevel(INC, LogLevel), + ok = leveled_log:set_loglevel(LogLevel), + CDBopts = State#state.cdb_options, + CDBopts0 = CDBopts#cdb_options{log_options = leveled_log:get_opts()}, + {noreply, State#state{cdb_options = CDBopts0}}; +handle_cast({add_logs, ForcedLogs}, State) -> + INC = State#state.clerk, + ok = leveled_iclerk:clerk_addlogs(INC, ForcedLogs), + ok = leveled_log:add_forcedlogs(ForcedLogs), + CDBopts = State#state.cdb_options, + CDBopts0 = CDBopts#cdb_options{log_options = leveled_log:get_opts()}, + {noreply, State#state{cdb_options = CDBopts0}}; +handle_cast({remove_logs, ForcedLogs}, State) -> + INC = State#state.clerk, + ok = leveled_iclerk:clerk_removelogs(INC, ForcedLogs), + ok = leveled_log:remove_forcedlogs(ForcedLogs), + CDBopts = State#state.cdb_options, + CDBopts0 = CDBopts#cdb_options{log_options = leveled_log:get_opts()}, + {noreply, State#state{cdb_options = CDBopts0}}. %% handle the bookie stopping and stop this snapshot handle_info({'DOWN', BookieMonRef, process, _BookiePid, _Info}, diff --git a/src/leveled_log.erl b/src/leveled_log.erl index 31213fa..4ad7991 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -15,7 +15,8 @@ add_forcedlogs/1, remove_forcedlogs/1, get_opts/0, - save/1]). + save/1, + return_settings/0]). -record(log_options, {log_level = info :: log_level(), @@ -24,7 +25,7 @@ -type log_level() :: debug | info | warn | error | critical. -type log_options() :: #log_options{}. --export_type([log_options/0]). +-export_type([log_options/0, log_level/0]). -define(LOG_LEVELS, [debug, info, warn, error, critical]). -define(DEFAULT_LOG_LEVEL, error). @@ -442,6 +443,12 @@ get_opts() -> forced_logs = []} end. +-spec return_settings() -> {log_level(), list(string())}. +%% @doc +%% Return the settings outside of the record +return_settings() -> + LO = get_opts(), + {LO#log_options.log_level, LO#log_options.forced_logs}. %%%============================================================================ %%% Prompt Logs diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index ec2482b..5bbbae4 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -39,7 +39,10 @@ clerk_prompt/1, clerk_push/2, clerk_close/1, - clerk_promptdeletions/2 + clerk_promptdeletions/2, + clerk_loglevel/2, + clerk_addlogs/2, + clerk_removelogs/2 ]). -include_lib("eunit/include/eunit.hrl"). @@ -76,6 +79,24 @@ clerk_promptdeletions(Pid, ManifestSQN) -> clerk_push(Pid, Work) -> gen_server:cast(Pid, {push_work, Work}). +-spec clerk_loglevel(pid(), leveled_log:log_level()) -> ok. +%% @doc +%% Change the log level of the Journal +clerk_loglevel(Pid, LogLevel) -> + gen_server:cast(Pid, {log_level, LogLevel}). + +-spec clerk_addlogs(pid(), list(string())) -> ok. +%% @doc +%% Add to the list of forced logs, a list of more forced logs +clerk_addlogs(Pid, ForcedLogs) -> + gen_server:cast(Pid, {add_logs, ForcedLogs}). + +-spec clerk_removelogs(pid(), list(string())) -> ok. +%% @doc +%% Remove from the list of forced logs, a list of forced logs +clerk_removelogs(Pid, ForcedLogs) -> + gen_server:cast(Pid, {remove_logs, ForcedLogs}). + clerk_close(Pid) -> gen_server:call(Pid, close, 20000). @@ -103,7 +124,22 @@ handle_cast({prompt_deletions, ManifestSQN}, State) -> {Deletions, UpdD} = return_deletions(ManifestSQN, State#state.pending_deletions), ok = notify_deletions(Deletions, State#state.owner), - {noreply, State#state{pending_deletions = UpdD}, ?MIN_TIMEOUT}. + {noreply, State#state{pending_deletions = UpdD}, ?MIN_TIMEOUT}; +handle_cast({log_level, LogLevel}, State) -> + ok = leveled_log:set_loglevel(LogLevel), + SSTopts = State#state.sst_options, + SSTopts0 = SSTopts#sst_options{log_options = leveled_log:get_opts()}, + {noreply, State#state{sst_options = SSTopts0}}; +handle_cast({add_logs, ForcedLogs}, State) -> + ok = leveled_log:add_forcedlogs(ForcedLogs), + SSTopts = State#state.sst_options, + SSTopts0 = SSTopts#sst_options{log_options = leveled_log:get_opts()}, + {noreply, State#state{sst_options = SSTopts0}}; +handle_cast({remove_logs, ForcedLogs}, State) -> + ok = leveled_log:remove_forcedlogs(ForcedLogs), + SSTopts = State#state.sst_options, + SSTopts0 = SSTopts#sst_options{log_options = leveled_log:get_opts()}, + {noreply, State#state{sst_options = SSTopts0}}. handle_info(timeout, State) -> request_work(State), diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 607b5a1..c1e9fdc 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -191,7 +191,10 @@ pcl_getstartupsequencenumber/1, pcl_checkbloomtest/2, pcl_checkforwork/1, - pcl_persistedsqn/1]). + pcl_persistedsqn/1, + pcl_loglevel/2, + pcl_addlogs/2, + pcl_removelogs/2]). -export([ sst_rootpath/1, @@ -578,7 +581,6 @@ pcl_close(Pid) -> pcl_doom(Pid) -> gen_server:call(Pid, doom, 60000). - -spec pcl_checkbloomtest(pid(), tuple()) -> boolean(). %% @doc %% Function specifically added to help testing. In particular to make sure @@ -597,6 +599,24 @@ pcl_checkbloomtest(Pid, Key) -> pcl_checkforwork(Pid) -> gen_server:call(Pid, check_for_work, 2000). +-spec pcl_loglevel(pid(), leveled_log:log_level()) -> ok. +%% @doc +%% Change the log level of the Journal +pcl_loglevel(Pid, LogLevel) -> + gen_server:cast(Pid, {log_level, LogLevel}). + +-spec pcl_addlogs(pid(), list(string())) -> ok. +%% @doc +%% Add to the list of forced logs, a list of more forced logs +pcl_addlogs(Pid, ForcedLogs) -> + gen_server:cast(Pid, {add_logs, ForcedLogs}). + +-spec pcl_removelogs(pid(), list(string())) -> ok. +%% @doc +%% Remove from the list of forced logs, a list of forced logs +pcl_removelogs(Pid, ForcedLogs) -> + gen_server:cast(Pid, {remove_logs, ForcedLogs}). + %%%============================================================================ %%% gen_server callbacks %%%============================================================================ @@ -1002,7 +1022,28 @@ handle_cast(work_for_clerk, State) -> end; _ -> {noreply, State} - end. + end; +handle_cast({log_level, LogLevel}, State) -> + PC = State#state.clerk, + ok = leveled_pclerk:clerk_loglevel(PC, LogLevel), + ok = leveled_log:set_loglevel(LogLevel), + SSTopts = State#state.sst_options, + SSTopts0 = SSTopts#sst_options{log_options = leveled_log:get_opts()}, + {noreply, State#state{sst_options = SSTopts0}}; +handle_cast({add_logs, ForcedLogs}, State) -> + PC = State#state.clerk, + ok = leveled_pclerk:clerk_addlogs(PC, ForcedLogs), + ok = leveled_log:add_forcedlogs(ForcedLogs), + SSTopts = State#state.sst_options, + SSTopts0 = SSTopts#sst_options{log_options = leveled_log:get_opts()}, + {noreply, State#state{sst_options = SSTopts0}}; +handle_cast({remove_logs, ForcedLogs}, State) -> + PC = State#state.clerk, + ok = leveled_pclerk:clerk_removelogs(PC, ForcedLogs), + ok = leveled_log:remove_forcedlogs(ForcedLogs), + SSTopts = State#state.sst_options, + SSTopts0 = SSTopts#sst_options{log_options = leveled_log:get_opts()}, + {noreply, State#state{sst_options = SSTopts0}}. %% handle the bookie stopping and stop this snapshot diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 0a3b2b1..b610440 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -41,6 +41,7 @@ simple_put_fetch_head_delete(_Config) -> simple_test_withlog(error, ["B0015", "B0016", "B0017", "B0018", "P0032", "SST12", "CDB19", "SST13", "I0019"]). + simple_test_withlog(LogLevel, ForcedLogs) -> RootPath = testutil:reset_filestructure(), StartOpts1 = [{root_path, RootPath}, @@ -59,6 +60,7 @@ simple_test_withlog(LogLevel, ForcedLogs) -> {log_level, LogLevel}, {forced_logs, ForcedLogs}], {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), + testutil:check_forobject(Bookie2, TestObject), ObjList1 = testutil:generate_objects(5000, 2), testutil:riakload(Bookie2, ObjList1), @@ -106,14 +108,20 @@ many_put_fetch_head(_Config) -> {sync_strategy, testutil:sync_strategy()}, {compression_point, on_receipt}], {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), + ok = leveled_bookie:book_loglevel(Bookie2, error), + ok = leveled_bookie:book_addlogs(Bookie2, ["B0015"]), testutil:check_forobject(Bookie2, TestObject), GenList = [2, 20002, 40002, 60002, 80002, 100002, 120002, 140002, 160002, 180002], CLs = testutil:load_objects(20000, GenList, Bookie2, TestObject, fun testutil:generate_smallobjects/2), + {error, ["B0015"]} = leveled_bookie:book_logsettings(Bookie2), + ok = leveled_bookie:book_removelogs(Bookie2, ["B0015"]), CL1A = lists:nth(1, CLs), ChkListFixed = lists:nth(length(CLs), CLs), testutil:check_forlist(Bookie2, CL1A), + {error, []} = leveled_bookie:book_logsettings(Bookie2), + ok = leveled_bookie:book_loglevel(Bookie2, info), ObjList2A = testutil:generate_objects(5000, 2), testutil:riakload(Bookie2, ObjList2A), ChkList2A = lists:sublist(lists:sort(ObjList2A), 1000),