Allow for run-time changes to log_level and forced_logs

Will not lead to immediate run time changes in SST or CDB logs.  These log settings will only change once the new files are re-written.

To completely change the log level - a restart of the store is necessary with new startup options.

src/leveled_bookie.erl

@@ -67,7 +67,11 @@
         book_hotbackup/1,
         book_close/1,
         book_destroy/1,
-        book_isempty/2]).
+        book_isempty/2,
+        book_logsettings/1,
+        book_loglevel/2,
+        book_addlogs/2,
+        book_removelogs/2]).
 
 %% folding API
 -export([
@@ -1047,6 +1051,31 @@ book_isempty(Pid, Tag) ->
     {async, Runner} = book_bucketlist(Pid, Tag, FoldAccT, first),
     Runner().
 
+-spec book_logsettings(pid()) -> {leveled_log:log_level(), list(string())}.
+%% @doc
+%% Retrieve the current log settings
+book_logsettings(Pid) ->
+    gen_server:call(Pid, log_settings, infinity).
+
+-spec book_loglevel(pid(), leveled_log:log_level()) -> ok.
+%% @doc
+%% Change the log level of the store
+book_loglevel(Pid, LogLevel) ->
+    gen_server:cast(Pid, {log_level, LogLevel}).
+
+-spec book_addlogs(pid(), list(string())) -> ok.
+%% @doc
+%% Add to the list of forced logs, a list of more forced logs
+book_addlogs(Pid, ForcedLogs) ->
+    gen_server:cast(Pid, {add_logs, ForcedLogs}).
+
+-spec book_removelogs(pid(), list(string())) -> ok.
+%% @doc
+%% Remove from the list of forced logs, a list of forced logs
+book_removelogs(Pid, ForcedLogs) ->
+    gen_server:cast(Pid, {remove_logs, ForcedLogs}).
+
+
 %%%============================================================================
 %%% gen_server callbacks
 %%%============================================================================
@@ -1302,6 +1331,8 @@ handle_call({snapshot, SnapType, Query, LongRunning}, _From, State) ->
     % e.g. many minutes)
     Reply = snapshot_store(State, SnapType, Query, LongRunning),
     {reply, Reply, State};
+handle_call(log_settings, _From, State) ->
+    {reply, leveled_log:return_settings(), State};
 handle_call({return_runner, QueryType}, _From, State) ->
     SW = os:timestamp(),
     Runner = get_runner(State, QueryType),
@@ -1352,9 +1383,30 @@ handle_call(destroy, _From, State=#state{is_snapshot=Snp}) when Snp == false ->
 handle_call(Msg, _From, State) ->
     {reply, {unsupported_message, element(1, Msg)}, State}.
 
-handle_cast(_Msg, State) ->
+
+handle_cast({log_level, LogLevel}, State) ->
+    PCL = State#state.penciller,
+    INK = State#state.inker,
+    ok = leveled_penciller:pcl_loglevel(PCL, LogLevel),
+    ok = leveled_inker:ink_loglevel(INK, LogLevel),
+    ok = leveled_log:set_loglevel(LogLevel),
+    {noreply, State};
+handle_cast({add_logs, ForcedLogs}, State) ->
+    PCL = State#state.penciller,
+    INK = State#state.inker,
+    ok = leveled_penciller:pcl_addlogs(PCL, ForcedLogs),
+    ok = leveled_inker:ink_addlogs(INK, ForcedLogs),
+    ok = leveled_log:add_forcedlogs(ForcedLogs),
+    {noreply, State};
+handle_cast({remove_logs, ForcedLogs}, State) ->
+    PCL = State#state.penciller,
+    INK = State#state.inker,
+    ok = leveled_penciller:pcl_removelogs(PCL, ForcedLogs),
+    ok = leveled_inker:ink_removelogs(INK, ForcedLogs),
+    ok = leveled_log:remove_forcedlogs(ForcedLogs),
     {noreply, State}.
 
+
 handle_info(_Info, State) ->
     {noreply, State}.
 
@@ -2879,8 +2931,7 @@ longrunning_test() ->
 
 coverage_cheat_test() ->
     {noreply, _State0} = handle_info(timeout, #state{}),
-    {ok, _State1} = code_change(null, #state{}, null),
-    {noreply, _State2} = handle_cast(null, #state{}).
+    {ok, _State1} = code_change(null, #state{}, null).
 
 erase_journal_test() ->
     RootPath = reset_filestructure(),

src/leveled_iclerk.erl

@@ -79,12 +79,16 @@
         handle_cast/2,
         handle_info/2,
         terminate/2,
-        clerk_new/1,
+        code_change/3]).
+
+-export([clerk_new/1,
         clerk_compact/7,
         clerk_hashtablecalc/3,
         clerk_trim/3,
         clerk_stop/1,
-        code_change/3]).      
+        clerk_loglevel/2,
+        clerk_addlogs/2,
+        clerk_removelogs/2]).   
 
 -export([schedule_compaction/3]).
 
@@ -104,7 +108,7 @@
 
 -record(state, {inker :: pid() | undefined,
                 max_run_length :: integer() | undefined,
-                cdb_options,
+                cdb_options = #cdb_options{} :: #cdb_options{},
                 waste_retention_period :: integer() | undefined,
                 waste_path :: string() | undefined,
                 reload_strategy = ?DEFAULT_RELOAD_STRATEGY :: list(),
@@ -180,6 +184,24 @@ clerk_hashtablecalc(HashTree, StartPos, CDBpid) ->
 clerk_stop(Pid) ->
     gen_server:cast(Pid, stop).
 
+-spec clerk_loglevel(pid(), leveled_log:log_level()) -> ok.
+%% @doc
+%% Change the log level of the Journal
+clerk_loglevel(Pid, LogLevel) ->
+    gen_server:cast(Pid, {log_level, LogLevel}).
+
+-spec clerk_addlogs(pid(), list(string())) -> ok.
+%% @doc
+%% Add to the list of forced logs, a list of more forced logs
+clerk_addlogs(Pid, ForcedLogs) ->
+    gen_server:cast(Pid, {add_logs, ForcedLogs}).
+
+-spec clerk_removelogs(pid(), list(string())) -> ok.
+%% @doc
+%% Remove from the list of forced logs, a list of forced logs
+clerk_removelogs(Pid, ForcedLogs) ->
+    gen_server:cast(Pid, {remove_logs, ForcedLogs}).
+
 %%%============================================================================
 %%% gen_server callbacks
 %%%============================================================================
@@ -292,6 +314,21 @@ handle_cast({hashtable_calc, HashTree, StartPos, CDBpid}, State) ->
     {IndexList, HashTreeBin} = leveled_cdb:hashtable_calc(HashTree, StartPos),
     ok = leveled_cdb:cdb_returnhashtable(CDBpid, IndexList, HashTreeBin),
     {stop, normal, State};
+handle_cast({log_level, LogLevel}, State) ->
+    ok = leveled_log:set_loglevel(LogLevel),
+    CDBopts = State#state.cdb_options,
+    CDBopts0 = CDBopts#cdb_options{log_options = leveled_log:get_opts()},
+    {noreply, State#state{cdb_options = CDBopts0}};
+handle_cast({add_logs, ForcedLogs}, State) ->
+    ok = leveled_log:add_forcedlogs(ForcedLogs),
+    CDBopts = State#state.cdb_options,
+    CDBopts0 = CDBopts#cdb_options{log_options = leveled_log:get_opts()},
+    {noreply, State#state{cdb_options = CDBopts0}};
+handle_cast({remove_logs, ForcedLogs}, State) ->
+    ok = leveled_log:remove_forcedlogs(ForcedLogs),
+    CDBopts = State#state.cdb_options,
+    CDBopts0 = CDBopts#cdb_options{log_options = leveled_log:get_opts()},
+    {noreply, State#state{cdb_options = CDBopts0}};
 handle_cast(stop, State) ->
     {stop, normal, State}.
 

src/leveled_inker.erl

@@ -116,7 +116,11 @@
         ink_roll/1,
         ink_backup/2,
         ink_checksqn/2,
-        build_dummy_journal/0,
+        ink_loglevel/2,
+        ink_addlogs/2,
+        ink_removelogs/2]).
+
+-export([build_dummy_journal/0,
         clean_testdir/1,
         filepath/2,
         filepath/3]).
@@ -447,6 +451,24 @@ ink_printmanifest(Pid) ->
 ink_checksqn(Pid, LedgerSQN) ->
     gen_server:call(Pid, {check_sqn, LedgerSQN}).
 
+-spec ink_loglevel(pid(), leveled_log:log_level()) -> ok.
+%% @doc
+%% Change the log level of the Journal
+ink_loglevel(Pid, LogLevel) ->
+    gen_server:cast(Pid, {log_level, LogLevel}).
+
+-spec ink_addlogs(pid(), list(string())) -> ok.
+%% @doc
+%% Add to the list of forced logs, a list of more forced logs
+ink_addlogs(Pid, ForcedLogs) ->
+    gen_server:cast(Pid, {add_logs, ForcedLogs}).
+
+-spec ink_removelogs(pid(), list(string())) -> ok.
+%% @doc
+%% Remove from the list of forced logs, a list of forced logs
+ink_removelogs(Pid, ForcedLogs) ->
+    gen_server:cast(Pid, {remove_logs, ForcedLogs}).
+
 %%%============================================================================
 %%% gen_server callbacks
 %%%============================================================================
@@ -698,7 +720,28 @@ handle_cast({release_snapshot, Snapshot}, State) ->
     Rs = lists:keydelete(Snapshot, 1, State#state.registered_snapshots),
     leveled_log:log("I0003", [Snapshot]),
     leveled_log:log("I0004", [length(Rs)]),
-    {noreply, State#state{registered_snapshots=Rs}}.
+    {noreply, State#state{registered_snapshots=Rs}};
+handle_cast({log_level, LogLevel}, State) ->
+    INC = State#state.clerk,
+    ok = leveled_iclerk:clerk_loglevel(INC, LogLevel),
+    ok = leveled_log:set_loglevel(LogLevel),
+    CDBopts = State#state.cdb_options,
+    CDBopts0 = CDBopts#cdb_options{log_options = leveled_log:get_opts()},
+    {noreply, State#state{cdb_options = CDBopts0}};
+handle_cast({add_logs, ForcedLogs}, State) ->
+    INC = State#state.clerk,
+    ok = leveled_iclerk:clerk_addlogs(INC, ForcedLogs),
+    ok = leveled_log:add_forcedlogs(ForcedLogs),
+    CDBopts = State#state.cdb_options,
+    CDBopts0 = CDBopts#cdb_options{log_options = leveled_log:get_opts()},
+    {noreply, State#state{cdb_options = CDBopts0}};
+handle_cast({remove_logs, ForcedLogs}, State) ->
+    INC = State#state.clerk,
+    ok = leveled_iclerk:clerk_removelogs(INC, ForcedLogs),
+    ok = leveled_log:remove_forcedlogs(ForcedLogs),
+    CDBopts = State#state.cdb_options,
+    CDBopts0 = CDBopts#cdb_options{log_options = leveled_log:get_opts()},
+    {noreply, State#state{cdb_options = CDBopts0}}.
 
 %% handle the bookie stopping and stop this snapshot
 handle_info({'DOWN', BookieMonRef, process, _BookiePid, _Info},

src/leveled_log.erl

@@ -15,7 +15,8 @@
             add_forcedlogs/1,
             remove_forcedlogs/1,
             get_opts/0,
-            save/1]).
+            save/1,
+            return_settings/0]).
 
 
 -record(log_options, {log_level = info :: log_level(), 
@@ -24,7 +25,7 @@
 -type log_level()  ::  debug | info | warn | error | critical.
 -type log_options() :: #log_options{}.
 
--export_type([log_options/0]).
+-export_type([log_options/0, log_level/0]).
 
 -define(LOG_LEVELS, [debug, info, warn, error, critical]).
 -define(DEFAULT_LOG_LEVEL, error).
@@ -442,6 +443,12 @@ get_opts() ->
                          forced_logs = []}
     end.
 
+-spec return_settings() -> {log_level(), list(string())}.
+%% @doc
+%% Return the settings outside of the record
+return_settings() ->
+    LO = get_opts(),
+    {LO#log_options.log_level, LO#log_options.forced_logs}.
 
 %%%============================================================================
 %%% Prompt Logs

src/leveled_pclerk.erl

@@ -39,7 +39,10 @@
         clerk_prompt/1,
         clerk_push/2,
         clerk_close/1,
-        clerk_promptdeletions/2
+        clerk_promptdeletions/2,
+        clerk_loglevel/2,
+        clerk_addlogs/2,
+        clerk_removelogs/2
         ]).      
 
 -include_lib("eunit/include/eunit.hrl").
@@ -76,6 +79,24 @@ clerk_promptdeletions(Pid, ManifestSQN) ->
 clerk_push(Pid, Work) ->
     gen_server:cast(Pid, {push_work, Work}).
 
+-spec clerk_loglevel(pid(), leveled_log:log_level()) -> ok.
+%% @doc
+%% Change the log level of the Journal
+clerk_loglevel(Pid, LogLevel) ->
+    gen_server:cast(Pid, {log_level, LogLevel}).
+
+-spec clerk_addlogs(pid(), list(string())) -> ok.
+%% @doc
+%% Add to the list of forced logs, a list of more forced logs
+clerk_addlogs(Pid, ForcedLogs) ->
+    gen_server:cast(Pid, {add_logs, ForcedLogs}).
+
+-spec clerk_removelogs(pid(), list(string())) -> ok.
+%% @doc
+%% Remove from the list of forced logs, a list of forced logs
+clerk_removelogs(Pid, ForcedLogs) ->
+    gen_server:cast(Pid, {remove_logs, ForcedLogs}).
+
 clerk_close(Pid) ->
     gen_server:call(Pid, close, 20000).
 
@@ -103,7 +124,22 @@ handle_cast({prompt_deletions, ManifestSQN}, State) ->
     {Deletions, UpdD} = return_deletions(ManifestSQN,
                                             State#state.pending_deletions),
     ok = notify_deletions(Deletions, State#state.owner),
-    {noreply, State#state{pending_deletions = UpdD}, ?MIN_TIMEOUT}.
+    {noreply, State#state{pending_deletions = UpdD}, ?MIN_TIMEOUT};
+handle_cast({log_level, LogLevel}, State) ->
+    ok = leveled_log:set_loglevel(LogLevel),
+    SSTopts = State#state.sst_options,
+    SSTopts0 = SSTopts#sst_options{log_options = leveled_log:get_opts()},
+    {noreply, State#state{sst_options = SSTopts0}};
+handle_cast({add_logs, ForcedLogs}, State) ->
+    ok = leveled_log:add_forcedlogs(ForcedLogs),
+    SSTopts = State#state.sst_options,
+    SSTopts0 = SSTopts#sst_options{log_options = leveled_log:get_opts()},
+    {noreply, State#state{sst_options = SSTopts0}};
+handle_cast({remove_logs, ForcedLogs}, State) ->
+    ok = leveled_log:remove_forcedlogs(ForcedLogs),
+    SSTopts = State#state.sst_options,
+    SSTopts0 = SSTopts#sst_options{log_options = leveled_log:get_opts()},
+    {noreply, State#state{sst_options = SSTopts0}}.
 
 handle_info(timeout, State) ->
     request_work(State),

src/leveled_penciller.erl

@@ -191,7 +191,10 @@
         pcl_getstartupsequencenumber/1,
         pcl_checkbloomtest/2,
         pcl_checkforwork/1,
-        pcl_persistedsqn/1]).
+        pcl_persistedsqn/1,
+        pcl_loglevel/2,
+        pcl_addlogs/2,
+        pcl_removelogs/2]).
 
 -export([
         sst_rootpath/1,
@@ -578,7 +581,6 @@ pcl_close(Pid) ->
 pcl_doom(Pid) ->
     gen_server:call(Pid, doom, 60000).
 
-
 -spec pcl_checkbloomtest(pid(), tuple()) -> boolean().
 %% @doc
 %% Function specifically added to help testing.  In particular to make sure 
@@ -597,6 +599,24 @@ pcl_checkbloomtest(Pid, Key) ->
 pcl_checkforwork(Pid) ->
     gen_server:call(Pid, check_for_work, 2000).
 
+-spec pcl_loglevel(pid(), leveled_log:log_level()) -> ok.
+%% @doc
+%% Change the log level of the Journal
+pcl_loglevel(Pid, LogLevel) ->
+    gen_server:cast(Pid, {log_level, LogLevel}).
+
+-spec pcl_addlogs(pid(), list(string())) -> ok.
+%% @doc
+%% Add to the list of forced logs, a list of more forced logs
+pcl_addlogs(Pid, ForcedLogs) ->
+    gen_server:cast(Pid, {add_logs, ForcedLogs}).
+
+-spec pcl_removelogs(pid(), list(string())) -> ok.
+%% @doc
+%% Remove from the list of forced logs, a list of forced logs
+pcl_removelogs(Pid, ForcedLogs) ->
+    gen_server:cast(Pid, {remove_logs, ForcedLogs}).
+
 %%%============================================================================
 %%% gen_server callbacks
 %%%============================================================================
@@ -1002,7 +1022,28 @@ handle_cast(work_for_clerk, State) ->
             end;
         _ ->
             {noreply, State}
-    end.
+    end;
+handle_cast({log_level, LogLevel}, State) ->
+    PC = State#state.clerk,
+    ok = leveled_pclerk:clerk_loglevel(PC, LogLevel),
+    ok = leveled_log:set_loglevel(LogLevel),
+    SSTopts = State#state.sst_options,
+    SSTopts0 = SSTopts#sst_options{log_options = leveled_log:get_opts()},
+    {noreply, State#state{sst_options = SSTopts0}};
+handle_cast({add_logs, ForcedLogs}, State) ->
+    PC = State#state.clerk,
+    ok = leveled_pclerk:clerk_addlogs(PC, ForcedLogs),
+    ok = leveled_log:add_forcedlogs(ForcedLogs),
+    SSTopts = State#state.sst_options,
+    SSTopts0 = SSTopts#sst_options{log_options = leveled_log:get_opts()},
+    {noreply, State#state{sst_options = SSTopts0}};
+handle_cast({remove_logs, ForcedLogs}, State) ->
+    PC = State#state.clerk,
+    ok = leveled_pclerk:clerk_removelogs(PC, ForcedLogs),
+    ok = leveled_log:remove_forcedlogs(ForcedLogs),
+    SSTopts = State#state.sst_options,
+    SSTopts0 = SSTopts#sst_options{log_options = leveled_log:get_opts()},
+    {noreply, State#state{sst_options = SSTopts0}}.
 
 
 %% handle the bookie stopping and stop this snapshot

test/end_to_end/basic_SUITE.erl

@@ -41,6 +41,7 @@ simple_put_fetch_head_delete(_Config) ->
     simple_test_withlog(error, ["B0015", "B0016", "B0017", "B0018", 
                                 "P0032", "SST12", "CDB19", "SST13", "I0019"]).
 
+
 simple_test_withlog(LogLevel, ForcedLogs) ->
     RootPath = testutil:reset_filestructure(),
     StartOpts1 = [{root_path, RootPath},
@@ -59,6 +60,7 @@ simple_test_withlog(LogLevel, ForcedLogs) ->
                     {log_level, LogLevel},
                     {forced_logs, ForcedLogs}],
     {ok, Bookie2} = leveled_bookie:book_start(StartOpts2),
+    
     testutil:check_forobject(Bookie2, TestObject),
     ObjList1 = testutil:generate_objects(5000, 2),
     testutil:riakload(Bookie2, ObjList1),
@@ -106,14 +108,20 @@ many_put_fetch_head(_Config) ->
                     {sync_strategy, testutil:sync_strategy()},
                     {compression_point, on_receipt}],
     {ok, Bookie2} = leveled_bookie:book_start(StartOpts2),
+    ok = leveled_bookie:book_loglevel(Bookie2, error),
+    ok = leveled_bookie:book_addlogs(Bookie2, ["B0015"]),
     testutil:check_forobject(Bookie2, TestObject),
     GenList = [2, 20002, 40002, 60002, 80002,
                 100002, 120002, 140002, 160002, 180002],
     CLs = testutil:load_objects(20000, GenList, Bookie2, TestObject,
                                 fun testutil:generate_smallobjects/2),
+    {error, ["B0015"]} = leveled_bookie:book_logsettings(Bookie2),
+    ok = leveled_bookie:book_removelogs(Bookie2, ["B0015"]),
     CL1A = lists:nth(1, CLs),
     ChkListFixed = lists:nth(length(CLs), CLs),
     testutil:check_forlist(Bookie2, CL1A),
+    {error, []} = leveled_bookie:book_logsettings(Bookie2),
+    ok = leveled_bookie:book_loglevel(Bookie2, info),
     ObjList2A = testutil:generate_objects(5000, 2),
     testutil:riakload(Bookie2, ObjList2A),
     ChkList2A = lists:sublist(lists:sort(ObjList2A), 1000),