From 534bf87a51aba52ddeea641ace53c9eb8f484aec Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 14 Sep 2018 10:19:25 +0100 Subject: [PATCH 1/3] Set minimum cache sizes This avoids a divide by zero (if the Bookie's memory cache size is smaller than 4). --- src/leveled_bookie.erl | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index c377393..a307084 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -91,6 +91,8 @@ -include_lib("eunit/include/eunit.hrl"). -define(CACHE_SIZE, 2500). +-define(MIN_CACHE_SIZE, 100). +-define(MIN_PCL_CACHE_SIZE, 400). -define(SNAPSHOT_TIMEOUT, 300000). -define(CACHE_SIZE_JITTER, 25). -define(JOURNAL_SIZE_JITTER, 20). @@ -196,6 +198,7 @@ % additions to the ledger. Defaults to ?CACHE_SIZE, plus some % randomised jitter (randomised jitter will still be added to % configured values + % The minimum value is 100 - any lower value will be ignored {max_journalsize, pos_integer()} | % The maximum size of a journal file in bytes. The abolute % maximum must be 4GB due to 4 byte file pointers being used @@ -280,7 +283,10 @@ {max_pencillercachesize, pos_integer()|undefined} | % How many ledger keys should the penciller retain in memory % between flushing new level zero files. - % Defaults ot leveled_penciller:?MAX_TABLESIZE when undefined + % Defaults to leveled_penciller:?MAX_TABLESIZE when undefined + % The minimum size 400 - attempt to set this vlaue lower will be + % ignored. As a rule the value should be at least 4 x the Bookie's + % cache size {compression_method, native|lz4} | % Compression method and point allow Leveled to be switched from % using bif based compression (zlib) to using nif based compression @@ -926,12 +932,12 @@ init([Opts]) -> % Start from file not snapshot {InkerOpts, PencillerOpts} = set_options(Opts), + ConfiguredCacheSize = + max(proplists:get_value(cache_size, Opts), ?MIN_CACHE_SIZE), CacheJitter = - proplists:get_value(cache_size, Opts) - div (100 div ?CACHE_SIZE_JITTER), + ConfiguredCacheSize div (100 div ?CACHE_SIZE_JITTER), CacheSize = - proplists:get_value(cache_size, Opts) - + erlang:phash2(self()) rem CacheJitter, + ConfiguredCacheSize + erlang:phash2(self()) rem CacheJitter, RecentAAE = case proplists:get_value(recent_aae, Opts) of false -> @@ -1350,7 +1356,9 @@ set_options(Opts) -> AltStrategy = proplists:get_value(reload_strategy, Opts), ReloadStrategy = leveled_codec:inker_reload_strategy(AltStrategy), - PCLL0CacheSize = proplists:get_value(max_pencillercachesize, Opts), + PCLL0CacheSize = + max(?MIN_PCL_CACHE_SIZE, + proplists:get_value(max_pencillercachesize, Opts)), RootPath = proplists:get_value(root_path, Opts), JournalFP = filename:join(RootPath, ?JOURNAL_FP), @@ -2489,6 +2497,13 @@ folder_cache_test(CacheSize) -> ok = book_close(Bookie1), reset_filestructure(). +small_cachesize_test() -> + RootPath = reset_filestructure(), + {ok, Bookie1} = book_start([{root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 1}]), + ok = leveled_bookie:book_close(Bookie1). + is_empty_test() -> RootPath = reset_filestructure(), {ok, Bookie1} = book_start([{root_path, RootPath}, From e18d6810434d2368bd3955cfa0786eecae4b6a80 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 14 Sep 2018 10:36:10 +0100 Subject: [PATCH 2/3] Need to support undefined max penciller cache size Without first converting it to the minimum. So catch the undefined, at set_default --- src/leveled_bookie.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index a307084..3f1f511 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -93,6 +93,8 @@ -define(CACHE_SIZE, 2500). -define(MIN_CACHE_SIZE, 100). -define(MIN_PCL_CACHE_SIZE, 400). +-define(MAX_PCL_CACHE_SIZE, 28000). + % This is less than actual max - but COIN_SIDECOUNT -define(SNAPSHOT_TIMEOUT, 300000). -define(CACHE_SIZE_JITTER, 25). -define(JOURNAL_SIZE_JITTER, 20). @@ -117,7 +119,7 @@ {singlefile_compactionpercentage, 50.0}, {maxrunlength_compactionpercentage, 70.0}, {reload_strategy, []}, - {max_pencillercachesize, undefined}, + {max_pencillercachesize, ?MAX_PCL_CACHE_SIZE}, {compression_method, ?COMPRESSION_METHOD}, {compression_point, ?COMPRESSION_POINT}]). @@ -283,7 +285,7 @@ {max_pencillercachesize, pos_integer()|undefined} | % How many ledger keys should the penciller retain in memory % between flushing new level zero files. - % Defaults to leveled_penciller:?MAX_TABLESIZE when undefined + % Defaults to ?MAX_PCL_CACHE_SIZE when undefined % The minimum size 400 - attempt to set this vlaue lower will be % ignored. As a rule the value should be at least 4 x the Bookie's % cache size @@ -2504,6 +2506,7 @@ small_cachesize_test() -> {cache_size, 1}]), ok = leveled_bookie:book_close(Bookie1). + is_empty_test() -> RootPath = reset_filestructure(), {ok, Bookie1} = book_start([{root_path, RootPath}, From 8ada5e78fa634d88b212bf325c60b6cd9ba3cdad Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 14 Sep 2018 17:22:25 +0100 Subject: [PATCH 3/3] Max penciller cache change Missed a bit --- src/leveled_penciller.erl | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index c7e8b10..776da87 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -226,7 +226,6 @@ -define(SST_FILEX, ".sst"). -define(ARCHIVE_FILEX, ".bak"). -define(MEMTABLE, mem). --define(MAX_TABLESIZE, 28000). % This is less than max - but COIN_SIDECOUNT -define(SUPER_MAX_TABLE_SIZE, 40000). -define(PROMPT_WAIT_ONL0, 5). -define(WORKQUEUE_BACKLOG_TOLERANCE, 4). @@ -991,13 +990,7 @@ sst_filename(ManSQN, Level, Count) -> %% filesystem and reconstruct the ledger from the files that it finds start_from_file(PCLopts) -> RootPath = PCLopts#penciller_options.root_path, - MaxTableSize = - case PCLopts#penciller_options.max_inmemory_tablesize of - undefined -> - ?MAX_TABLESIZE; - M when is_integer(M) -> - M - end, + MaxTableSize = PCLopts#penciller_options.max_inmemory_tablesize, PressMethod = PCLopts#penciller_options.compression_method, {ok, MergeClerk} = leveled_pclerk:clerk_new(self(), RootPath, PressMethod),