Mas d31 i416 (#418)
* Add compression controls (#417) * Add compression controls Add configuration options to allow for a compression algorithm of `none` to disable compression altogether. Also an option to change the point in the LSM tree when compression is applied. * Handle configurable defaults consistently Move them into leveled.hrl. This forces double-definitions to be resolved. There are some other constants in leveled_bookie that are relevant outside of leveled_bookie. These are all now in the non-configurable startup defaults section. * Clarify referred-to default is OTP not leveled * Update leveled_bookie.erl Handle xref issue with eunit include
This commit is contained in:
Martin Sumner
GitHub
parent
b96518c32a
commit
9e804924a8
11 changed files with 338 additions and 163 deletions
|
|
@ -1,34 +1,79 @@
|
|||
% File paths
|
||||
%%%============================================================================
|
||||
%%% File paths
|
||||
%%%============================================================================
|
||||
-define(JOURNAL_FP, "journal").
|
||||
-define(LEDGER_FP, "ledger").
|
||||
%%%============================================================================
|
||||
|
||||
%%%============================================================================
|
||||
%%% Configurable startup defaults
|
||||
%%%============================================================================
|
||||
-define(CACHE_SIZE, 2500).
|
||||
-define(MAX_CACHE_MULTTIPLE, 2).
|
||||
-define(MIN_CACHE_SIZE, 100).
|
||||
-define(MIN_PCL_CACHE_SIZE, 400).
|
||||
-define(MAX_PCL_CACHE_SIZE, 28000).
|
||||
% This is less than actual max - but COIN_SIDECOUNT
|
||||
-define(ABSOLUTEMAX_JOURNALSIZE, 4000000000).
|
||||
-define(COMPRESSION_METHOD, lz4).
|
||||
-define(COMPRESSION_POINT, on_receipt).
|
||||
-define(COMPRESSION_LEVEL, 1).
|
||||
-define(LOG_LEVEL, info).
|
||||
-define(DEFAULT_DBID, 65536).
|
||||
-define(OPEN_LASTMOD_RANGE, {0, infinity}).
|
||||
-define(SNAPTIMEOUT_SHORT, 900). % 15 minutes
|
||||
-define(SNAPTIMEOUT_LONG, 43200). % 12 hours
|
||||
-define(SST_PAGECACHELEVEL_NOLOOKUP, 1).
|
||||
-define(SST_PAGECACHELEVEL_LOOKUP, 4).
|
||||
-define(DEFAULT_STATS_PERC, 10).
|
||||
-define(DEFAULT_SYNC_STRATEGY, none).
|
||||
%%%============================================================================
|
||||
|
||||
%%%============================================================================
|
||||
%%% Non-configurable startup defaults
|
||||
%%%============================================================================
|
||||
-define(MAX_SSTSLOTS, 256).
|
||||
-define(LOADING_PAUSE, 1000).
|
||||
-define(LOADING_BATCH, 1000).
|
||||
|
||||
%% Tag to be used on standard Riak KV objects
|
||||
-define(RIAK_TAG, o_rkv).
|
||||
%% Tag to be used on K/V objects for non-Riak purposes
|
||||
-define(STD_TAG, o).
|
||||
%% Tag used for secondary index keys
|
||||
-define(IDX_TAG, i).
|
||||
%% Tag used for head-only objects
|
||||
-define(HEAD_TAG, h).
|
||||
|
||||
%% Inker key type used for 'normal' objects
|
||||
-define(INKT_STND, stnd).
|
||||
%% Inker key type used for 'batch' objects
|
||||
-define(INKT_MPUT, mput).
|
||||
%% Inker key type used for objects which contain no value, only key changes
|
||||
%% This is used currently for objects formed under a 'retain' strategy on Inker
|
||||
%% compaction
|
||||
-define(INKT_KEYD, keyd).
|
||||
%% Inker key type used for tombstones
|
||||
-define(INKT_TOMB, tomb).
|
||||
|
||||
-define(CACHE_TYPE, skpl).
|
||||
|
||||
-define(CACHE_SIZE_JITTER, 25).
|
||||
-define(JOURNAL_SIZE_JITTER, 20).
|
||||
-define(LONG_RUNNING, 1000000).
|
||||
% An individual task taking > 1s gets a specific log
|
||||
-define(MAX_KEYCHECK_FREQUENCY, 100).
|
||||
-define(MIN_KEYCHECK_FREQUENCY, 1).
|
||||
-define(MAX_LEVELS, 8).
|
||||
%% Should equal the length of the LEVEL_SCALEFACTOR
|
||||
%% Should equal the length of the LEVEL_SCALEFACTOR
|
||||
-define(CACHE_TYPE, skpl).
|
||||
%%%============================================================================
|
||||
|
||||
%%%============================================================================
|
||||
%%% Tags
|
||||
%%%============================================================================
|
||||
-define(RIAK_TAG, o_rkv).
|
||||
%% Tag to be used on standard Riak KV objects
|
||||
-define(STD_TAG, o).
|
||||
%% Tag to be used on K/V objects for non-Riak purposes
|
||||
-define(IDX_TAG, i).
|
||||
%% Tag used for secondary index keys
|
||||
-define(HEAD_TAG, h).
|
||||
%% Tag used for head-only objects
|
||||
|
||||
-define(INKT_STND, stnd).
|
||||
%% Inker key type used for 'normal' objects
|
||||
-define(INKT_MPUT, mput).
|
||||
%% Inker key type used for 'batch' objects
|
||||
-define(INKT_KEYD, keyd).
|
||||
%% Inker key type used for objects which contain no value, only key changes
|
||||
%% This is used currently for objects formed under a 'retain' strategy
|
||||
%% on Inker compaction
|
||||
-define(INKT_TOMB, tomb).
|
||||
%% Inker key type used for tombstones
|
||||
%%%============================================================================
|
||||
|
||||
|
||||
%%%============================================================================
|
||||
%%% Shared records
|
||||
%%%============================================================================
|
||||
-record(level,
|
||||
{level :: integer(),
|
||||
is_basement = false :: boolean(),
|
||||
|
|
@ -47,19 +92,26 @@
|
|||
file_path :: string() | undefined,
|
||||
waste_path :: string() | undefined,
|
||||
binary_mode = false :: boolean(),
|
||||
sync_strategy = sync,
|
||||
% Default set by bookie to be `true`
|
||||
% `false` set here due to legacy of unit tests
|
||||
% using non-binary keys
|
||||
sync_strategy = ?DEFAULT_SYNC_STRATEGY,
|
||||
log_options = leveled_log:get_opts()
|
||||
:: leveled_log:log_options(),
|
||||
monitor = {no_monitor, 0} :: leveled_monitor:monitor()}).
|
||||
monitor = {no_monitor, 0}
|
||||
:: leveled_monitor:monitor()}).
|
||||
|
||||
-record(sst_options,
|
||||
{press_method = native
|
||||
{press_method = ?COMPRESSION_METHOD
|
||||
:: leveled_sst:press_method(),
|
||||
press_level = ?COMPRESSION_LEVEL :: non_neg_integer(),
|
||||
log_options = leveled_log:get_opts()
|
||||
:: leveled_log:log_options(),
|
||||
max_sstslots = 256 :: pos_integer(),
|
||||
pagecache_level = 1 :: pos_integer(),
|
||||
monitor = {no_monitor, 0} :: leveled_monitor:monitor()}).
|
||||
max_sstslots = ?MAX_SSTSLOTS :: pos_integer(),
|
||||
pagecache_level = ?SST_PAGECACHELEVEL_NOLOOKUP
|
||||
:: pos_integer(),
|
||||
monitor = {no_monitor, 0}
|
||||
:: leveled_monitor:monitor()}).
|
||||
|
||||
-record(inker_options,
|
||||
{cdb_max_size :: integer() | undefined,
|
||||
|
|
@ -70,14 +122,16 @@
|
|||
source_inker :: pid() | undefined,
|
||||
reload_strategy = [] :: list(),
|
||||
waste_retention_period :: integer() | undefined,
|
||||
compression_method = native :: lz4|native,
|
||||
compression_method = ?COMPRESSION_METHOD
|
||||
:: lz4|native|none,
|
||||
compress_on_receipt = false :: boolean(),
|
||||
max_run_length,
|
||||
singlefile_compactionperc :: float()|undefined,
|
||||
maxrunlength_compactionperc :: float()|undefined,
|
||||
score_onein = 1 :: pos_integer(),
|
||||
snaptimeout_long :: pos_integer() | undefined,
|
||||
monitor = {no_monitor, 0} :: leveled_monitor:monitor()}).
|
||||
monitor = {no_monitor, 0}
|
||||
:: leveled_monitor:monitor()}).
|
||||
|
||||
-record(penciller_options,
|
||||
{root_path :: string() | undefined,
|
||||
|
|
@ -89,19 +143,23 @@
|
|||
bookies_mem :: tuple() | undefined,
|
||||
source_penciller :: pid() | undefined,
|
||||
snapshot_longrunning = true :: boolean(),
|
||||
compression_method = native :: lz4|native,
|
||||
compression_method = ?COMPRESSION_METHOD
|
||||
:: lz4|native|none,
|
||||
levelzero_cointoss = false :: boolean(),
|
||||
snaptimeout_short :: pos_integer() | undefined,
|
||||
snaptimeout_long :: pos_integer() | undefined,
|
||||
monitor = {no_monitor, 0} :: leveled_monitor:monitor()}).
|
||||
monitor = {no_monitor, 0}
|
||||
:: leveled_monitor:monitor()}).
|
||||
|
||||
-record(iclerk_options,
|
||||
{inker :: pid() | undefined,
|
||||
max_run_length :: integer() | undefined,
|
||||
cdb_options = #cdb_options{} :: #cdb_options{},
|
||||
waste_retention_period :: integer() | undefined,
|
||||
compression_method = native :: lz4|native,
|
||||
compression_method = ?COMPRESSION_METHOD
|
||||
:: lz4|native|none,
|
||||
singlefile_compactionperc :: float()|undefined,
|
||||
maxrunlength_compactionperc :: float()|undefined,
|
||||
score_onein = 1 :: pos_integer(),
|
||||
reload_strategy = [] :: list()}).
|
||||
%%%============================================================================
|
||||
Loading…
Add table
Add a link
Reference in a new issue