
* Add test to replicate issue 459 Nothing actually crashes due to the issue - but looking at the logs there is the polarised stats associated with the issue. When merging into L3, you would normally expect to merge into 4 files - but actually we see FileCounter occasionally spiking. * Add partial merge support There is a `max_mergebelow` size which can be a positive integer, or infinity. It defaults to 32. If a merge from Level N covers less than `max_mergebelow` files in level N + 1 - the merge will proceesd as before. If it has >= `max_mergebelow`, the merge will be curtailed when `max_mergebelow div 2` files have been created at that level. The remainder for Level N will then be written, as well as for Level N + 1 up to the next whole file that has no yet been touched by the merge. The backlog that prompted the merge will still exist - as the files in Level N have not been changed. However, it is likely the next file picked will not be the same one, and will in probability have a lower number of files to merge (as the average is =< 8). This will stop progress from being halted by long merge jobs, as they will exit out in a safe way after partial completion. In the case where the majority of files covered do not require a merge, then those files will be skipped the next time the remainder file is picked up for merge at Level N
170 lines
No EOL
7.9 KiB
Erlang
170 lines
No EOL
7.9 KiB
Erlang
%%%============================================================================
|
|
%%% File paths
|
|
%%%============================================================================
|
|
-define(JOURNAL_FP, "journal").
|
|
-define(LEDGER_FP, "ledger").
|
|
%%%============================================================================
|
|
|
|
%%%============================================================================
|
|
%%% Configurable startup defaults
|
|
%%%============================================================================
|
|
-define(CACHE_SIZE, 2500).
|
|
-define(MAX_CACHE_MULTTIPLE, 2).
|
|
-define(MIN_CACHE_SIZE, 100).
|
|
-define(MIN_PCL_CACHE_SIZE, 400).
|
|
-define(MAX_PCL_CACHE_SIZE, 28000).
|
|
% This is less than actual max - but COIN_SIDECOUNT
|
|
-define(ABSOLUTEMAX_JOURNALSIZE, 4000000000).
|
|
-define(COMPRESSION_METHOD, lz4).
|
|
-define(COMPRESSION_POINT, on_receipt).
|
|
-define(COMPRESSION_LEVEL, 1).
|
|
-define(LOG_LEVEL, info).
|
|
-define(DEFAULT_DBID, 65536).
|
|
-define(OPEN_LASTMOD_RANGE, {0, infinity}).
|
|
-define(SNAPTIMEOUT_SHORT, 900). % 15 minutes
|
|
-define(SNAPTIMEOUT_LONG, 43200). % 12 hours
|
|
-define(SST_PAGECACHELEVEL_NOLOOKUP, 1).
|
|
-define(SST_PAGECACHELEVEL_LOOKUP, 4).
|
|
-define(DEFAULT_STATS_PERC, 10).
|
|
-define(DEFAULT_SYNC_STRATEGY, none).
|
|
%%%============================================================================
|
|
|
|
%%%============================================================================
|
|
%%% Non-configurable startup defaults
|
|
%%%============================================================================
|
|
-define(MAX_SSTSLOTS, 256).
|
|
-define(MAX_MERGEBELOW, 24).
|
|
-define(LOADING_PAUSE, 1000).
|
|
-define(LOADING_BATCH, 1000).
|
|
-define(CACHE_SIZE_JITTER, 25).
|
|
-define(JOURNAL_SIZE_JITTER, 20).
|
|
-define(LONG_RUNNING, 1000000).
|
|
% An individual task taking > 1s gets a specific log
|
|
-define(MAX_KEYCHECK_FREQUENCY, 100).
|
|
-define(MIN_KEYCHECK_FREQUENCY, 1).
|
|
-define(MAX_LEVELS, 8).
|
|
%% Should equal the length of the LEVEL_SCALEFACTOR
|
|
-define(CACHE_TYPE, skpl).
|
|
%%%============================================================================
|
|
|
|
%%%============================================================================
|
|
%%% Tags
|
|
%%%============================================================================
|
|
-define(RIAK_TAG, o_rkv).
|
|
%% Tag to be used on standard Riak KV objects
|
|
-define(STD_TAG, o).
|
|
%% Tag to be used on K/V objects for non-Riak purposes
|
|
-define(IDX_TAG, i).
|
|
%% Tag used for secondary index keys
|
|
-define(HEAD_TAG, h).
|
|
%% Tag used for head-only objects
|
|
|
|
-define(INKT_STND, stnd).
|
|
%% Inker key type used for 'normal' objects
|
|
-define(INKT_MPUT, mput).
|
|
%% Inker key type used for 'batch' objects
|
|
-define(INKT_KEYD, keyd).
|
|
%% Inker key type used for objects which contain no value, only key changes
|
|
%% This is used currently for objects formed under a 'retain' strategy
|
|
%% on Inker compaction
|
|
-define(INKT_TOMB, tomb).
|
|
%% Inker key type used for tombstones
|
|
%%%============================================================================
|
|
|
|
%%%============================================================================
|
|
%%% Helper Function
|
|
%%%============================================================================
|
|
|
|
-define(IS_DEF(Attribute), Attribute =/= undefined).
|
|
|
|
-if(?OTP_RELEASE < 26).
|
|
-type dynamic() :: any().
|
|
-endif.
|
|
|
|
%%%============================================================================
|
|
%%% Shared records
|
|
%%%============================================================================
|
|
-record(level,
|
|
{level :: integer(),
|
|
is_basement = false :: boolean(),
|
|
timestamp :: integer()}).
|
|
|
|
-record(cdb_options,
|
|
{max_size :: pos_integer() | undefined,
|
|
max_count :: pos_integer() | undefined,
|
|
file_path :: string() | undefined,
|
|
waste_path :: string() | undefined,
|
|
binary_mode = false :: boolean(),
|
|
% Default set by bookie to be `true`
|
|
% `false` set here due to legacy of unit tests
|
|
% using non-binary keys
|
|
sync_strategy = ?DEFAULT_SYNC_STRATEGY,
|
|
log_options = leveled_log:get_opts()
|
|
:: leveled_log:log_options(),
|
|
monitor = {no_monitor, 0}
|
|
:: leveled_monitor:monitor()}).
|
|
|
|
-record(sst_options,
|
|
{press_method = ?COMPRESSION_METHOD
|
|
:: leveled_sst:press_method(),
|
|
press_level = ?COMPRESSION_LEVEL :: non_neg_integer(),
|
|
log_options = leveled_log:get_opts()
|
|
:: leveled_log:log_options(),
|
|
max_sstslots = ?MAX_SSTSLOTS :: pos_integer()|infinity,
|
|
max_mergebelow = ?MAX_MERGEBELOW :: pos_integer()|infinity,
|
|
pagecache_level = ?SST_PAGECACHELEVEL_NOLOOKUP
|
|
:: pos_integer(),
|
|
monitor = {no_monitor, 0}
|
|
:: leveled_monitor:monitor()}).
|
|
|
|
-record(inker_options,
|
|
{cdb_max_size :: integer() | undefined,
|
|
root_path :: string() | undefined,
|
|
cdb_options = #cdb_options{} :: #cdb_options{},
|
|
start_snapshot = false :: boolean(),
|
|
bookies_pid :: pid() | undefined,
|
|
source_inker :: pid() | undefined,
|
|
reload_strategy = [] :: list(),
|
|
waste_retention_period :: integer() | undefined,
|
|
compression_method = ?COMPRESSION_METHOD
|
|
:: lz4|native|none,
|
|
compress_on_receipt = false :: boolean(),
|
|
max_run_length,
|
|
singlefile_compactionperc :: float()|undefined,
|
|
maxrunlength_compactionperc :: float()|undefined,
|
|
score_onein = 1 :: pos_integer(),
|
|
snaptimeout_long = 60 :: pos_integer(),
|
|
monitor = {no_monitor, 0}
|
|
:: leveled_monitor:monitor()}).
|
|
|
|
-record(penciller_options,
|
|
{root_path :: string() | undefined,
|
|
sst_options = #sst_options{} :: #sst_options{},
|
|
max_inmemory_tablesize = ?MIN_PCL_CACHE_SIZE
|
|
:: pos_integer(),
|
|
start_snapshot = false :: boolean(),
|
|
snapshot_query,
|
|
bookies_pid :: pid() | undefined,
|
|
bookies_mem :: tuple() | undefined,
|
|
source_penciller :: pid() | undefined,
|
|
snapshot_longrunning = true :: boolean(),
|
|
compression_method = ?COMPRESSION_METHOD
|
|
:: lz4|native|none,
|
|
levelzero_cointoss = false :: boolean(),
|
|
snaptimeout_short :: pos_integer() | undefined,
|
|
snaptimeout_long :: pos_integer() | undefined,
|
|
monitor = {no_monitor, 0}
|
|
:: leveled_monitor:monitor()}).
|
|
|
|
-record(iclerk_options,
|
|
{inker :: pid() | undefined,
|
|
max_run_length :: integer() | undefined,
|
|
cdb_options = #cdb_options{} :: #cdb_options{},
|
|
waste_retention_period :: integer() | undefined,
|
|
compression_method = ?COMPRESSION_METHOD
|
|
:: lz4|native|none,
|
|
singlefile_compactionperc :: float()|undefined,
|
|
maxrunlength_compactionperc :: float()|undefined,
|
|
score_onein = 1 :: pos_integer(),
|
|
reload_strategy = [] :: list()}).
|
|
%%%============================================================================ |