
* Add performance/profiling test Add test to perf_SUITE to do performance tests and also profile different activities in leveled. This can then be used to highlight functions with unexpectedly high execution times, and prove the impact of changes. Switch between riak_ctperf and riak_fullperf to change from standard test (with profile option) to full-scale performance test * Change shape of default perfTest * Refactor SST Compare and contrast profile for guess, before and after refactor: pre ``` lists:map_1/2 313370 2.33 32379 [ 0.10] lists:foldl_1/3 956590 4.81 66992 [ 0.07] leveled_sst:'-expand_list_by_pointer/5-fun-0-'/4 925020 6.13 85318 [ 0.09] erlang:binary_to_term/1 3881 8.55 119012 [ 30.67] erlang:'++'/2 974322 11.55 160724 [ 0.16] lists:member/2 4000180 15.00 208697 [ 0.05] leveled_sst:find_pos/4 4029220 21.01 292347 [ 0.07] leveled_sst:member_check/2 4000000 21.17 294601 [ 0.07] -------------------------------------------------- -------- ------- ------- [----------] Total: 16894665 100.00% 1391759 [ 0.08] ``` post ``` lists:map_1/2 63800 0.79 6795 [ 0.11] erlang:term_to_binary/1 15726 0.81 6950 [ 0.44] lists:keyfind/3 180967 0.92 7884 [ 0.04] erlang:spawn_link/3 15717 1.08 9327 [ 0.59] leveled_sst:'-read_slots/5-fun-1-'/8 31270 1.15 9895 [ 0.32] gen:do_call/4 7881 1.31 11243 [ 1.43] leveled_penciller:find_nextkey/8 180936 2.01 17293 [ 0.10] prim_file:pread_nif/3 15717 3.89 33437 [ 2.13] leveled_sst:find_pos/4 4028940 17.85 153554 [ 0.04] erlang:binary_to_term/1 15717 51.97 447048 [ 28.44] -------------------------------------------------- ------- ------- ------ [----------] Total: 6704100 100.00% 860233 [ 0.13] ``` * Update leveled_penciller.erl * Mas d31 nhskv16sstpcl (#426) Performance updates to leveled: - Refactoring of pointer expansion when fetching from leveled_sst files to avoid expensive list concatenation. - Refactoring of leveled_ebloom to make more flexible, reduce code, and improve check time. - Refactoring of querying within leveled_sst to reduce the number of blocks that need to be de-serialised per query. - Refactoring of the leveled_penciller's query key comparator, to make use of maps and simplify the filtering. - General speed-up of frequently called functions.
165 lines
No EOL
7.8 KiB
Erlang
165 lines
No EOL
7.8 KiB
Erlang
%%%============================================================================
|
|
%%% File paths
|
|
%%%============================================================================
|
|
-define(JOURNAL_FP, "journal").
|
|
-define(LEDGER_FP, "ledger").
|
|
%%%============================================================================
|
|
|
|
%%%============================================================================
|
|
%%% Configurable startup defaults
|
|
%%%============================================================================
|
|
-define(CACHE_SIZE, 2500).
|
|
-define(MAX_CACHE_MULTTIPLE, 2).
|
|
-define(MIN_CACHE_SIZE, 100).
|
|
-define(MIN_PCL_CACHE_SIZE, 400).
|
|
-define(MAX_PCL_CACHE_SIZE, 28000).
|
|
% This is less than actual max - but COIN_SIDECOUNT
|
|
-define(ABSOLUTEMAX_JOURNALSIZE, 4000000000).
|
|
-define(COMPRESSION_METHOD, lz4).
|
|
-define(COMPRESSION_POINT, on_receipt).
|
|
-define(COMPRESSION_LEVEL, 1).
|
|
-define(LOG_LEVEL, info).
|
|
-define(DEFAULT_DBID, 65536).
|
|
-define(OPEN_LASTMOD_RANGE, {0, infinity}).
|
|
-define(SNAPTIMEOUT_SHORT, 900). % 15 minutes
|
|
-define(SNAPTIMEOUT_LONG, 43200). % 12 hours
|
|
-define(SST_PAGECACHELEVEL_NOLOOKUP, 1).
|
|
-define(SST_PAGECACHELEVEL_LOOKUP, 4).
|
|
-define(DEFAULT_STATS_PERC, 10).
|
|
-define(DEFAULT_SYNC_STRATEGY, none).
|
|
%%%============================================================================
|
|
|
|
%%%============================================================================
|
|
%%% Non-configurable startup defaults
|
|
%%%============================================================================
|
|
-define(MAX_SSTSLOTS, 256).
|
|
-define(LOADING_PAUSE, 1000).
|
|
-define(LOADING_BATCH, 1000).
|
|
-define(CACHE_SIZE_JITTER, 25).
|
|
-define(JOURNAL_SIZE_JITTER, 20).
|
|
-define(LONG_RUNNING, 1000000).
|
|
% An individual task taking > 1s gets a specific log
|
|
-define(MAX_KEYCHECK_FREQUENCY, 100).
|
|
-define(MIN_KEYCHECK_FREQUENCY, 1).
|
|
-define(MAX_LEVELS, 8).
|
|
%% Should equal the length of the LEVEL_SCALEFACTOR
|
|
-define(CACHE_TYPE, skpl).
|
|
%%%============================================================================
|
|
|
|
%%%============================================================================
|
|
%%% Tags
|
|
%%%============================================================================
|
|
-define(RIAK_TAG, o_rkv).
|
|
%% Tag to be used on standard Riak KV objects
|
|
-define(STD_TAG, o).
|
|
%% Tag to be used on K/V objects for non-Riak purposes
|
|
-define(IDX_TAG, i).
|
|
%% Tag used for secondary index keys
|
|
-define(HEAD_TAG, h).
|
|
%% Tag used for head-only objects
|
|
|
|
-define(INKT_STND, stnd).
|
|
%% Inker key type used for 'normal' objects
|
|
-define(INKT_MPUT, mput).
|
|
%% Inker key type used for 'batch' objects
|
|
-define(INKT_KEYD, keyd).
|
|
%% Inker key type used for objects which contain no value, only key changes
|
|
%% This is used currently for objects formed under a 'retain' strategy
|
|
%% on Inker compaction
|
|
-define(INKT_TOMB, tomb).
|
|
%% Inker key type used for tombstones
|
|
%%%============================================================================
|
|
|
|
|
|
%%%============================================================================
|
|
%%% Shared records
|
|
%%%============================================================================
|
|
-record(level,
|
|
{level :: integer(),
|
|
is_basement = false :: boolean(),
|
|
timestamp :: integer()}).
|
|
|
|
-record(manifest_entry,
|
|
{start_key :: tuple() | undefined,
|
|
end_key :: tuple() | undefined,
|
|
owner :: pid()|list(),
|
|
filename :: string() | undefined,
|
|
bloom = none :: leveled_ebloom:bloom() | none}).
|
|
|
|
-record(cdb_options,
|
|
{max_size :: pos_integer() | undefined,
|
|
max_count :: pos_integer() | undefined,
|
|
file_path :: string() | undefined,
|
|
waste_path :: string() | undefined,
|
|
binary_mode = false :: boolean(),
|
|
% Default set by bookie to be `true`
|
|
% `false` set here due to legacy of unit tests
|
|
% using non-binary keys
|
|
sync_strategy = ?DEFAULT_SYNC_STRATEGY,
|
|
log_options = leveled_log:get_opts()
|
|
:: leveled_log:log_options(),
|
|
monitor = {no_monitor, 0}
|
|
:: leveled_monitor:monitor()}).
|
|
|
|
-record(sst_options,
|
|
{press_method = ?COMPRESSION_METHOD
|
|
:: leveled_sst:press_method(),
|
|
press_level = ?COMPRESSION_LEVEL :: non_neg_integer(),
|
|
log_options = leveled_log:get_opts()
|
|
:: leveled_log:log_options(),
|
|
max_sstslots = ?MAX_SSTSLOTS :: pos_integer(),
|
|
pagecache_level = ?SST_PAGECACHELEVEL_NOLOOKUP
|
|
:: pos_integer(),
|
|
monitor = {no_monitor, 0}
|
|
:: leveled_monitor:monitor()}).
|
|
|
|
-record(inker_options,
|
|
{cdb_max_size :: integer() | undefined,
|
|
root_path :: string() | undefined,
|
|
cdb_options = #cdb_options{} :: #cdb_options{},
|
|
start_snapshot = false :: boolean(),
|
|
bookies_pid :: pid() | undefined,
|
|
source_inker :: pid() | undefined,
|
|
reload_strategy = [] :: list(),
|
|
waste_retention_period :: integer() | undefined,
|
|
compression_method = ?COMPRESSION_METHOD
|
|
:: lz4|native|none,
|
|
compress_on_receipt = false :: boolean(),
|
|
max_run_length,
|
|
singlefile_compactionperc :: float()|undefined,
|
|
maxrunlength_compactionperc :: float()|undefined,
|
|
score_onein = 1 :: pos_integer(),
|
|
snaptimeout_long :: pos_integer() | undefined,
|
|
monitor = {no_monitor, 0}
|
|
:: leveled_monitor:monitor()}).
|
|
|
|
-record(penciller_options,
|
|
{root_path :: string() | undefined,
|
|
sst_options = #sst_options{} :: #sst_options{},
|
|
max_inmemory_tablesize :: integer() | undefined,
|
|
start_snapshot = false :: boolean(),
|
|
snapshot_query,
|
|
bookies_pid :: pid() | undefined,
|
|
bookies_mem :: tuple() | undefined,
|
|
source_penciller :: pid() | undefined,
|
|
snapshot_longrunning = true :: boolean(),
|
|
compression_method = ?COMPRESSION_METHOD
|
|
:: lz4|native|none,
|
|
levelzero_cointoss = false :: boolean(),
|
|
snaptimeout_short :: pos_integer() | undefined,
|
|
snaptimeout_long :: pos_integer() | undefined,
|
|
monitor = {no_monitor, 0}
|
|
:: leveled_monitor:monitor()}).
|
|
|
|
-record(iclerk_options,
|
|
{inker :: pid() | undefined,
|
|
max_run_length :: integer() | undefined,
|
|
cdb_options = #cdb_options{} :: #cdb_options{},
|
|
waste_retention_period :: integer() | undefined,
|
|
compression_method = ?COMPRESSION_METHOD
|
|
:: lz4|native|none,
|
|
singlefile_compactionperc :: float()|undefined,
|
|
maxrunlength_compactionperc :: float()|undefined,
|
|
score_onein = 1 :: pos_integer(),
|
|
reload_strategy = [] :: list()}).
|
|
%%%============================================================================ |