leveled/priv/leveled_multi.schema
Martin Sumner 999ce8ba5b
Add ZSTD compression (#430)
* Add support for zstd and split compression

Add support for using zstd as an alternative to native, lz4.

Upgrade lz4 to v1.9.4 (with ARM enhancements).

Allow for split compression algorithms - i.e. use native on journal, but lz4 on ledger.

* Switch to AdRoll zstd

Development appears to be active and ongoing.  No issues running on different linux flavours.

* Use realistic bucket name

* Update README.md

* Switch branch

* Add comment following review
2024-01-23 16:25:03 +00:00

222 lines
8 KiB
Erlang

%% -*- erlang -*-
%%%% leveled
%% @doc A path under which leveled data files will be stored.
{mapping, "multi_backend.$name.leveled.data_root", "riak_kv.multi_backend", [
{default, "$(platform_data_dir)/leveled"},
{datatype, directory},
hidden
]}.
%% @doc Strategy for flushing data to disk
%% Can be set to riak_sync, sync (if OTP > 16) or none. Use none, and the OS
%% will flush when most efficient. Use riak_sync or sync to flush after every
%% PUT (not recommended without some hardware support e.g. flash drives and/or
%% Flash-backed Write Caches)
{mapping, "multi_backend.$name.leveled.sync_strategy", "riak_kv.multi_backend", [
{default, none},
{datatype, atom},
hidden
]}.
%% @doc The key size of the Bookie's in-memory cache
{mapping, "multi_backend.$name.leveled.cache_size", "riak_kv.multi_backend", [
{default, 2000},
{datatype, integer},
hidden
]}.
%% @doc The key size of the Penciller's in-memory cache
{mapping, "multi_backend.$name.leveled.penciller_cache_size", "riak_kv.multi_backend", [
{default, 20000},
{datatype, integer},
hidden
]}.
%% @doc Compression method
%% Can be lz4 or native (which will use the Erlang native zlib compression)
%% within term_to_binary
{mapping, "multi_backend.$name.leveled.compression_method", "riak_kv.multi_backend", [
{datatype, {enum, [native, lz4, zstd, none]}},
{default, native},
hidden
]}.
%% @doc Ledger compression
%% If an alternative compression method is preferred specifically for the
%% ledger, it can be specified here. Default is as_store - use whatever method
%% has been defined in leveled.compression.method. Alternatives are native,
%% lz4, ztsd and none
{mapping, "multi_backend.$name.ledger_compression", "riak_kv.multi_backend", [
{datatype, {enum, [as_store, native, lz4, zstd, none]}},
{default, as_store}
]}.
%% @doc Compression point
%% The point at which compression is applied to the Journal (the Ledger is
%% always compressed). Use on_receipt or on_compact. on_compact is suitable
%% when values are unlikely to yield much benefit from compression
%% (compression is only attempted when compacting)
{mapping, "multi_backend.$name.leveled.compression_point", "riak_kv.multi_backend", [
{default, on_receipt},
{datatype, atom},
hidden
]}.
%% @doc Compresstion Level (Ledger LSM)
%% Specify the level in the LSM tree from which compression should occur.
%% Defaults to L1, so only L0 writes are uncompressed.
{mapping, "multi_backend.$name.leveled.compression_level", "riak_kv.multi_backend", [
{default, 1},
{datatype, integer},
{validators, ["range:0-7"]},
hidden
]}.
%% @doc The approximate size (in bytes) when a Journal file should be rolled.
%% Normally keep this as around the size of o(100K) objects. Default is 1GB.
%% Note that on startup an actual maximum size will be chosen which varies by
%% a random factor from this point - to avoid coordination of roll events
%% across vnodes.
{mapping, "multi_backend.$name.leveled.journal_size", "riak_kv.multi_backend", [
{default, 1000000000},
{datatype, integer}
]}.
%% @doc The approximate count of objects when a Journal file should be rolled.
%% This time measured in object count, a file will be rolled if either the
%% object count or the journal size limit is reached. Default 200K.
%% Note that on startup an actual maximum size will be chosen which varies by
%% a random factor from this point - to avoid coordination of roll events
%% across vnodes.
{mapping, "multi_backend.$name.leveled.journal_objectcount", "riak_kv.multi_backend", [
{default, 200000},
{datatype, integer}
]}.
%% @doc The level of the ledger to be pre-loaded into the page cache
%% Depending on how much memory is available for the page cache, and how much
%% disk I/O activity can be tolerated at startup - then the level at which the
%% ledger is forced into the page cache can be controlled by configuration.
{mapping, "multi_backend.$name.leveled.ledger_pagecachelevel", "riak_kv.multi_backend", [
{default, 4},
{datatype, integer}
]}.
%% @doc The number of journal compactions per vnode per day
%% The higher the value, the more compaction runs, and the sooner space is
%% recovered. But each run has a cost
{mapping, "multi_backend.$name.leveled.compaction_runs_perday", "riak_kv.multi_backend", [
{default, 24},
{datatype, integer},
hidden
]}.
%% @doc The number of times per day to score an individual file for compaction
%% The default value will lead to each file, on average, being scored once
%% every 12 hours
{mapping, "multi_backend.$name.leveled.compaction_scores_perday", "riak_kv.multi_backend", [
{default, 2},
{datatype, integer}
]}.
%% @doc Compaction Low Hour
%% The hour of the day in which journal compaction can start. Use Low hour
%% of 0 and High hour of 23 to have no compaction window (i.e. always compact
%% regardless of time of day)
{mapping, "multi_backend.$name.leveled.compaction_low_hour", "riak_kv.multi_backend", [
{default, 0},
{datatype, integer},
hidden
]}.
%% @doc Compaction Top Hour
%% The hour of the day, after which journal compaction should stop.
%% If low hour > top hour then, compaction will work overnight between low
%% hour and top hour (inclusive). Timings rely on server's view of local time
{mapping, "multi_backend.$name.leveled.compaction_top_hour", "riak_kv.multi_backend", [
{default, 23},
{datatype, integer},
hidden
]}.
%% @doc Max Journal Files Per Compaction Run
%% In a single compaction run, what is the maximum number of consecutive files
%% which may be compacted. If increasing this value, the average number of
%% files per run may decrease, unless adjustments are also made to the
%% maxrunlength and singlefile compactionpercentage settings.
{mapping, "multi_backend.$name.leveled.max_run_length", "riak_kv.multi_backend", [
{default, 4},
{datatype, integer},
hidden
]}.
%% @doc Target Percentage for Max Run
%% What is the target score for a maximum run of files, to qualify for
%% compaction. If less than this percentage would be retained after compaction
%% then it is a candidate (e.g. in default case if 25% of space would be
%% recovered)
{mapping, "multi_backend.$name.leveled.maxrunlength_compactionpercentage", "riak_kv.multi_backend", [
{default, 75.0},
{datatype, float},
hidden
]}.
%% @doc Target Percentage for Single File
%% What is the target score for a run of a single file, to qualify for
%% compaction. If less than this percentage would be retained after compaction
%% then it is a candidate (e.g. in default case if 25% of space would be
%% recovered)
{mapping, "multi_backend.$name.leveled.singlefile_compactionpercentage", "riak_kv.multi_backend", [
{default, 25.0},
{datatype, float},
hidden
]}.
%% @doc Snapshot timeout (short)
%% Maximum expected time for an index query. A query which is taking longer
%% than this may fail as it will be released - potentially allowing for some
%% file processes to delete. Timeout is in seconds.
{mapping, "multi_backend.$name.leveled.snapshot_timeout_short", "riak_kv.multi_backend", [
{default, 1800},
{datatype, integer},
hidden
]}.
%% @doc Snapshot timeout (long)
%% Maximum expected time for any other fold. A fold which is taking longer
%% than this may fail as it will be released - potentially allowing for some
%% file processes to delete. Timeout is in seconds.
{mapping, "multi_backend.$name.leveled.snapshot_timeout_long", "riak_kv.multi_backend", [
{default, 86400},
{datatype, integer},
hidden
]}.
%% @doc Statistic monitoring proportion
%% The proportion of requests to be convered by stats, an integer between
%% 0 and 100. There is no flow control, so setting this too high could
%% possibly overflow the leveled_monitor mailbox.
{mapping, "multi_backend.$name.leveled.stats_percentage", "riak_kv.multi_backend", [
{default, 10},
{datatype, integer},
{validators, ["range:0-100"]},
hidden
]}.
%% @doc Statistic log frequency (seconds)
%% The wait in seconds between logs from each leveled_monitor (there is one
%% monitor per vnode)
{mapping, "multi_backend.$name.leveled.stats_logfrequency", "riak_kv.multi_backend", [
{default, 30},
{datatype, integer},
hidden
]}.