From 52c523dbab232d3bb365de29258aa712cafb4366 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 12 Dec 2018 09:56:06 +0000 Subject: [PATCH 1/4] Add multi_backend schema --- priv/leveled_multi.schema | 123 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 priv/leveled_multi.schema diff --git a/priv/leveled_multi.schema b/priv/leveled_multi.schema new file mode 100644 index 0000000..528cf1b --- /dev/null +++ b/priv/leveled_multi.schema @@ -0,0 +1,123 @@ +%% -*- erlang -*- + +%%%% leveled + +%% @doc A path under which leveled data files will be stored. +{mapping, "multi_backend.$name.leveled.data_root", "riak_kv.multi_backend", [ + {default, "$(platform_data_dir)/leveled"}, + {datatype, directory} +]}. + +%% @doc Strategy for flushing data to disk +%% Can be set to riak_sync, sync (if OTP > 16) or none. Use none, and the OS +%% will flush when most efficient. Use riak_sync or sync to flush after every +%% PUT (not recommended wihtout some hardware support e.g. flash drives and/or +%% Flash-backed Write Caches) +{mapping, "multi_backend.$name.leveled.sync_strategy", "riak_kv.multi_backend", [ + {default, none}, + {datatype, atom} +]}. + + +%% @doc The key size of the Bookie's in-memory cache +{mapping, "multi_backend.$name.leveled.cache_size", "riak_kv.multi_backend", [ + {default, 4000}, + {datatype, integer}, + hidden +]}. + +%% @doc The key size of the Penciller's in-memory cache +{mapping, "multi_backend.$name.leveled.penciller_cache_size", "riak_kv.multi_backend", [ + {default, 28000}, + {datatype, integer}, + hidden +]}. + +%% @doc Compression method +%% Can be lz4 or native (which will use the Erlang native zlib compression) +%% within term_to_binary +{mapping, "multi_backend.$name.leveled.compression_method", "riak_kv.multi_backend", [ + {default, lz4}, + {datatype, atom} +]}. + +%% @doc Compression point +%% The point at which compression is applied to the Journal (the Ledger is +%% always compressed). Use on_receipt or on_compact. on_compact is suitable +%% when values are unlikely to yield much benefit from compression +%% (compression is only attempted when compacting) +{mapping, "multi_backend.$name.leveled.compression_point", "riak_kv.multi_backend", [ + {default, on_receipt}, + {datatype, atom} +]}. + + +%% @doc The approximate size (in bytes) when a Journal file should be rolled. +%% Normally keep this as around the size of o(100K) objects. Default is 500MB +{mapping, "multi_backend.$name.leveled.journal_size", "riak_kv.multi_backend", [ + {default, 500000000}, + {datatype, integer} +]}. + +%% @doc The number of journal compactions per vnode per day +%% The higher the value, the more compaction runs, and the sooner space is +%% recovered. But each run has a cost +{mapping, "multi_backend.$name.leveled.compaction_runs_perday", "riak_kv.multi_backend", [ + {default, 16}, + {datatype, integer} +]}. + +%% @doc Compaction Low Hour +%% The hour of the day in which journal compaction can start. Use Low hour +%% of 0 and High hour of 23 to have no compaction window (i.e. always compact +%% regardless of time of day) +{mapping, "multi_backend.$name.leveled.compaction_low_hour", "riak_kv.multi_backend", [ + {default, 0}, + {datatype, integer} +]}. + +%% @doc Compaction Top Hour +%% The hour of the day, after which journal compaction should stop. +%% If low hour > top hour then, compaction will work overnight between low +%% hour and top hour (inclusive). Timings rely on server's view of local time +{mapping, "multi_backend.$name.leveled.compaction_top_hour", "riak_kv.multi_backend", [ + {default, 23}, + {datatype, integer} +]}. + +%% @doc Max Journal Files Per Compaction Run +%% In a single compaction run, what is the maximum number of consecutive files +%% which may be compacted. +{mapping, "multi_backend.$name.leveled.max_run_length", "riak_kv.multi_backend", [ + {default, 6}, + {datatype, integer} +]}. + +%% @doc Target Percentage for Max Run +%% What is the target score for a maximum run of files, to qualify for +%% compaction. If less than this percentage would be retained after compaction +%% then it is a candidate (e.g. in default case if 25% of space would be +%% recovered) +{mapping, "multi_backend.$name.leveled.maxrunlength_compactionpercentage", "riak_kv.multi_backend", [ + {default, 75.0}, + {datatype, float}, + hidden +]}. + +%% @doc Target Percentage for Single File +%% What is the target score for a run of a single file, to qualify for +%% compaction. If less than this percentage would be retained after compaction +%% then it is a candidate (e.g. in default case if 50% of space would be +%% recovered) +{mapping, "multi_backend.$name.leveled.singlefile_compactionpercentage", "riak_kv.multi_backend", [ + {default, 50.0}, + {datatype, float}, + hidden +]}. + + + + + + + From f211e587f73b5925c319cd5c9473f58e036af6d2 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 12 Dec 2018 16:16:47 +0000 Subject: [PATCH 2/4] Make multi_schema mainly hidden As wiht other schemas - make the multi_backend schema mainly hidden --- priv/leveled_multi.schema | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/priv/leveled_multi.schema b/priv/leveled_multi.schema index 528cf1b..cf58d18 100644 --- a/priv/leveled_multi.schema +++ b/priv/leveled_multi.schema @@ -38,7 +38,8 @@ %% within term_to_binary {mapping, "multi_backend.$name.leveled.compression_method", "riak_kv.multi_backend", [ {default, lz4}, - {datatype, atom} + {datatype, atom}, + hidden ]}. %% @doc Compression point @@ -48,7 +49,8 @@ %% (compression is only attempted when compacting) {mapping, "multi_backend.$name.leveled.compression_point", "riak_kv.multi_backend", [ {default, on_receipt}, - {datatype, atom} + {datatype, atom}, + hidden ]}. @@ -56,7 +58,8 @@ %% Normally keep this as around the size of o(100K) objects. Default is 500MB {mapping, "multi_backend.$name.leveled.journal_size", "riak_kv.multi_backend", [ {default, 500000000}, - {datatype, integer} + {datatype, integer}, + hidden ]}. %% @doc The number of journal compactions per vnode per day @@ -64,7 +67,8 @@ %% recovered. But each run has a cost {mapping, "multi_backend.$name.leveled.compaction_runs_perday", "riak_kv.multi_backend", [ {default, 16}, - {datatype, integer} + {datatype, integer}, + hidden ]}. %% @doc Compaction Low Hour @@ -73,7 +77,8 @@ %% regardless of time of day) {mapping, "multi_backend.$name.leveled.compaction_low_hour", "riak_kv.multi_backend", [ {default, 0}, - {datatype, integer} + {datatype, integer}, + hidden ]}. %% @doc Compaction Top Hour @@ -82,7 +87,8 @@ %% hour and top hour (inclusive). Timings rely on server's view of local time {mapping, "multi_backend.$name.leveled.compaction_top_hour", "riak_kv.multi_backend", [ {default, 23}, - {datatype, integer} + {datatype, integer}, + hidden ]}. %% @doc Max Journal Files Per Compaction Run @@ -90,7 +96,8 @@ %% which may be compacted. {mapping, "multi_backend.$name.leveled.max_run_length", "riak_kv.multi_backend", [ {default, 6}, - {datatype, integer} + {datatype, integer}, + hidden ]}. %% @doc Target Percentage for Max Run From 45f62ffa56314a288e133fd50b706138f6f5fd0b Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 13 Dec 2018 16:24:33 +0000 Subject: [PATCH 3/4] Make all multi schema hidden --- priv/leveled_multi.schema | 2 ++ 1 file changed, 2 insertions(+) diff --git a/priv/leveled_multi.schema b/priv/leveled_multi.schema index cf58d18..ee9d3d2 100644 --- a/priv/leveled_multi.schema +++ b/priv/leveled_multi.schema @@ -6,6 +6,7 @@ {mapping, "multi_backend.$name.leveled.data_root", "riak_kv.multi_backend", [ {default, "$(platform_data_dir)/leveled"}, {datatype, directory} + hidden ]}. %% @doc Strategy for flushing data to disk @@ -16,6 +17,7 @@ {mapping, "multi_backend.$name.leveled.sync_strategy", "riak_kv.multi_backend", [ {default, none}, {datatype, atom} + hidden ]}. From 5b578ea2bba6a7551db334a32f38e2401e361b82 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 14 Dec 2018 19:57:32 +0000 Subject: [PATCH 4/4] Add new snapshot timeout to multi-backend --- priv/leveled_multi.schema | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/priv/leveled_multi.schema b/priv/leveled_multi.schema index ee9d3d2..1cd6303 100644 --- a/priv/leveled_multi.schema +++ b/priv/leveled_multi.schema @@ -125,6 +125,25 @@ ]}. +%% @doc Snapshot timeout (short) +%% Maximum expected time for an index query. A query which is taking longer +%% than this may fail as it will be released - potentially allowing for some +%% file processes to delete. Timeout is in seconds. +{mapping, "multi_backend.$name.leveled.snapshot_timeout_short", "riak_kv.multi_backend", [ + {default, 1800}, + {datatype, integer}, + hidden +]}. + +%% @doc Snapshot timeout (long) +%% Maximum expected time for any othe rfold. A fold which is taking longer +%% than this may fail as it will be released - potentially allowing for some +%% file processes to delete. Timeout is in seconds. +{mapping, "multi_backend.$name.leveled.snapshot_timeout_long", "riak_kv.multi_backend", [ + {default, 86400}, + {datatype, integer}, + hidden +]}.