Mas d31 i416 (#418)

* Add compression controls (#417) * Add compression controls Add configuration options to allow for a compression algorithm of `none` to disable compression altogether. Also an option to change the point in the LSM tree when compression is applied. * Handle configurable defaults consistently Move them into leveled.hrl. This forces double-definitions to be resolved. There are some other constants in leveled_bookie that are relevant outside of leveled_bookie. These are all now in the non-configurable startup defaults section. * Clarify referred-to default is OTP not leveled * Update leveled_bookie.erl Handle xref issue with eunit include
2023-11-07 14:58:43 +00:00 · 2023-11-07 14:58:43 +00:00 · 9e804924a8
commit 9e804924a8
parent b96518c32a
11 changed files with 338 additions and 163 deletions
--- a/src/leveled_bookie.erl
+++ b/src/leveled_bookie.erl
@ -102,34 +102,8 @@
 -export([book_returnactors/1]).
 -endif.

-define(LOADING_PAUSE, 1000).
-define(CACHE_SIZE, 2500).
-define(MAX_CACHE_MULTTIPLE, 2).
-define(MIN_CACHE_SIZE, 100).
-define(MIN_PCL_CACHE_SIZE, 400).
-define(MAX_PCL_CACHE_SIZE, 28000). 
-    % This is less than actual max - but COIN_SIDECOUNT
-define(CACHE_SIZE_JITTER, 25).
-define(JOURNAL_SIZE_JITTER, 20).
-define(ABSOLUTEMAX_JOURNALSIZE, 4000000000).
-define(LONG_RUNNING, 1000000).
-    % An individual task taking > 1s gets a specific log
-define(COMPRESSION_METHOD, lz4).
-define(COMPRESSION_POINT, on_receipt).
-define(LOG_LEVEL, info).
-define(TIMING_SAMPLESIZE, 100).
-define(DEFAULT_DBID, 65536).
-define(TIMING_SAMPLECOUNTDOWN, 50000).
 -define(DUMMY, dummy). % Dummy key used for mput operations
-define(MAX_KEYCHECK_FREQUENCY, 100).
-define(MIN_KEYCHECK_FREQUENCY, 1).
-define(OPEN_LASTMOD_RANGE, {0, infinity}).
-define(SNAPTIMEOUT_SHORT, 900). % 15 minutes
-define(SNAPTIMEOUT_LONG, 43200). % 12 hours
-define(SST_PAGECACHELEVEL_NOLOOKUP, 1).
-define(SST_PAGECACHELEVEL_LOOKUP, 4).
-define(CACHE_LOGPOINT, 50000).
-define(DEFAULT_STATS_PERC, 10).
+
 -define(OPTION_DEFAULTS,
            [{root_path, undefined},
                {snapshot_bookie, undefined},
@ -138,7 +112,7 @@
                {max_journalsize, 1000000000},
                {max_journalobjectcount, 200000},
                {max_sstslots, 256},
-                {sync_strategy, none},
+                {sync_strategy, ?DEFAULT_SYNC_STRATEGY},
                {head_only, false},
                {waste_retention_period, undefined},
                {max_run_length, undefined},
@ -150,6 +124,7 @@
                {ledger_preloadpagecache_level, ?SST_PAGECACHELEVEL_LOOKUP},
                {compression_method, ?COMPRESSION_METHOD},
                {compression_point, ?COMPRESSION_POINT},
+                {compression_level, ?COMPRESSION_LEVEL},
                {log_level, ?LOG_LEVEL},
                {forced_logs, []},
                {database_id, ?DEFAULT_DBID},
@ -314,10 +289,12 @@
            % To which level of the ledger should the ledger contents be
            % pre-loaded into the pagecache (using fadvise on creation and
            % startup)
-        {compression_method, native|lz4} |
+        {compression_method, native|lz4|none} |
            % Compression method and point allow Leveled to be switched from
            % using bif based compression (zlib) to using nif based compression
-            % (lz4).
+            % (lz4).  To disable compression use none.  This will disable in
+            % the ledger as well as the journla (both on_receipt and
+            % on_compact).
            % Defaults to ?COMPRESSION_METHOD
        {compression_point, on_compact|on_receipt} |
            % The =compression point can be changed between on_receipt (all
@ -325,6 +302,10 @@
            % values are originally stored uncompressed (speeding PUT times),
            % and are only compressed when they are first subject to compaction
            % Defaults to ?COMPRESSION_POINT
+        {compression_level, 0..7} |
+            % At what level of the LSM tree in the ledger should compression be
+            % enabled.
+            % Defaults to ?COMPRESSION_LEVEL
        {log_level, debug|info|warn|error|critical} |
            % Set the log level.  The default log_level of info is noisy - the
            % current implementation was targetted at environments that have
@ -1805,6 +1786,7 @@ set_options(Opts, Monitor) ->
                % If using lz4 this is not recommended
                false 
        end,
+    CompressionLevel = proplists:get_value(compression_level, Opts),
    
    MaxSSTSlots = proplists:get_value(max_sstslots, Opts),

@ -1837,6 +1819,7 @@ set_options(Opts, Monitor) ->
                            sst_options =
                                #sst_options{
                                    press_method = CompressionMethod,
+                                    press_level = CompressionLevel,
                                    log_options = leveled_log:get_opts(),
                                    max_sstslots = MaxSSTSlots,
                                    monitor = Monitor},
--- a/src/leveled_codec.erl
+++ b/src/leveled_codec.erl
@ -106,7 +106,7 @@
 -type object_spec() ::
        object_spec_v0()|object_spec_v1().
 -type compression_method() ::
-        lz4|native.
+        lz4|native|none.
 -type index_specs() ::
        list({add|remove, any(), any()}).
 -type journal_keychanges() :: 
@ -508,7 +508,9 @@ serialise_object(Object, true, Method) when is_binary(Object) ->
            {ok, Bin} = lz4:pack(Object),
            Bin;
        native ->
-            zlib:compress(Object)
+            zlib:compress(Object);
+        none ->
+            Object
    end;
 serialise_object(Object, false, _Method) ->
    term_to_binary(Object);
@ -554,7 +556,8 @@ encode_valuetype(IsBinary, IsCompressed, Method) ->
    Bit3 = 
        case Method of 
            lz4 -> 4;
-            native -> 0
+            native -> 0;
+            none -> 0
        end,
    Bit2 =
        case IsBinary of            
@ -562,7 +565,7 @@ encode_valuetype(IsBinary, IsCompressed, Method) ->
            false -> 0
        end,
    Bit1 =
-        case IsCompressed of
+        case IsCompressed and (Method =/= none) of
            true -> 1;
            false -> 0
        end,
--- a/src/leveled_iclerk.erl
+++ b/src/leveled_iclerk.erl
@ -113,7 +113,7 @@
                reload_strategy = ?DEFAULT_RELOAD_STRATEGY :: list(),
                singlefile_compactionperc  = ?SINGLEFILE_COMPACTION_TARGET :: float(),
                maxrunlength_compactionperc = ?MAXRUNLENGTH_COMPACTION_TARGET ::float(),
-                compression_method = native :: lz4|native,
+                compression_method = native :: lz4|native|none,
                scored_files = [] :: list(candidate()),
                scoring_state :: scoring_state()|undefined,
                score_onein = 1 :: pos_integer()}).
--- a/src/leveled_inker.erl
+++ b/src/leveled_inker.erl
@ -151,7 +151,7 @@
                compaction_pending = false :: boolean(),
                bookie_monref :: reference() | undefined,
                is_snapshot = false :: boolean(),
-                compression_method = native :: lz4|native,
+                compression_method = native :: lz4|native|none,
                compress_on_receipt = false :: boolean(),
                snap_timeout :: pos_integer() | undefined, % in seconds
                source_inker :: pid() | undefined}).
--- a/src/leveled_penciller.erl
+++ b/src/leveled_penciller.erl
@ -221,7 +221,6 @@
 -define(ITERATOR_SCANWIDTH, 4).
 -define(TIMING_SAMPLECOUNTDOWN, 10000).
 -define(TIMING_SAMPLESIZE, 100).
-define(OPEN_LASTMOD_RANGE, {0, infinity}).
 -define(SHUTDOWN_PAUSE, 10000).
    % How long to wait for snapshots to be released on shutdown
    % before forcing closure of snapshots
--- a/src/leveled_sst.erl
+++ b/src/leveled_sst.erl
@ -68,8 +68,12 @@
 -define(LOOK_BLOCKSIZE, {24, 32}). % 4x + y = ?LOOK_SLOTSIZE
 -define(NOLOOK_SLOTSIZE, 256).
 -define(NOLOOK_BLOCKSIZE, {56, 32}). % 4x + y = ?NOLOOK_SLOTSIZE
-define(COMPRESSION_LEVEL, 1).
-define(BINARY_SETTINGS, [{compressed, ?COMPRESSION_LEVEL}]).
+-define(COMPRESSION_FACTOR, 1).
+    % When using native compression - how hard should the compression code
+    % try to reduce the size of the compressed output. 1 Is to imply minimal
+    % effort, 6 is default in OTP:
+    % https://www.erlang.org/doc/man/erlang.html#term_to_binary-2
+-define(BINARY_SETTINGS, [{compressed, ?COMPRESSION_FACTOR}]).
 -define(MERGE_SCANWIDTH, 16).
 -define(DISCARD_EXT, ".discarded").
 -define(DELETE_TIMEOUT, 10000).
@ -80,7 +84,6 @@
 -define(BLOCK_LENGTHS_LENGTH, 20).
 -define(LMD_LENGTH, 4).
 -define(FLIPPER32, 4294967295).
-define(COMPRESS_AT_LEVEL, 1).
 -define(DOUBLESIZE_LEVEL, 3).
 -define(INDEX_MODDATE, true).
 -define(TOMB_COUNT, true).
@ -270,12 +273,7 @@ sst_new(RootPath, Filename, Level, KVList, MaxSQN, OptsSST) ->

 sst_new(RootPath, Filename, Level, KVList, MaxSQN, OptsSST, IndexModDate) ->
    {ok, Pid} = gen_statem:start_link(?MODULE, [], ?START_OPTS),
-    PressMethod0 = compress_level(Level, OptsSST#sst_options.press_method),
-    MaxSlots0 = maxslots_level(Level, OptsSST#sst_options.max_sstslots),
-    OptsSST0 =
-        OptsSST#sst_options{
-            press_method = PressMethod0, max_sstslots = MaxSlots0},
-
+    OptsSST0 = update_options(OptsSST, Level),
    {[], [], SlotList, FK, _CountOfTombs}  =
        merge_lists(KVList, OptsSST0, IndexModDate),
    case gen_statem:call(Pid, {sst_new,
@ -325,14 +323,15 @@ sst_newmerge(RootPath, Filename,
 sst_newmerge(RootPath, Filename,
        KVL1, KVL2, IsBasement, Level,
        MaxSQN, OptsSST, IndexModDate, TombCount) ->
-    PressMethod0 = compress_level(Level, OptsSST#sst_options.press_method),
-    MaxSlots0 = maxslots_level(Level, OptsSST#sst_options.max_sstslots),
-    OptsSST0 =
-        OptsSST#sst_options{press_method = PressMethod0,
-                            max_sstslots = MaxSlots0},
-    {Rem1, Rem2, SlotList, FK, CountOfTombs} =
-        merge_lists(KVL1, KVL2, {IsBasement, Level}, OptsSST0,
-                    IndexModDate, TombCount),
+    OptsSST0 = update_options(OptsSST, Level),
+    {Rem1, Rem2, SlotList, FK, CountOfTombs} = 
+        merge_lists(
+            KVL1,
+            KVL2,
+            {IsBasement, Level},
+            OptsSST0,
+            IndexModDate,
+            TombCount),
    case SlotList of
        [] ->
            empty;
@ -371,11 +370,7 @@ sst_newmerge(RootPath, Filename,
 sst_newlevelzero(RootPath, Filename,
                    Slots, Fetcher, Penciller,
                    MaxSQN, OptsSST) ->
-    PressMethod0 = compress_level(0, OptsSST#sst_options.press_method),
-    MaxSlots0 = maxslots_level(0, OptsSST#sst_options.max_sstslots),
-    OptsSST0 =
-        OptsSST#sst_options{press_method = PressMethod0,
-                            max_sstslots = MaxSlots0},
+    OptsSST0 = update_options(OptsSST, 0),
    {ok, Pid} = gen_statem:start_link(?MODULE, [], ?START_OPTS),
    %% Initiate the file into the "starting" state
    ok = gen_statem:call(Pid, {sst_newlevelzero,
@ -1253,6 +1248,16 @@ tune_seglist(SegList) ->
 %%% Internal Functions
 %%%============================================================================

+
+-spec update_options(sst_options(), non_neg_integer()) -> sst_options().
+update_options(OptsSST, Level) ->
+    CompressLevel = OptsSST#sst_options.press_level,
+    PressMethod0 =
+        compress_level(Level, CompressLevel, OptsSST#sst_options.press_method),
+    MaxSlots0 =
+        maxslots_level(Level, OptsSST#sst_options.max_sstslots),
+    OptsSST#sst_options{press_method = PressMethod0, max_sstslots = MaxSlots0}.
+
 -spec new_blockindex_cache(pos_integer()) -> blockindex_cache().
 new_blockindex_cache(Size) ->
    {0, array:new([{size, Size}, {default, none}]), 0}.
@ -1494,12 +1499,14 @@ fetch_range(StartKey, EndKey, ScanWidth, SegList, LowLastMod, State) ->
                    State#state.index_moddate),
    {NeededBlockIdx, SlotsToFetchBinList, SlotsToPoint}.

-spec compress_level(integer(), press_method()) -> press_method().
+-spec compress_level(
+    non_neg_integer(), non_neg_integer(), press_method()) -> press_method().
 %% @doc
 %% disable compression at higher levels for improved performance
-compress_level(Level, _PressMethod) when Level < ?COMPRESS_AT_LEVEL ->
+compress_level(
+        Level, LevelToCompress, _PressMethod) when Level < LevelToCompress ->
    none;
-compress_level(_Level, PressMethod) ->
+compress_level(_Level, _LevelToCompress, PressMethod) ->
    PressMethod.

 -spec maxslots_level(level(), pos_integer()) ->  pos_integer().