From 3d3d284805d614eb7a975067671c7830b13315b3 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Mon, 13 Mar 2023 11:46:08 +0000 Subject: [PATCH] Mas p401 coverage (#404) * refactor leveled_sst from gen_fsm to gen_statem * format_status/2 takes State and State Data but this function is deprecated... put in for backward compatibility * refactor leveled_cdb from gen_fsm to gen_statem * disable irrelevant warning ignorer * Remove unnecessary code paths Only support messages, especially info messages, where they are possible. * Mas i1820 offlinedeserialisation cbo (#403) * Log report GC Info by manifest level * Hibernate on range query If Block Index Cache is not full, and we're not yielding * Spawn to deserialise blocks offline Hypothesis is that the growth in the heap necessary due to continual term_to_binary calls to deserialise blocks is wasting memory - so do this memory-intensive task in a short-lived process. * Start with hibernate_after option * Always build BIC Testing indicates that the BIC itself is not a primary memory issue - the primary issue is due to a lack of garbage collection and a growing heap. This change enhances the patch to offline serialisation so that: - get_sqn & get_kv are standardised to build the BIC, and hibernate when it is built. - the offline PId is linked to crash this process on failure (as would happen now). * Standardise spawning for check_block/3 Now deserialise in both parts of the code. * Only spawn for check_block if cache not full * Update following review * Standardise formatting Make test more reliable. Show no new compaction after third compaction. * Update comments --------- Co-authored-by: Thomas Arts --- rebar.config | 1 - src/leveled_cdb.erl | 821 +++++++------- src/leveled_log.erl | 2 + src/leveled_pclerk.erl | 12 +- src/leveled_penciller.erl | 2 +- src/leveled_pmanifest.erl | 38 +- src/leveled_sst.erl | 1631 ++++++++++++++-------------- test/end_to_end/recovery_SUITE.erl | 1 + 8 files changed, 1255 insertions(+), 1253 deletions(-) diff --git a/rebar.config b/rebar.config index e1112ba..2a55d96 100644 --- a/rebar.config +++ b/rebar.config @@ -2,7 +2,6 @@ {erl_opts, [warnings_as_errors, - {platform_define, "^2[0-5]{1}", fsm_deprecated}, {platform_define, "^2[2-5]{1}", if_check}]}. {xref_checks, [undefined_function_calls,undefined_functions]}. diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index c50c08a..8939a39 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -1,6 +1,6 @@ %% -------- CDB File Clerk --------- %% -%% This is a modified version of the cdb module provided by Tom Whitcomb. +%% This is a modified version of the cdb module provided by Tom Whitcomb. %% %% - https://github.com/thomaswhitcomb/erlang-cdb %% @@ -9,37 +9,37 @@ %% %% - https://cr.yp.to/cdb.html %% -%% The primary differences are: -%% - Support for incrementally writing a CDB file while keeping the hash table +%% The primary differences are: +%% - Support for incrementally writing a CDB file while keeping the hash table %% in memory %% - The ability to scan a database in blocks of sequence numbers %% - The applictaion of a CRC check by default to all values %% -%% Because of the final delta - this is incompatible with standard CDB files -%% (in that you won't be able to fetch values if the file was written by +%% Because of the final delta - this is incompatible with standard CDB files +%% (in that you won't be able to fetch values if the file was written by %% another CDB writer as the CRC check is missing) %% %% This module provides functions to create and query a CDB (constant database). -%% A CDB implements a two-level hashtable which provides fast {key,value} +%% A CDB implements a two-level hashtable which provides fast {key,value} %% lookups that remain fairly constant in speed regardless of the CDBs size. %% -%% The first level in the CDB occupies the first 255 doublewords in the file. -%% Each doubleword slot contains two values. The first is a file pointer to -%% the primary hashtable (at the end of the file) and the second value is the -%% number of entries in the hashtable. The first level table of 255 entries +%% The first level in the CDB occupies the first 255 doublewords in the file. +%% Each doubleword slot contains two values. The first is a file pointer to +%% the primary hashtable (at the end of the file) and the second value is the +%% number of entries in the hashtable. The first level table of 255 entries %% is indexed with the lower eight bits of the hash of the input key. %% -%% Following the 255 doublewords are the {key,value} tuples. The tuples are -%% packed in the file without regard to word boundaries. Each {key,value} +%% Following the 255 doublewords are the {key,value} tuples. The tuples are +%% packed in the file without regard to word boundaries. Each {key,value} %% tuple is represented with a four byte key length, a four byte value length, %% the actual key value followed by the actual value. %% -%% Following the {key,value} tuples are the primary hash tables. There are -%% at most 255 hash tables. Each hash table is referenced by one of the 255 -%% doubleword entries at the top of the file. For efficiency reasons, each -%% hash table is allocated twice the number of entries that it will need. +%% Following the {key,value} tuples are the primary hash tables. There are +%% at most 255 hash tables. Each hash table is referenced by one of the 255 +%% doubleword entries at the top of the file. For efficiency reasons, each +%% hash table is allocated twice the number of entries that it will need. %% Each entry in the hash table is a doubleword. -%% The first word is the corresponding hash value and the second word is a +%% The first word is the corresponding hash value and the second word is a %% file pointer to the actual {key,value} tuple higher in the file. %% %% @@ -47,37 +47,20 @@ -module(leveled_cdb). --behaviour(gen_fsm). +-behaviour(gen_statem). -include("include/leveled.hrl"). - --ifdef(fsm_deprecated). --compile({nowarn_deprecated_function, - [{gen_fsm, start_link, 3}, - {gen_fsm, sync_send_event, 3}, - {gen_fsm, sync_send_event, 2}, - {gen_fsm, send_event, 2}, - {gen_fsm, sync_send_all_state_event, 3}, - {gen_fsm, send_all_state_event, 2}, - {gen_fsm, reply, 2}]}). --endif. - - -export([init/1, - handle_sync_event/4, - handle_event/3, - handle_info/3, + callback_mode/0, terminate/3, - code_change/4, - starting/3, + code_change/4]). + +%% states +-export([starting/3, writer/3, - writer/2, - rolling/2, rolling/3, reader/3, - reader/2, - delete_pending/3, - delete_pending/2]). + delete_pending/3]). -export([cdb_open_writer/1, cdb_open_writer/2, @@ -125,7 +108,7 @@ -define(TIMING_SAMPLECOUNTDOWN, 5000). -define(TIMING_SAMPLESIZE, 100). -define(GETPOS_FACTOR, 8). --define(MAX_OBJECT_SIZE, 1000000000). +-define(MAX_OBJECT_SIZE, 1000000000). % 1GB but really should be much smaller than this -define(MEGA, 1000000). -define(CACHE_LIFE, 86400). @@ -177,8 +160,8 @@ cdb_open_writer(Filename) -> %% hashtree cached in memory (the file will need to be scanned to build the %% hashtree) cdb_open_writer(Filename, Opts) -> - {ok, Pid} = gen_fsm:start_link(?MODULE, [Opts], []), - ok = gen_fsm:sync_send_event(Pid, {open_writer, Filename}, infinity), + {ok, Pid} = gen_statem:start_link(?MODULE, [Opts], []), + ok = gen_statem:call(Pid, {open_writer, Filename}, infinity), {ok, Pid}. -spec cdb_reopen_reader(string(), binary(), cdb_options()) -> {ok, pid()}. @@ -191,20 +174,20 @@ cdb_open_writer(Filename, Opts) -> %% The LastKey is the Key of the last object added to the file - and is used to %% determine when scans over a file have completed. cdb_reopen_reader(Filename, LastKey, CDBopts) -> - {ok, Pid} = - gen_fsm:start_link(?MODULE, - [CDBopts#cdb_options{binary_mode=true}], - []), - ok = gen_fsm:sync_send_event(Pid, - {open_reader, Filename, LastKey}, - infinity), + {ok, Pid} = + gen_statem:start_link(?MODULE, + [CDBopts#cdb_options{binary_mode=true}], + []), + ok = gen_statem:call(Pid, + {open_reader, Filename, LastKey}, + infinity), {ok, Pid}. -spec cdb_open_reader(string()) -> {ok, pid()}. %% @doc %% Open an existing file that has already been moved into read-only mode. %% Don't use this if the LastKey is known, as this requires an expensive scan -%% to discover the LastKey. +%% to discover the LastKey. cdb_open_reader(Filename) -> cdb_open_reader(Filename, #cdb_options{binary_mode=true}). @@ -215,15 +198,15 @@ cdb_open_reader(Filename) -> %% to discover the LastKey. %% Allows non-default cdb_options to be passed cdb_open_reader(Filename, Opts) -> - {ok, Pid} = gen_fsm:start_link(?MODULE, [Opts], []), - ok = gen_fsm:sync_send_event(Pid, {open_reader, Filename}, infinity), + {ok, Pid} = gen_statem:start_link(?MODULE, [Opts], []), + ok = gen_statem:call(Pid, {open_reader, Filename}, infinity), {ok, Pid}. -spec cdb_get(pid(), any()) -> {any(), any()}|missing. %% @doc -%% Extract a Key and Value from a CDB file by passing in a Key. +%% Extract a Key and Value from a CDB file by passing in a Key. cdb_get(Pid, Key) -> - gen_fsm:sync_send_event(Pid, {get_kv, Key}, infinity). + gen_statem:call(Pid, {get_kv, Key}, infinity). -spec cdb_put(pid(), any(), any()) -> ok|roll. %% @doc @@ -232,7 +215,7 @@ cdb_get(Pid, Key) -> %% %% Response can be roll - if there is no space to put this value in the file. %% It is assumed that the response to a "roll" will be to roll the file, which -%% will close this file for writing after persisting the hashtree. +%% will close this file for writing after persisting the hashtree. cdb_put(Pid, Key, Value) -> cdb_put(Pid, Key, Value, false). @@ -241,8 +224,7 @@ cdb_put(Pid, Key, Value) -> %% See cdb_put/3. Addition of force-sync option, to be used when sync mode is %% none to force a sync to disk on this particlar put. cdb_put(Pid, Key, Value, Sync) -> - - gen_fsm:sync_send_event(Pid, {put_kv, Key, Value, Sync}, infinity). + gen_statem:call(Pid, {put_kv, Key, Value, Sync}, infinity). -spec cdb_mput(pid(), list()) -> ok|roll. %% @doc @@ -253,7 +235,7 @@ cdb_put(Pid, Key, Value, Sync) -> %% It may be preferable to respond to roll by trying individual PUTs until %% roll is returned again cdb_mput(Pid, KVList) -> - gen_fsm:sync_send_event(Pid, {mput_kv, KVList}, infinity). + gen_statem:call(Pid, {mput_kv, KVList}, infinity). -spec cdb_getpositions(pid(), integer()|all) -> list(). %% @doc @@ -269,7 +251,7 @@ cdb_getpositions(Pid, SampleSize) -> % interleaved case SampleSize of all -> - FoldFun = + FoldFun = fun(Index, Acc) -> PosList = cdb_getpositions_fromidx(Pid, all, Index, []), lists:merge(Acc, lists:sort(PosList)) @@ -278,7 +260,7 @@ cdb_getpositions(Pid, SampleSize) -> lists:foldl(FoldFun, [], IdxList); S0 -> FC = ?GETPOS_FACTOR * S0, - FoldFun = + FoldFun = fun({_R, Index}, Acc) -> case length(Acc) of FC -> @@ -296,9 +278,8 @@ cdb_getpositions(Pid, SampleSize) -> end. cdb_getpositions_fromidx(Pid, SampleSize, Index, Acc) -> - gen_fsm:sync_send_event(Pid, - {get_positions, SampleSize, Index, Acc}, - infinity). + gen_statem:call(Pid, + {get_positions, SampleSize, Index, Acc}, infinity). -spec cdb_directfetch(pid(), list(), key_only|key_size|key_value_check) -> list(). @@ -307,26 +288,26 @@ cdb_getpositions_fromidx(Pid, SampleSize, Index, Acc) -> %% key_value_check (with the check part indicating if the CRC is correct for %% the value) cdb_directfetch(Pid, PositionList, Info) -> - gen_fsm:sync_send_event(Pid, {direct_fetch, PositionList, Info}, infinity). + gen_statem:call(Pid, {direct_fetch, PositionList, Info}, infinity). -spec cdb_close(pid()) -> ok. %% @doc %% RONSEAL cdb_close(Pid) -> - gen_fsm:sync_send_all_state_event(Pid, cdb_close, infinity). + gen_statem:call(Pid, cdb_close, infinity). -spec cdb_deleteconfirmed(pid()) -> ok. %% @doc %% Delete has been confirmed, so close (state should be delete_pending) cdb_deleteconfirmed(Pid) -> - gen_fsm:send_event(Pid, delete_confirmed). + gen_statem:cast(Pid, delete_confirmed). -spec cdb_complete(pid()) -> {ok, string()}. %% @doc %% Persists the hashtable to the end of the file, to close it for further %% writing then exit. Returns the filename that was saved. cdb_complete(Pid) -> - gen_fsm:sync_send_event(Pid, cdb_complete, infinity). + gen_statem:call(Pid, cdb_complete, infinity). -spec cdb_roll(pid()) -> ok. %% @doc @@ -335,7 +316,7 @@ cdb_complete(Pid) -> %% rolling state whilst the hashtable is being written, and will become a %% reader (read-only) CDB file process on completion cdb_roll(Pid) -> - gen_fsm:send_event(Pid, cdb_roll). + gen_statem:cast(Pid, cdb_roll). -spec cdb_returnhashtable(pid(), list(), binary()) -> ok. %% @doc @@ -345,20 +326,20 @@ cdb_roll(Pid) -> %% [{Index, CurrPos, IndexLength}] which can be used to locate the slices of %% the hashtree within that binary cdb_returnhashtable(Pid, IndexList, HashTreeBin) -> - gen_fsm:sync_send_event(Pid, {return_hashtable, IndexList, HashTreeBin}, infinity). + gen_statem:call(Pid, {return_hashtable, IndexList, HashTreeBin}, infinity). -spec cdb_checkhashtable(pid()) -> boolean(). %% @doc %% Hash the hashtable been written for this file? cdb_checkhashtable(Pid) -> - % only used in tests - so OK to be sync_send_event/2 - gen_fsm:sync_send_event(Pid, check_hashtable). + % only used in tests - so OK to be call + gen_statem:call(Pid, check_hashtable). -spec cdb_destroy(pid()) -> ok. %% @doc %% If the file is in a delete_pending state close (and will destroy) cdb_destroy(Pid) -> - gen_fsm:send_event(Pid, destroy). + gen_statem:cast(Pid, destroy). cdb_deletepending(Pid) -> % Only used in unit tests @@ -373,7 +354,7 @@ cdb_deletepending(Pid) -> %% Passing no_poll means there's no inker to poll, and the process will close %% on timeout rather than poll. cdb_deletepending(Pid, ManSQN, Inker) -> - gen_fsm:send_event(Pid, {delete_pending, ManSQN, Inker}). + gen_statem:cast(Pid, {delete_pending, ManSQN, Inker}). -spec cdb_scan( pid(), filter_fun(), any(), integer()|undefined) @@ -386,63 +367,60 @@ cdb_deletepending(Pid, ManSQN, Inker) -> %% LastPosition could be the atom complete when the last key processed was at %% the end of the file. last_key must be defined in LoopState. cdb_scan(Pid, FilterFun, InitAcc, StartPosition) -> - gen_fsm:sync_send_all_state_event(Pid, - {cdb_scan, - FilterFun, - InitAcc, - StartPosition}, - infinity). + gen_statem:call(Pid, + {cdb_scan, FilterFun, InitAcc, StartPosition}, + infinity). -spec cdb_lastkey(pid()) -> any(). %% @doc %% Get the last key to be added to the file (which will have the highest %% sequence number) cdb_lastkey(Pid) -> - gen_fsm:sync_send_all_state_event(Pid, cdb_lastkey, infinity). + gen_statem:call(Pid, cdb_lastkey, infinity). -spec cdb_firstkey(pid()) -> any(). cdb_firstkey(Pid) -> - gen_fsm:sync_send_all_state_event(Pid, cdb_firstkey, infinity). + gen_statem:call(Pid, cdb_firstkey, infinity). -spec cdb_filename(pid()) -> string(). %% @doc %% Get the filename of the database cdb_filename(Pid) -> - gen_fsm:sync_send_all_state_event(Pid, cdb_filename, infinity). + gen_statem:call(Pid, cdb_filename, infinity). -spec cdb_keycheck(pid(), any()) -> probably|missing. %% @doc %% Check to see if the key is probably present, will return either %% probably or missing. Does not do a definitive check cdb_keycheck(Pid, Key) -> - gen_fsm:sync_send_event(Pid, {key_check, Key}, infinity). + gen_statem:call(Pid, {key_check, Key}, infinity). -spec cdb_isrolling(pid()) -> boolean(). %% @doc %% Check to see if a cdb file is still rolling cdb_isrolling(Pid) -> - gen_fsm:sync_send_all_state_event(Pid, cdb_isrolling, infinity). + gen_statem:call(Pid, cdb_isrolling, infinity). -spec cdb_clerkcomplete(pid()) -> ok. %% @doc %% When an Inker's clerk has finished with a CDB process, then it will call %% complete. Currently this will prompt hibernation, as the CDB process may -%% not be needed for a period. +%% not be needed for a period. cdb_clerkcomplete(Pid) -> - gen_fsm:send_all_state_event(Pid, clerk_complete). + gen_statem:cast(Pid, clerk_complete). -spec cdb_getcachedscore(pid(), erlang:timestamp()) -> undefined|float(). %% @doc %% Return the cached score for a CDB file cdb_getcachedscore(Pid, Now) -> - gen_fsm:sync_send_all_state_event(Pid, {get_cachedscore, Now}, infinity). + gen_statem:call(Pid, {get_cachedscore, Now}, infinity). -spec cdb_putcachedscore(pid(), float()) -> ok. %% @doc %% Return the cached score for a CDB file cdb_putcachedscore(Pid, Score) -> - gen_fsm:sync_send_all_state_event(Pid, {put_cachedscore, Score}, infinity). + gen_statem:call(Pid, {put_cachedscore, Score}, infinity). @@ -451,7 +429,7 @@ cdb_putcachedscore(Pid, Score) -> %%%============================================================================ init([Opts]) -> - MaxSize = + MaxSize = case Opts#cdb_options.max_size of undefined -> ?MAX_FILE_SIZE; @@ -475,7 +453,10 @@ init([Opts]) -> log_options=Opts#cdb_options.log_options, monitor=Opts#cdb_options.monitor}}. -starting({open_writer, Filename}, _From, State) -> +callback_mode() -> + state_functions. + +starting({call, From}, {open_writer, Filename}, State) -> leveled_log:save(State#state.log_options), leveled_log:log(cdb01, [Filename]), {LastPosition, HashTree, LastKey} = open_active_file(Filename), @@ -489,8 +470,8 @@ starting({open_writer, Filename}, _From, State) -> last_key=LastKey, filename=Filename, hashtree=HashTree}, - {reply, ok, writer, State0, hibernate}; -starting({open_reader, Filename}, _From, State) -> + {next_state, writer, State0, [{reply, From, ok}, hibernate]}; +starting({call, From}, {open_reader, Filename}, State) -> leveled_log:save(State#state.log_options), leveled_log:log(cdb02, [Filename]), {Handle, Index, LastKey} = open_for_readonly(Filename, false), @@ -498,8 +479,8 @@ starting({open_reader, Filename}, _From, State) -> last_key=LastKey, filename=Filename, hash_index=Index}, - {reply, ok, reader, State0, hibernate}; -starting({open_reader, Filename, LastKey}, _From, State) -> + {next_state, reader, State0, [{reply, From, ok}, hibernate]}; +starting({call, From}, {open_reader, Filename, LastKey}, State) -> leveled_log:save(State#state.log_options), leveled_log:log(cdb02, [Filename]), {Handle, Index, LastKey} = open_for_readonly(Filename, LastKey), @@ -507,30 +488,33 @@ starting({open_reader, Filename, LastKey}, _From, State) -> last_key=LastKey, filename=Filename, hash_index=Index}, - {reply, ok, reader, State0, hibernate}. + {next_state, reader, State0, [{reply, From, ok}, hibernate]}. -writer({get_kv, Key}, _From, State) -> - {reply, - get_mem(Key, - State#state.handle, - State#state.hashtree, - State#state.binary_mode), - writer, - State}; -writer({key_check, Key}, _From, State) -> - {reply, - get_mem(Key, - State#state.handle, - State#state.hashtree, - State#state.binary_mode, - loose_presence), - writer, - State}; -writer({put_kv, Key, Value, Sync}, _From, State) -> + +writer({call, From}, {get_kv, Key}, State) -> + {keep_state_and_data, + [{reply, + From, + get_mem( + Key, + State#state.handle, + State#state.hashtree, + State#state.binary_mode)}]}; +writer({call, From}, {key_check, Key}, State) -> + {keep_state_and_data, + [{reply, + From, + get_mem( + Key, + State#state.handle, + State#state.hashtree, + State#state.binary_mode, + loose_presence)}]}; +writer({call, From}, {put_kv, Key, Value, Sync}, State) -> NewCount = State#state.current_count + 1, case NewCount >= State#state.max_count of true -> - {reply, roll, writer, State}; + {keep_state_and_data, [{reply, From, roll}]}; false -> Result = put(State#state.handle, Key, @@ -542,7 +526,7 @@ writer({put_kv, Key, Value, Sync}, _From, State) -> case Result of roll -> %% Key and value could not be written - {reply, roll, writer, State}; + {keep_state_and_data, [{reply, From, roll}]}; {UpdHandle, NewPosition, HashTree} -> ok = case {State#state.sync_strategy, Sync} of @@ -553,22 +537,25 @@ writer({put_kv, Key, Value, Sync}, _From, State) -> _ -> ok end, - {reply, ok, writer, State#state{handle=UpdHandle, - current_count=NewCount, - last_position=NewPosition, - last_key=Key, - hashtree=HashTree}} + {keep_state, + State#state{ + handle=UpdHandle, + current_count=NewCount, + last_position=NewPosition, + last_key=Key, + hashtree=HashTree}, + [{reply, From, ok}]} end end; -writer({mput_kv, []}, _From, State) -> - {reply, ok, writer, State}; -writer({mput_kv, KVList}, _From, State) -> +writer({call, From}, {mput_kv, []}, _State) -> + {keep_state_and_data, [{reply, From, ok}]}; +writer({call, From}, {mput_kv, KVList}, State) -> NewCount = State#state.current_count + length(KVList), TooMany = NewCount >= State#state.max_count, NotEmpty = State#state.current_count > 0, case (TooMany and NotEmpty) of true -> - {reply, roll, writer, State}; + {keep_state_and_data, [{reply, From, roll}]}; false -> Result = mput(State#state.handle, KVList, @@ -578,53 +565,61 @@ writer({mput_kv, KVList}, _From, State) -> case Result of roll -> %% Keys and values could not be written - {reply, roll, writer, State}; + {keep_state_and_data, [{reply, From, roll}]}; {UpdHandle, NewPosition, HashTree, LastKey} -> - {reply, ok, writer, State#state{handle=UpdHandle, - current_count=NewCount, - last_position=NewPosition, - last_key=LastKey, - hashtree=HashTree}} + {keep_state, + State#state{ + handle=UpdHandle, + current_count=NewCount, + last_position=NewPosition, + last_key=LastKey, + hashtree=HashTree}, + [{reply, From, ok}]} end end; -writer(cdb_complete, _From, State) -> +writer({call, From}, cdb_complete, State) -> NewName = determine_new_filename(State#state.filename), ok = close_file(State#state.handle, State#state.hashtree, State#state.last_position), ok = rename_for_read(State#state.filename, NewName), - {stop, normal, {ok, NewName}, State}. - -writer(cdb_roll, State) -> - ok = leveled_iclerk:clerk_hashtablecalc(State#state.hashtree, - State#state.last_position, - self()), + {stop_and_reply, normal, [{reply, From, {ok, NewName}}]}; +writer({call, From}, Event, State) -> + handle_sync_event(Event, From, State); +writer(cast, cdb_roll, State) -> + ok = + leveled_iclerk:clerk_hashtablecalc( + State#state.hashtree, State#state.last_position, self()), {next_state, rolling, State}. -rolling({get_kv, Key}, _From, State) -> - {reply, - get_mem(Key, - State#state.handle, - State#state.hashtree, - State#state.binary_mode), - rolling, - State}; -rolling({key_check, Key}, _From, State) -> - {reply, - get_mem(Key, - State#state.handle, - State#state.hashtree, - State#state.binary_mode, - loose_presence), - rolling, - State}; -rolling({get_positions, _SampleSize, _Index, SampleAcc}, _From, State) -> - {reply, SampleAcc, rolling, State}; -rolling({return_hashtable, IndexList, HashTreeBin}, _From, State) -> +rolling({call, From}, {get_kv, Key}, State) -> + {keep_state_and_data, + [{reply, + From, + get_mem( + Key, + State#state.handle, + State#state.hashtree, + State#state.binary_mode)}]}; +rolling({call, From}, {key_check, Key}, State) -> + {keep_state_and_data, + [{reply, + From, + get_mem( + Key, + State#state.handle, + State#state.hashtree, + State#state.binary_mode, + loose_presence)}]}; +rolling({call, From}, + {get_positions, _SampleSize, _Index, SampleAcc}, + _State) -> + {keep_state_and_data, [{reply, From, SampleAcc}]}; +rolling({call, From}, {return_hashtable, IndexList, HashTreeBin}, State) -> SW = os:timestamp(), Handle = State#state.handle, - {ok, BasePos} = file:position(Handle, State#state.last_position), + {ok, BasePos} = file:position(Handle, State#state.last_position), NewName = determine_new_filename(State#state.filename), ok = perform_write_hash_tables(Handle, HashTreeBin, BasePos), ok = write_top_index_table(Handle, BasePos, IndexList), @@ -632,54 +627,57 @@ rolling({return_hashtable, IndexList, HashTreeBin}, _From, State) -> ok = rename_for_read(State#state.filename, NewName), leveled_log:log(cdb03, [NewName]), ets:delete(State#state.hashtree), - {NewHandle, Index, LastKey} = open_for_readonly(NewName, - State#state.last_key), + {NewHandle, Index, LastKey} = + open_for_readonly(NewName, State#state.last_key), State0 = State#state{handle=NewHandle, last_key=LastKey, filename=NewName, hash_index=Index}, case State#state.deferred_delete of true -> - {reply, ok, delete_pending, State0}; + {next_state, delete_pending, State0, [{reply, From, ok}]}; false -> leveled_log:log_timer(cdb18, [], SW), - {reply, ok, reader, State0, hibernate} + {next_state, reader, State0, [{reply, From, ok}, hibernate]} end; -rolling(check_hashtable, _From, State) -> - {reply, false, rolling, State}. - -rolling({delete_pending, ManSQN, Inker}, State) -> - {next_state, - rolling, +rolling({call, From}, check_hashtable, _State) -> + {keep_state_and_data, [{reply, From, false}]}; +rolling({call, From}, cdb_isrolling, _State) -> + {keep_state_and_data, [{reply, From, true}]}; +rolling({call, From}, Event, State) -> + handle_sync_event(Event, From, State); +rolling(cast, {delete_pending, ManSQN, Inker}, State) -> + {keep_state, State#state{delete_point=ManSQN, inker=Inker, deferred_delete=true}}. -reader({get_kv, Key}, _From, State) -> - Result = - get_withcache(State#state.handle, - Key, +reader({call, From}, {get_kv, Key}, State) -> + Result = + get_withcache(State#state.handle, + Key, State#state.hash_index, State#state.binary_mode, State#state.monitor), - {reply, Result, reader, State}; -reader({key_check, Key}, _From, State) -> - Result = - get_withcache(State#state.handle, - Key, + {keep_state_and_data, [{reply, From, Result}]}; +reader({call, From}, {key_check, Key}, State) -> + Result = + get_withcache(State#state.handle, + Key, State#state.hash_index, loose_presence, State#state.binary_mode, {no_monitor, 0}), - {reply, Result, reader, State}; -reader({get_positions, SampleSize, Index, Acc}, _From, State) -> + {keep_state_and_data, [{reply, From, Result}]}; +reader({call, From}, {get_positions, SampleSize, Index, Acc}, State) -> {Pos, Count} = element(Index + 1, State#state.hash_index), UpdAcc = scan_index_returnpositions(State#state.handle, Pos, Count, Acc), case SampleSize of all -> - {reply, UpdAcc, reader, State}; + {keep_state_and_data, [{reply, From, UpdAcc}]}; _ -> - {reply, lists:sublist(UpdAcc, SampleSize), reader, State} + {keep_state_and_data, + [{reply, From, lists:sublist(UpdAcc, SampleSize)}]} end; -reader({direct_fetch, PositionList, Info}, From, State) -> +reader({call, From}, {direct_fetch, PositionList, Info}, State) -> H = State#state.handle, FilterFalseKey = fun(Tpl) -> @@ -690,7 +688,7 @@ reader({direct_fetch, PositionList, Info}, From, State) -> {true, Tpl} end end, - + case Info of key_only -> FM = lists:filtermap( @@ -698,10 +696,12 @@ reader({direct_fetch, PositionList, Info}, From, State) -> FilterFalseKey(extract_key(H, P)) end, PositionList), MapFun = fun(T) -> element(1, T) end, - {reply, lists:map(MapFun, FM), reader, State}; + {keep_state_and_data, + [{reply, From, lists:map(MapFun, FM)}]}; key_size -> FilterFun = fun(P) -> FilterFalseKey(extract_key_size(H, P)) end, - {reply, lists:filtermap(FilterFun, PositionList), reader, State}; + {keep_state_and_data, + [{reply, From, lists:filtermap(FilterFun, PositionList)}]}; key_value_check -> BM = State#state.binary_mode, MapFun = fun(P) -> extract_key_value_check(H, P, BM) end, @@ -709,82 +709,88 @@ reader({direct_fetch, PositionList, Info}, From, State) -> % hibernate the process that is likely to be used again. However, % a significant amount of unused binary references may have % accumulated, so push a GC at this point - gen_fsm:reply(From, lists:map(MapFun, PositionList)), + gen_statem:reply(From, lists:map(MapFun, PositionList)), garbage_collect(), - {next_state, reader, State} + {keep_state_and_data, []} end; -reader(cdb_complete, _From, State) -> +reader({call, From}, cdb_complete, State) -> leveled_log:log(cdb05, [State#state.filename, reader, cdb_ccomplete]), ok = file:close(State#state.handle), - {stop, normal, {ok, State#state.filename}, State#state{handle=undefined}}; -reader(check_hashtable, _From, State) -> - {reply, true, reader, State}. - - -reader({delete_pending, 0, no_poll}, State) -> - {next_state, - delete_pending, - State#state{delete_point=0}}; -reader({delete_pending, ManSQN, Inker}, State) -> + {stop_and_reply, + normal, + [{reply, From, {ok, State#state.filename}}], + State#state{handle=undefined}}; +reader({call, From}, check_hashtable, _State) -> + {keep_state_and_data, [{reply, From, true}]}; +reader({call, From}, Event, State) -> + handle_sync_event(Event, From, State); +reader(cast, {delete_pending, 0, no_poll}, State) -> + {next_state, delete_pending, State#state{delete_point=0}}; +reader(cast, {delete_pending, ManSQN, Inker}, State) -> {next_state, delete_pending, State#state{delete_point=ManSQN, inker=Inker}, - ?DELETE_TIMEOUT}. + ?DELETE_TIMEOUT}; +reader(cast, clerk_complete, _State) -> + {keep_state_and_data, [hibernate]}. -delete_pending({get_kv, Key}, _From, State) -> - Result = +delete_pending({call, From}, {get_kv, Key}, State) -> + Result = get_withcache(State#state.handle, Key, State#state.hash_index, State#state.binary_mode, State#state.monitor), - {reply, Result, delete_pending, State, ?DELETE_TIMEOUT}; -delete_pending({key_check, Key}, _From, State) -> - Result = + {keep_state_and_data, [{reply, From, Result}, ?DELETE_TIMEOUT]}; +delete_pending({call, From}, {key_check, Key}, State) -> + Result = get_withcache(State#state.handle, Key, State#state.hash_index, loose_presence, State#state.binary_mode, {no_monitor, 0}), - {reply, Result, delete_pending, State, ?DELETE_TIMEOUT}. - -delete_pending(timeout, State=#state{delete_point=ManSQN}) when ManSQN > 0 -> + {keep_state_and_data, [{reply, From, Result}, ?DELETE_TIMEOUT]}; +delete_pending({call, From}, cdb_close, State) -> + leveled_log:log(cdb05, [State#state.filename, delete_pending, cdb_close]), + close_pendingdelete(State#state.handle, + State#state.filename, + State#state.waste_path), + {stop_and_reply, normal, [{reply, From, ok}]}; +delete_pending(cast, delete_confirmed, State=#state{delete_point=ManSQN}) -> + leveled_log:log(cdb04, [State#state.filename, ManSQN]), + close_pendingdelete(State#state.handle, + State#state.filename, + State#state.waste_path), + {stop, normal}; +delete_pending(cast, destroy, State) -> + leveled_log:log(cdb05, [State#state.filename, delete_pending, destroy]), + close_pendingdelete(State#state.handle, + State#state.filename, + State#state.waste_path), + {stop, normal}; +delete_pending( + timeout, _, State=#state{delete_point=ManSQN}) when ManSQN > 0 -> case is_process_alive(State#state.inker) of true -> - ok = + ok = leveled_inker:ink_confirmdelete(State#state.inker, ManSQN, self()), - {next_state, delete_pending, State, ?DELETE_TIMEOUT}; + {keep_state_and_data, [?DELETE_TIMEOUT]}; false -> leveled_log:log(cdb04, [State#state.filename, ManSQN]), - close_pendingdelete(State#state.handle, - State#state.filename, + close_pendingdelete(State#state.handle, + State#state.filename, State#state.waste_path), - {stop, normal, State} - end; -delete_pending(delete_confirmed, State=#state{delete_point=ManSQN}) -> - leveled_log:log(cdb04, [State#state.filename, ManSQN]), - close_pendingdelete(State#state.handle, - State#state.filename, - State#state.waste_path), - {stop, normal, State}; -delete_pending(destroy, State) -> - leveled_log:log(cdb05, [State#state.filename, delete_pending, destroy]), - close_pendingdelete(State#state.handle, - State#state.filename, - State#state.waste_path), - {stop, normal, State}. + {stop, normal} + end. -handle_sync_event({cdb_scan, FilterFun, Acc, StartPos}, - From, - StateName, - State) -> +handle_sync_event({cdb_scan, FilterFun, Acc, StartPos}, From, State) -> {ok, EndPos0} = file:position(State#state.handle, eof), - {ok, StartPos0} = + {ok, StartPos0} = case StartPos of undefined -> file:position(State#state.handle, ?BASE_POSITION); @@ -795,7 +801,7 @@ handle_sync_event({cdb_scan, FilterFun, Acc, StartPos}, MaybeEnd = (check_last_key(State#state.last_key) == empty) or (StartPos0 >= (EndPos0 - ?DWORD_SIZE)), - {LastPosition, Acc2} = + {LastPosition, Acc2} = case MaybeEnd of true -> {eof, Acc}; @@ -806,7 +812,7 @@ handle_sync_event({cdb_scan, FilterFun, Acc, StartPos}, Acc, State#state.last_key) end, - % The scan may have created a lot of binary references, clear up the + % The scan may have created a lot of binary references, clear up the % reference counters for this process here manually. The cdb process % may be inactive for a period after the scan, and so GC may not kick in % otherwise @@ -814,12 +820,12 @@ handle_sync_event({cdb_scan, FilterFun, Acc, StartPos}, % garbage_collect/0 is used in preference to hibernate, as we're generally % scanning in batches at startup - so the process will be needed straight % away. - gen_fsm:reply(From, {LastPosition, Acc2}), + gen_statem:reply(From, {LastPosition, Acc2}), garbage_collect(), - {next_state, StateName, State}; -handle_sync_event(cdb_lastkey, _From, StateName, State) -> - {reply, State#state.last_key, StateName, State}; -handle_sync_event(cdb_firstkey, _From, StateName, State) -> + {keep_state_and_data, []}; +handle_sync_event(cdb_lastkey, From, State) -> + {keep_state_and_data, [{reply, From, State#state.last_key}]}; +handle_sync_event(cdb_firstkey, From, State) -> {ok, EOFPos} = file:position(State#state.handle, eof), FilterFun = fun(Key, _V, _P, _O, _Fun) -> {stop, Key} end, FirstKey = @@ -835,13 +841,12 @@ handle_sync_event(cdb_firstkey, _From, StateName, State) -> State#state.last_key), FirstScanKey end, - {reply, FirstKey, StateName, State}; -handle_sync_event(cdb_filename, _From, StateName, State) -> - {reply, State#state.filename, StateName, State}; -handle_sync_event(cdb_isrolling, _From, StateName, State) -> - {reply, StateName == rolling, StateName, State}; -handle_sync_event({get_cachedscore, {NowMega, NowSecs, _}}, - _From, StateName, State) -> + {keep_state_and_data, [{reply, From, FirstKey}]}; +handle_sync_event(cdb_filename, From, State) -> + {keep_state_and_data, [{reply, From, State#state.filename}]}; +handle_sync_event(cdb_isrolling, From, _State) -> + {keep_state_and_data, [{reply, From, false}]}; +handle_sync_event({get_cachedscore, {NowMega, NowSecs, _}}, From, State) -> ScoreToReturn = case State#state.cached_score of undefined -> @@ -855,28 +860,17 @@ handle_sync_event({get_cachedscore, {NowMega, NowSecs, _}}, Score end end, - {reply, ScoreToReturn, StateName, State}; -handle_sync_event({put_cachedscore, Score}, _From, StateName, State) -> - {reply, ok, StateName, State#state{cached_score = {Score,os:timestamp()}}}; -handle_sync_event(cdb_close, _From, delete_pending, State) -> - leveled_log:log(cdb05, [State#state.filename, delete_pending, cdb_close]), - close_pendingdelete(State#state.handle, - State#state.filename, - State#state.waste_path), - {stop, normal, ok, State}; -handle_sync_event(cdb_close, _From, _StateName, State) -> + {keep_state_and_data, [{reply, From, ScoreToReturn}]}; +handle_sync_event({put_cachedscore, Score}, From, State) -> + {keep_state, + State#state{cached_score = {Score,os:timestamp()}}, + [{reply, From, ok}]}; +handle_sync_event(cdb_close, From, State) -> file:close(State#state.handle), - {stop, normal, ok, State}. - -handle_event(clerk_complete, StateName, State) -> - {next_state, StateName, State, hibernate}. - -handle_info(_Msg, StateName, State) -> - {next_state, StateName, State}. + {stop_and_reply, normal, [{reply, From, ok}]}. terminate(_Reason, _StateName, _State) -> ok. - code_change(_OldVsn, StateName, State, _Extra) -> @@ -889,9 +883,9 @@ code_change(_OldVsn, StateName, State, _Extra) -> finished_rolling(CDB) -> - RollerFun = + RollerFun = fun(Sleep, FinishedRolling) -> - case FinishedRolling of + case FinishedRolling of true -> true; false -> @@ -908,13 +902,13 @@ finished_rolling(CDB) -> -spec close_pendingdelete(file:io_device(), list(), list()|undefined) -> ok. %% @doc -%% If delete is pending - then the close behaviour needs to actuallly delete +%% If delete is pending - then the close behaviour needs to actuallly delete %% the file close_pendingdelete(Handle, Filename, WasteFP) -> ok = file:close(Handle), case filelib:is_file(Filename) of true -> - case WasteFP of + case WasteFP of undefined -> ok = file:delete(Filename); WasteFP -> @@ -945,21 +939,21 @@ set_writeops(SyncStrategy) -> -spec open_active_file(list()) -> {integer(), ets:tid(), any()}. %% @doc -%% Open an active file - one for which it is assumed the hash tables have not +%% Open an active file - one for which it is assumed the hash tables have not %% yet been written %% -%% Needs to scan over file to incrementally produce the hash list, starting at +%% Needs to scan over file to incrementally produce the hash list, starting at %% the end of the top index table. %% -%% Should return a dictionary keyed by index containing a list of {Hash, Pos} -%% tuples as the write_key_value_pairs function, and the current position, and +%% Should return a dictionary keyed by index containing a list of {Hash, Pos} +%% tuples as the write_key_value_pairs function, and the current position, and %% the file handle open_active_file(FileName) when is_list(FileName) -> {ok, Handle} = file:open(FileName, ?WRITE_OPS), {ok, Position} = file:position(Handle, {bof, 256 * ?DWORD_SIZE}), {LastPosition, {HashTree, LastKey}} = startup_scan_over_file(Handle, Position), - case file:position(Handle, eof) of + case file:position(Handle, eof) of {ok, LastPosition} -> ok = file:close(Handle); {ok, EndPosition} -> @@ -975,13 +969,13 @@ open_active_file(FileName) when is_list(FileName) -> end, {LastPosition, HashTree, LastKey}. --spec put(list()|file:io_device(), - any(), any(), +-spec put(list()|file:io_device(), + any(), any(), {integer(), ets:tid()}, boolean(), integer(), boolean()) -> roll|{file:io_device(), integer(), ets:tid()}. %% @doc %% put(Handle, Key, Value, {LastPosition, HashDict}) -> {NewPosition, KeyDict} -%% Append to an active file a new key/value pair returning an updated +%% Append to an active file a new key/value pair returning an updated %% dictionary of Keys and positions. Returns an updated Position %% put(FileName, @@ -992,9 +986,9 @@ put(FileName, MaxSize, IsEmpty) when is_list(FileName) -> {ok, Handle} = file:open(FileName, ?WRITE_OPS), - put(Handle, Key, Value, {LastPosition, HashTree}, + put(Handle, Key, Value, {LastPosition, HashTree}, BinaryMode, MaxSize, IsEmpty); -put(Handle, Key, Value, {LastPosition, HashTree}, +put(Handle, Key, Value, {LastPosition, HashTree}, BinaryMode, MaxSize, IsEmpty) -> Bin = key_value_to_record({Key, Value}, BinaryMode), ObjectSize = byte_size(Bin), @@ -1014,13 +1008,13 @@ put(Handle, Key, Value, {LastPosition, HashTree}, end. --spec mput(file:io_device(), - list(tuple()), +-spec mput(file:io_device(), + list(tuple()), {integer(), ets:tid()}, boolean(), integer()) -> roll|{file:io_device(), integer(), ets:tid(), any()}. %% @doc %% Multiple puts - either all will succeed or it will return roll with non -%% succeeding. +%% succeeding. mput(Handle, KVList, {LastPosition, HashTree0}, BinaryMode, MaxSize) -> {KPList, Bin, LastKey} = multi_key_value_to_record(KVList, BinaryMode, @@ -1045,9 +1039,9 @@ mput(Handle, KVList, {LastPosition, HashTree0}, BinaryMode, MaxSize) -> leveled_monitor:monitor()) -> missing|probably|tuple(). %% @doc %% -%% Using a cache of the Index array - get a K/V pair from the file using the -%% Key. should return an updated timings object (if timings are being taken) -%% along with the result (which may be missing if the no matching entry is +%% Using a cache of the Index array - get a K/V pair from the file using the +%% Key. should return an updated timings object (if timings are being taken) +%% along with the result (which may be missing if the no matching entry is %% found, or probably in QuickCheck scenarios) get_withcache(Handle, Key, Cache, BinaryMode, Monitor) -> get(Handle, Key, Cache, true, BinaryMode, Monitor). @@ -1060,24 +1054,24 @@ get(FileNameOrHandle, Key, BinaryMode) -> -spec get( - list()|file:io_device(), - any(), no_cache|tuple(), - loose_presence|any(), + list()|file:io_device(), + any(), no_cache|tuple(), + loose_presence|any(), boolean(), leveled_monitor:monitor()) -> tuple()|probably|missing. %% @doc %% -%% Get a K/V pair from the file using the Key. QuickCheck can be set to -%% loose_presence if all is required is a loose check of presence (that the -%% Key is probably present as there is a hash in the hash table which matches +%% Get a K/V pair from the file using the Key. QuickCheck can be set to +%% loose_presence if all is required is a loose check of presence (that the +%% Key is probably present as there is a hash in the hash table which matches %% that Key) %% %% Timings also passed in and can be updated based on results -get(FileName, Key, Cache, QuickCheck, BinaryMode, Monitor) +get(FileName, Key, Cache, QuickCheck, BinaryMode, Monitor) when is_list(FileName) -> {ok, Handle} = file:open(FileName,[binary, raw, read]), get(Handle, Key, Cache, QuickCheck, BinaryMode, Monitor); -get(Handle, Key, Cache, QuickCheck, BinaryMode, Monitor) +get(Handle, Key, Cache, QuickCheck, BinaryMode, Monitor) when is_tuple(Handle) -> SW0 = leveled_monitor:maybe_time(Monitor), Hash = hash(Key), @@ -1090,7 +1084,7 @@ get(Handle, Key, Cache, QuickCheck, BinaryMode, Monitor) missing; _ -> % Get starting slot in hashtable - {ok, FirstHashPosition} = + {ok, FirstHashPosition} = file:position(Handle, {bof, HashTable}), Slot = hash_to_slot(Hash, Count), {CycleCount, Result} = @@ -1113,7 +1107,7 @@ get_index(Handle, Index, no_cache) -> get_index(_Handle, Index, Cache) -> element(Index + 1, Cache). --spec get_mem(any(), list()|file:io_device(), ets:tid(), boolean()) -> +-spec get_mem(any(), list()|file:io_device(), ets:tid(), boolean()) -> tuple()|probably|missing. %% @doc %% Get a Key/Value pair from an active CDB file (with no hash table written) @@ -1152,17 +1146,17 @@ hashtable_calc(HashTree, StartPos) -> determine_new_filename(Filename) -> filename:rootname(Filename, ".pnd") ++ ".cdb". - + rename_for_read(Filename, NewName) -> %% Rename file leveled_log:log(cdb08, [Filename, NewName, filelib:is_file(NewName)]), file:rename(Filename, NewName). --spec open_for_readonly(string(), term()) +-spec open_for_readonly(string(), term()) -> {file:io_device(), hashtable_index(), term()}. %% @doc -%% Open a CDB file to accept read requests (e.g. key/value lookups) but no +%% Open a CDB file to accept read requests (e.g. key/value lookups) but no %% additions or changes open_for_readonly(Filename, LastKeyKnown) -> {ok, Handle} = file:open(Filename, [binary, raw, read]), @@ -1238,7 +1232,7 @@ scan_index_returnpositions(Handle, Position, Count, PosList0) -> %% Take an active file and write the hash details necessary to close that -%% file and roll a new active file if requested. +%% file and roll a new active file if requested. %% %% Base Pos should be at the end of the KV pairs written (the position for) %% the hash tables @@ -1255,12 +1249,12 @@ get_hashtree(Key, HashTree) -> Index = hash_to_index(Hash), lookup_positions(HashTree, Index, Hash). -%% Add to hash tree - this is an array of 256 skiplists that contains the Hash +%% Add to hash tree - this is an array of 256 skiplists that contains the Hash %% and position of objects which have been added to an open CDB file put_hashtree(Key, Position, HashTree) -> Hash = hash(Key), Index = hash_to_index(Hash), - add_position_tohashtree(HashTree, Index, Hash, Position). + add_position_tohashtree(HashTree, Index, Hash, Position). %% Function to extract a Key-Value pair given a file handle and a position %% Will confirm that the key matches and do a CRC check @@ -1272,7 +1266,7 @@ extract_kvpair(Handle, [Position|Rest], Key, BinaryMode) -> case safe_read_next_keybin(Handle, KeyLength) of {Key, KeyBin} -> % If same key as passed in, then found! case checkread_next_value(Handle, ValueLength, KeyBin) of - {false, _} -> + {false, _} -> crc_wonky; {_, Value} -> case BinaryMode of @@ -1299,7 +1293,7 @@ extract_key_size(Handle, Position) -> extract_key_value_check(Handle, Position, BinaryMode) -> {ok, _} = file:position(Handle, Position), - case {BinaryMode, saferead_keyvalue(Handle)} of + case {BinaryMode, saferead_keyvalue(Handle)} of {_, false} -> {null, crc_wonky, false}; {true, {Key, Value, _KeyL, _ValueL}} -> @@ -1309,10 +1303,10 @@ extract_key_value_check(Handle, Position, BinaryMode) -> end. --spec startup_scan_over_file(file:io_device(), file_location()) +-spec startup_scan_over_file(file:io_device(), file_location()) -> {file_location(), any()}. %% @doc -%% Scan through the file until there is a failure to crc check an input, and +%% Scan through the file until there is a failure to crc check an input, and %% at that point return the position and the key dictionary scanned so far startup_scan_over_file(Handle, Position) -> HashTree = new_hashtree(), @@ -1391,22 +1385,22 @@ check_last_key(_LK) -> ok. --spec saferead_keyvalue(file:io_device()) +-spec saferead_keyvalue(file:io_device()) -> false|{any(), any(), integer(), integer()}. %% @doc %% Read the Key/Value at this point, returning {ok, Key, Value} -%% catch expected exceptions associated with file corruption (or end) and +%% catch expected exceptions associated with file corruption (or end) and %% return eof saferead_keyvalue(Handle) -> - case read_next_2_integers(Handle) of + case read_next_2_integers(Handle) of eof -> false; {KeyL, ValueL} when is_integer(KeyL), is_integer(ValueL) -> - case safe_read_next_keybin(Handle, KeyL) of + case safe_read_next_keybin(Handle, KeyL) of false -> false; {Key, KeyBin} -> - case safe_read_next_value(Handle, ValueL, KeyBin) of + case safe_read_next_value(Handle, ValueL, KeyBin) of false -> false; TrueValue -> @@ -1421,27 +1415,27 @@ saferead_keyvalue(Handle) -> -spec safe_read_next_key(file:io_device(), integer()) -> false|term(). %% @doc -%% Return the next key or have false returned if there is some sort of +%% Return the next key or have false returned if there is some sort of %% potentially expected error (e.g. due to file truncation). Note that no %% CRC check has been performed safe_read_next_key(Handle, Length) -> ReadFun = fun(Bin) -> binary_to_term(Bin) end, safe_read_next(Handle, Length, ReadFun). --spec safe_read_next_keybin(file:io_device(), integer()) +-spec safe_read_next_keybin(file:io_device(), integer()) -> false|{term(), binary()}. %% @doc -%% Return the next key or have false returned if there is some sort of +%% Return the next key or have false returned if there is some sort of %% potentially expected error (e.g. due to file truncation). Note that no %% CRC check has been performed %% Returns both the Key and the Binary version, the binary version being -%% required for the CRC checking after the value fetch (see +%% required for the CRC checking after the value fetch (see %% safe_read_next_value/3) safe_read_next_keybin(Handle, Length) -> ReadFun = fun(Bin) -> {binary_to_term(Bin), Bin} end, safe_read_next(Handle, Length, ReadFun). --spec safe_read_next_value(file:io_device(), integer(), binary()) +-spec safe_read_next_value(file:io_device(), integer(), binary()) -> binary()|false. safe_read_next_value(Handle, Length, KeyBin) -> ReadFun = fun(VBin) -> crccheck(VBin, KeyBin) end, @@ -1457,7 +1451,7 @@ safe_read_next_value(Handle, Length, KeyBin) -> %% Previously catching error:badarg was sufficient to capture errors of %% corruption, but on some OS versions may need to catch error:einval as well safe_read_next(Handle, Length, ReadFun) -> - try + try loose_read(Handle, Length, ReadFun) catch error:ReadError -> @@ -1467,7 +1461,7 @@ safe_read_next(Handle, Length, ReadFun) -> -spec loose_read(file:io_device(), integer(), read_fun()) -> read_output(). %% @doc -%% Read with minimal error handling (only eof) - to be wrapped in +%% Read with minimal error handling (only eof) - to be wrapped in %% safe_read_next/3 to catch exceptions. loose_read(Handle, Length, ReadFun) -> case file:read(Handle, Length) of @@ -1484,10 +1478,10 @@ loose_read(Handle, Length, ReadFun) -> %% are a CRC check. If the binary is truncated, it could be a bitstring or %% less than 4 bytes - in which case return false to recognise the corruption. crccheck(<>, KeyBin) when is_binary(KeyBin) -> - case calc_crc(KeyBin, Value) of - CRC -> + case calc_crc(KeyBin, Value) of + CRC -> Value; - _ -> + _ -> leveled_log:log(cdb10, ["mismatch"]), false end; @@ -1502,14 +1496,14 @@ crccheck(_V, _KB) -> calc_crc(KeyBin, Value) -> erlang:crc32(<>). --spec checkread_next_value(file:io_device(), integer(), binary()) +-spec checkread_next_value(file:io_device(), integer(), binary()) -> {boolean(), binary()|crc_wonky}. %% @doc -%% Read next string where the string has a CRC prepended - stripping the crc +%% Read next string where the string has a CRC prepended - stripping the crc %% and checking if requested checkread_next_value(Handle, Length, KeyBin) -> {ok, <>} = file:read(Handle, Length), - case calc_crc(KeyBin, Value) of + case calc_crc(KeyBin, Value) of CRC -> {true, Value}; _ -> @@ -1521,9 +1515,9 @@ extract_valueandsize(ValueAsBin) -> {ValueAsBin, byte_size(ValueAsBin)}. -%% Used for reading lengths with CDB +%% Used for reading lengths with CDB read_next_2_integers(Handle) -> - case file:read(Handle, ?DWORD_SIZE) of + case file:read(Handle, ?DWORD_SIZE) of {ok, <>} -> {Int1, Int2}; ReadError -> @@ -1543,40 +1537,40 @@ read_integerpairs(< {pos_integer(), missing|probably|tuple()}. %% @doc %% -%% Seach the hash table for the matching hash and key. Be prepared for +%% Seach the hash table for the matching hash and key. Be prepared for %% multiple keys to have the same hash value. %% %% There are three possible values of CRCCheck: %% true - check the CRC before returning key & value %% false - don't check the CRC before returning key & value %% loose_presence - confirm that the hash of the key is present -search_hash_table(_Handle, - {_, _, TotalSlots, TotalSlots}, +search_hash_table(_Handle, + {_, _, TotalSlots, TotalSlots}, _Hash, _Key, - _QuickCheck, _BinaryMode) -> + _QuickCheck, _BinaryMode) -> % We have done the full loop - value must not be present {TotalSlots, missing}; -search_hash_table(Handle, - {FirstHashPosition, Slot, CycleCount, TotalSlots}, +search_hash_table(Handle, + {FirstHashPosition, Slot, CycleCount, TotalSlots}, Hash, Key, QuickCheck, BinaryMode) -> - % Read the next 2 integers at current position, see if it matches the hash + % Read the next 2 integers at current position, see if it matches the hash % we're after - Offset = - ((Slot + CycleCount - 1) rem TotalSlots) * ?DWORD_SIZE + Offset = + ((Slot + CycleCount - 1) rem TotalSlots) * ?DWORD_SIZE + FirstHashPosition, {ok, _} = file:position(Handle, Offset), - + case read_next_2_integers(Handle) of {0, 0} -> {CycleCount, missing}; {Hash, DataLoc} -> - KV = + KV = case QuickCheck of loose_presence -> probably; @@ -1592,12 +1586,12 @@ search_hash_table(Handle, Hash, Key, QuickCheck, BinaryMode); _ -> - {CycleCount, KV} + {CycleCount, KV} end; _ -> search_hash_table( - Handle, - {FirstHashPosition, Slot, CycleCount + 1, TotalSlots}, + Handle, + {FirstHashPosition, Slot, CycleCount + 1, TotalSlots}, Hash, Key, QuickCheck, BinaryMode) end. @@ -1621,7 +1615,7 @@ maybelog_get_timing({Pid, _StatsFreq}, IndexTime, ReadTime, CycleCount) -> % % Returns a dictionary that is keyed by % the least significant 8 bits of each hash with the -% values being a list of the hash and the position of the +% values being a list of the hash and the position of the % key/value binary in the file. write_key_value_pairs(Handle, KeyValueList) -> {ok, Position} = file:position(Handle, cur), @@ -1630,14 +1624,14 @@ write_key_value_pairs(Handle, KeyValueList) -> write_key_value_pairs(_, [], Acc) -> Acc; -write_key_value_pairs(Handle, [HeadPair|TailList], Acc) -> +write_key_value_pairs(Handle, [HeadPair|TailList], Acc) -> {Key, Value} = HeadPair, {Handle, NewPosition, HashTree} = put(Handle, Key, Value, Acc), write_key_value_pairs(Handle, TailList, {NewPosition, HashTree}). %% Write the actual hashtables at the bottom of the file. Each hash table -%% entry is a doubleword in length. The first word is the hash value -%% corresponding to a key and the second word is a file pointer to the +%% entry is a doubleword in length. The first word is the hash value +%% corresponding to a key and the second word is a file pointer to the %% corresponding {key,value} tuple. write_hash_tables(Handle, HashTree) -> {ok, StartPos} = file:position(Handle, cur), @@ -1654,12 +1648,12 @@ perform_write_hash_tables(Handle, HashTreeBin, StartPos) -> ok. -%% Write the top most 255 doubleword entries. First word is the -%% file pointer to a hashtable and the second word is the number of entries +%% Write the top most 255 doubleword entries. First word is the +%% file pointer to a hashtable and the second word is the number of entries %% in the hash table %% The List passed in should be made up of {Index, Position, Count} tuples write_top_index_table(Handle, BasePos, IndexList) -> - FnWriteIndex = + FnWriteIndex = fun({_Index, Pos, Count}, {AccBin, CurrPos}) -> {Position, NextPos} = case Count == 0 of @@ -1667,13 +1661,13 @@ write_top_index_table(Handle, BasePos, IndexList) -> {CurrPos, CurrPos}; false -> {Pos, Pos + (Count * ?DWORD_SIZE)} - end, + end, {<>, NextPos} end, - + {IndexBin, _Pos} = lists:foldl(FnWriteIndex, {<<>>, BasePos}, IndexList), @@ -1706,7 +1700,7 @@ key_value_to_record({Key, Value}, BinaryMode) -> KS = byte_size(BK), VS = byte_size(BV), CRC = calc_crc(BK, BV), - <>. @@ -1727,7 +1721,7 @@ lookup_positions(HashTree, Index, Hash) -> lookup_positions(HashTree, Index, Hash, -1, []). lookup_positions(HashTree, Index, Hash, Pos, PosList) -> - case ets:next(HashTree, {Index, Hash, Pos}) of + case ets:next(HashTree, {Index, Hash, Pos}) of {Index, Hash, NewPos} -> lookup_positions(HashTree, Index, Hash, NewPos, [NewPos|PosList]); _ -> @@ -1748,7 +1742,7 @@ to_list(HashTree, Index) -> to_list(HashTree, Index, {0, -1}, []). to_list(HashTree, Index, {LastHash, LastPos}, Acc) -> - case ets:next(HashTree, {Index, LastHash, LastPos}) of + case ets:next(HashTree, {Index, LastHash, LastPos}) of {Index, Hash, Pos} -> to_list(HashTree, Index, {Hash, Pos}, [{Hash, Pos}|Acc]); _ -> @@ -1813,12 +1807,12 @@ find_firstzero(Bin, Pos) -> _ -> find_firstzero(Bin, Pos - 1) end. - + write_hash_tables(Indexes, HashTree, CurrPos) -> write_hash_tables(Indexes, HashTree, CurrPos, CurrPos, [], [], {0, 0, 0}). -write_hash_tables([], _HashTree, _CurrPos, _BasePos, +write_hash_tables([], _HashTree, _CurrPos, _BasePos, IndexList, HT_BinList, {T1, T2, T3}) -> leveled_log:log(cdb14, [T1, T2, T3]), IL = lists:reverse(IndexList), @@ -1828,7 +1822,7 @@ write_hash_tables([Index|Rest], HashTree, CurrPos, BasePos, SW1 = os:timestamp(), SlotMap = to_slotmap(HashTree, Index), T1 = timer:now_diff(os:timestamp(), SW1) + element(1, Timers), - case SlotMap of + case SlotMap of [] -> write_hash_tables(Rest, HashTree, @@ -1857,8 +1851,8 @@ write_hash_tables([Index|Rest], HashTree, CurrPos, BasePos, %%%%%%%%%%%%%%%% -% T E S T -%%%%%%%%%%%%%%% +% T E S T +%%%%%%%%%%%%%%% -ifdef(TEST). %% To make this compatible with original Bernstein format this endian flip @@ -1899,10 +1893,10 @@ dump(FileName) -> Fn = fun(Index, Acc) -> {ok, _} = file:position(Handle, ?DWORD_SIZE * Index), {_, Count} = read_next_2_integers(Handle), - Acc + Count + Acc + Count end, NumberOfPairs = lists:foldl(Fn, 0, lists:seq(0,255)) bsr 1, - io:format("Count of keys in db is ~w~n", [NumberOfPairs]), + io:format("Count of keys in db is ~w~n", [NumberOfPairs]), {ok, _} = file:position(Handle, {bof, ?BASE_POSITION}), Fn1 = fun(_I, Acc) -> {KL, VL} = read_next_2_integers(Handle), @@ -2001,7 +1995,7 @@ magickey_test() -> % prompts potential issue when first hash slot is empty but % hash is 0 - MagicKey = + MagicKey = {315781, stnd, {o_rkv, @@ -2019,7 +2013,7 @@ magickey_test() -> {ok, P1} = cdb_open_writer("test/test_area/magic_hash.pnd", #cdb_options{binary_mode=true}), - + ok = cdb_put(P1, MagicKey, <<"MagicV0">>), lists:foreach(fun({K, V}) -> cdb_put(P1, K, V) end, Set1), ok = cdb_put(P1, MagicKey, <<"MagicV1">>), @@ -2028,13 +2022,13 @@ magickey_test() -> {ok, P2} = cdb_open_reader(F2, #cdb_options{binary_mode=true}), {GetK, GetV} = cdb_get(P2, MagicKey), ?assertEqual(<<"MagicV1">>, GetV), - + AllKeys = cdb_directfetch(P2, cdb_getpositions(P2, all), key_only), ?assertMatch(true, lists:member(MagicKey, AllKeys)), ok = cdb_close(P2), ok = file:delete("test/test_area/magic_hash.cdb"), - + {ok, P3} = cdb_open_writer("test/test_area/magic_hash.pnd", #cdb_options{binary_mode=true}), @@ -2047,7 +2041,7 @@ magickey_test() -> ?assertEqual(<<"MagicV1">>, GetV), ok = cdb_close(P4), ok = file:delete("test/test_area/magic_hash.cdb"), - + {ok, P5} = cdb_open_writer("test/test_area/magic_hash.pnd", #cdb_options{binary_mode=true}), @@ -2091,7 +2085,7 @@ cyclecount_test() -> ok = cdb_close(P2), ok = file:delete("test/test_area/cycle_count.cdb"). - + full_1_test() -> List1 = lists:sort([{"key1","value1"},{"key2","value2"}]), @@ -2103,7 +2097,7 @@ full_1_test() -> full_2_test() -> List1 = lists:sort([{lists:flatten(io_lib:format("~s~p",[Prefix,Plug])), - lists:flatten(io_lib:format("value~p",[Plug]))} + lists:flatten(io_lib:format("value~p",[Plug]))} || Plug <- lists:seq(1,200), Prefix <- ["dsd","so39ds","oe9%#*(","020dkslsldclsldowlslf%$#", "tiep4||","qweq"]]), @@ -2137,7 +2131,7 @@ to_dict_test() -> ok = file:delete("test/test_area/from_dict_test1.cdb"). crccheck_emptyvalue_test() -> - ?assertMatch(false, crccheck(<<>>, <<"Key">>)). + ?assertMatch(false, crccheck(<<>>, <<"Key">>)). crccheck_shortvalue_test() -> Value = <<128,128,32>>, @@ -2176,7 +2170,7 @@ activewrite_singlewrite_test() -> io:format("New db file created ~n", []), {LastPosition, KeyDict, _} = open_active_file("test/test_area/test_mem.cdb"), - io:format("File opened as new active file " + io:format("File opened as new active file " "with LastPosition=~w ~n", [LastPosition]), {_, _, UpdKeyDict} = put("test/test_area/test_mem.cdb", Key, Value, @@ -2203,8 +2197,8 @@ activewrite_singlewrite_test() -> search_hash_table_findinslot_test() -> Key1 = "key1", % this is in slot 3 if count is 8 - D = dict:from_list([{Key1, "value1"}, {"K2", "V2"}, {"K3", "V3"}, - {"K4", "V4"}, {"K5", "V5"}, {"K6", "V6"}, {"K7", "V7"}, + D = dict:from_list([{Key1, "value1"}, {"K2", "V2"}, {"K3", "V3"}, + {"K4", "V4"}, {"K5", "V5"}, {"K6", "V6"}, {"K7", "V7"}, {"K8", "V8"}]), ok = from_dict("test/test_area/hashtable1_test.cdb",D), {ok, Handle} = file:open("test/test_area/hashtable1_test.cdb", @@ -2226,10 +2220,10 @@ search_hash_table_findinslot_test() -> ?assertMatch({"key1", "value1"}, get(Handle, Key1, false)), NoMonitor = {no_monitor, 0}, ?assertMatch( - probably, + probably, get(Handle, Key1, no_cache, loose_presence, false, NoMonitor)), ?assertMatch( - missing, + missing, get(Handle, "Key99", no_cache, loose_presence, false, NoMonitor)), {ok, _} = file:position(Handle, FirstHashPosition), FlipH3 = endian_flip(ReadH3), @@ -2239,7 +2233,7 @@ search_hash_table_findinslot_test() -> 0:32/integer, 0:32/integer>>, io:format("Replacement binary of ~w~n", [RBin]), - {ok, OldBin} = file:pread(Handle, + {ok, OldBin} = file:pread(Handle, FirstHashPosition + (Slot -1) * ?DWORD_SIZE, 16), io:format("Bin to be replaced is ~w ~n", [OldBin]), ok = file:pwrite(Handle, @@ -2278,7 +2272,7 @@ empty_roll_test() -> #cdb_options{binary_mode=true}), ok = cdb_roll(P1), true = finished_rolling(P1), - {ok, P2} = cdb_open_reader("test/test_area/empty_roll.cdb", + {ok, P2} = cdb_open_reader("test/test_area/empty_roll.cdb", #cdb_options{binary_mode=true}), ok = cdb_close(P2), ok = file:delete("test/test_area/empty_roll.cdb"). @@ -2361,9 +2355,9 @@ get_keys_byposition_manykeys_test_to() -> % Should not return positions when rolling % There is an implicit race here - if cdb_roll is too fast, then the test % will fail. It appears to be safe that if KeyCount is set to a high value - % (e.g. > 10K) it is implausible that cdb_roll will ever finish before the + % (e.g. > 10K) it is implausible that cdb_roll will ever finish before the % call to cdb_getpositions is executed. So the race is tolerated - ?assertMatch([], cdb_getpositions(P1, 10)), + ?assertMatch([], cdb_getpositions(P1, 10)), lists:foldl(fun(X, Complete) -> case Complete of true -> @@ -2381,20 +2375,20 @@ get_keys_byposition_manykeys_test_to() -> lists:seq(1, 30)), ?assertMatch(10, length(cdb_getpositions(P1, 10))), {ok, F2} = cdb_complete(P1), - + {ok, P2} = cdb_open_reader(F2, #cdb_options{binary_mode=false}), PositionList = cdb_getpositions(P2, all), L1 = length(PositionList), io:format("Length of all positions ~w~n", [L1]), ?assertMatch(KeyCount, L1), - + SampleList1 = cdb_getpositions(P2, 10), ?assertMatch(10, length(SampleList1)), SampleList2 = cdb_getpositions(P2, KeyCount), ?assertMatch(KeyCount, length(SampleList2)), SampleList3 = cdb_getpositions(P2, KeyCount + 1), ?assertMatch(KeyCount, length(SampleList3)), - + ?assertMatch(undefined, cdb_getcachedscore(P2, os:timestamp())), ok = cdb_putcachedscore(P2, 80.0), ?assertMatch(80.0, cdb_getcachedscore(P2, os:timestamp())), @@ -2470,38 +2464,38 @@ hashclash_test() -> ?assertMatch(22, hash(Key1)), ?assertMatch(22, hash(Key99)), ?assertMatch(22, hash(KeyNF)), - + ok = cdb_mput(P1, [{Key1, 1}, {Key99, 99}]), - + ?assertMatch(probably, cdb_keycheck(P1, Key1)), ?assertMatch(probably, cdb_keycheck(P1, Key99)), ?assertMatch(probably, cdb_keycheck(P1, KeyNF)), - + ?assertMatch({Key1, 1}, cdb_get(P1, Key1)), ?assertMatch({Key99, 99}, cdb_get(P1, Key99)), ?assertMatch(missing, cdb_get(P1, KeyNF)), - + {ok, FN} = cdb_complete(P1), {ok, P2} = cdb_open_reader(FN, #cdb_options{binary_mode=false}), - + ?assertMatch(probably, cdb_keycheck(P2, Key1)), ?assertMatch(probably, cdb_keycheck(P2, Key99)), ?assertMatch(probably, cdb_keycheck(P2, KeyNF)), - + ?assertMatch({Key1, 1}, cdb_get(P2, Key1)), ?assertMatch({Key99, 99}, cdb_get(P2, Key99)), ?assertMatch(missing, cdb_get(P2, KeyNF)), - + ok = cdb_deletepending(P2), - + ?assertMatch(probably, cdb_keycheck(P2, Key1)), ?assertMatch(probably, cdb_keycheck(P2, Key99)), ?assertMatch(probably, cdb_keycheck(P2, KeyNF)), - + ?assertMatch({Key1, 1}, cdb_get(P2, Key1)), ?assertMatch({Key99, 99}, cdb_get(P2, Key99)), ?assertMatch(missing, cdb_get(P2, KeyNF)), - + ok = cdb_close(P2). corruptfile_test() -> @@ -2559,7 +2553,7 @@ crc_corrupt_writer_test() -> ok = cdb_close(P2). safe_read_test() -> - % should return the right thing or false, or the wrong thing if and + % should return the right thing or false, or the wrong thing if and % only if we understand why Key = term_to_binary(<<"Key">>), Value = <<"Value">>, @@ -2569,28 +2563,28 @@ safe_read_test() -> FlippedKeyL = endian_flip(KeyL), ValueL= byte_size(ValToWrite), FlippedValL = endian_flip(ValueL), - + TestFN = "test/test_area/saferead.pnd", - BinToWrite = - <>, - - TestCorruptedWriteFun = + + TestCorruptedWriteFun = fun(BitNumber) -> - <> = BinToWrite, + <> = BinToWrite, BadBit = Bit bxor 1, - AltBin = <>, file:delete(TestFN), {ok, Handle} = file:open(TestFN, ?WRITE_OPS), ok = file:pwrite(Handle, 0, AltBin), {ok, _} = file:position(Handle, bof), - case saferead_keyvalue(Handle) of + case saferead_keyvalue(Handle) of false -> % Result OK to be false - should get that on error ok; @@ -2599,16 +2593,16 @@ safe_read_test() -> % for example if Value Length is too big % % This can only happen with a corrupted value length at - % the end of the file - which is just a peculiarity of + % the end of the file - which is just a peculiarity of % the test ?assertMatch(true, BadValueL > ValueL) end, ok = file:close(Handle) end, - lists:foreach(TestCorruptedWriteFun, + lists:foreach(TestCorruptedWriteFun, lists:seq(1, -1 + 8 * (KeyL + ValueL + 8))), - + {ok, HandleK} = file:open(TestFN, ?WRITE_OPS), ok = file:pwrite(HandleK, 0, BinToWrite), {ok, _} = file:position(HandleK, 8 + KeyL + ValueL), @@ -2635,9 +2629,9 @@ safe_read_test() -> {ok, HandleHappy} = file:open(TestFN, ?WRITE_OPS), ok = file:pwrite(HandleHappy, 0, BinToWrite), {ok, _} = file:position(HandleHappy, bof), - ?assertMatch({<<"Key">>, Value, KeyL, ValueL}, + ?assertMatch({<<"Key">>, Value, KeyL, ValueL}, saferead_keyvalue(HandleHappy)), - + file:delete(TestFN). @@ -2658,23 +2652,23 @@ get_positions_corruption_test() -> {ok, Handle} = file:open(F2, ?WRITE_OPS), Positions = lists:sublist(PositionList, 200, 10), - CorruptFun = + CorruptFun = fun(Offset) -> ok = file:pwrite(Handle, Offset, <<0:8/integer>>) end, ok = lists:foreach(CorruptFun, Positions), ok = file:close(Handle), - + {ok, P3} = cdb_open_reader(F2, #cdb_options{binary_mode=false}), - + PositionList = cdb_getpositions(P3, all), ?assertMatch(1000, length(PositionList)), - + KVCL = cdb_directfetch(P3, PositionList, key_size), ?assertMatch(true, length(KVCL) < 1000), ok = cdb_close(P3), file:delete(F2). - + badly_written_test() -> F1 = "test/test_area/badfirstwrite_test.pnd", file:delete(F1), @@ -2731,9 +2725,6 @@ getpositions_sample_test() -> nonsense_coverage_test() -> - ?assertMatch({next_state, reader, #state{}}, handle_info(nonsense, - reader, - #state{})), ?assertMatch({ok, reader, #state{}}, code_change(nonsense, reader, #state{}, diff --git a/src/leveled_log.erl b/src/leveled_log.erl index 7fe0d62..481976c 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -168,6 +168,8 @@ {info, <<"At level=~w file_count=~w avg_mem=~w file with most memory fn=~s p=~w mem=~w">>}, pc024 => {info, <<"Grooming compaction picked file with tomb_count=~w">>}, + pc025 => + {info, <<"At level=~w file_count=~w average words for heap_block_size=~w heap_size=~w recent_size=~w bin_vheap_size=~w">>}, pm002 => {info, <<"Completed dump of L0 cache to list of l0cache_size=~w">>}, sst03 => diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index c5c7618..1735962 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -200,11 +200,15 @@ handle_work( list(leveled_pmanifest:manifest_entry())}. merge(SrcLevel, Manifest, RootPath, OptsSST) -> case leveled_pmanifest:report_manifest_level(Manifest, SrcLevel + 1) of - {0, 0, undefined} -> + {0, 0, undefined, 0, 0, 0, 0} -> ok; - {FCnt, AvgMem, {MaxFN, MaxP, MaxMem}} -> - leveled_log:log(pc023, - [SrcLevel + 1, FCnt, AvgMem, MaxFN, MaxP, MaxMem]) + {FCnt, MnMem, {MaxFN, MaxP, MaxMem}, MnHBS, MnHS, MnLHS, MnBVHS} -> + leveled_log:log( + pc023, + [SrcLevel + 1, FCnt, MnMem, MaxFN, MaxP, MaxMem]), + leveled_log:log( + pc025, + [SrcLevel + 1, FCnt, MnHBS, MnHS, MnLHS, MnBVHS]) end, SelectMethod = case leveled_rand:uniform(100) of diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index ce7320f..9991cca 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -498,7 +498,7 @@ pcl_workforclerk(Pid) -> -spec pcl_manifestchange(pid(), leveled_pmanifest:manifest()) -> ok. %% @doc %% Provide a manifest record (i.e. the output of the leveled_pmanifest module) -%% that is required to beocme the new manifest. +%% that is required to become the new manifest. pcl_manifestchange(Pid, Manifest) -> gen_server:cast(Pid, {manifest_change, Manifest}). diff --git a/src/leveled_pmanifest.erl b/src/leveled_pmanifest.erl index 5286e15..3f419ac 100644 --- a/src/leveled_pmanifest.erl +++ b/src/leveled_pmanifest.erl @@ -266,11 +266,16 @@ remove_manifest(RootPath, GC_SQN) -> end. --spec report_manifest_level(manifest(), non_neg_integer()) -> - {non_neg_integer(), - non_neg_integer(), - {string(), pid(), non_neg_integer()} | - undefined}. +-spec report_manifest_level( + manifest(), non_neg_integer()) -> + {non_neg_integer(), + non_neg_integer(), + {string(), pid(), non_neg_integer()} | + undefined, + non_neg_integer(), + non_neg_integer(), + non_neg_integer(), + non_neg_integer()}. %% @doc %% Report on a level in the manifest %% - How many files in the level @@ -287,7 +292,7 @@ report_manifest_level(Manifest, LevelIdx) -> {leveled_tree:tsize(Level), leveled_tree:to_list(Level)} end, AccMemFun = - fun(MaybeME, {MemAcc, Max}) -> + fun(MaybeME, {MemAcc, Max, HBSAcc, HSAcc, LHSAcc, BVHSAcc}) -> ME = get_manifest_entry(MaybeME), P = ME#manifest_entry.owner, {memory, PM} = process_info(P, memory), @@ -298,15 +303,26 @@ report_manifest_level(Manifest, LevelIdx) -> _ -> {ME#manifest_entry.filename, P, PM} end, - {MemAcc + PM, UpdMax} + {garbage_collection_info, GCI} = + process_info(P, garbage_collection_info), + HBS = proplists:get_value(heap_block_size, GCI), + HS = proplists:get_value(heap_size, GCI), + LHS = proplists:get_value(recent_size, GCI), + BVHS = proplists:get_value(bin_vheap_size, GCI), + {MemAcc + PM, UpdMax, + HBSAcc + HBS, HSAcc + HS, LHSAcc + LHS, BVHSAcc + BVHS} end, case LevelSize of 0 -> - {0, 0, undefined}; + {0, 0, undefined, 0, 0, 0, 0}; _ -> - {TotalMem, BiggestMem} = - lists:foldl(AccMemFun, {0, undefined}, LevelList), - {LevelSize, TotalMem div LevelSize, BiggestMem} + {TotalMem, BiggestMem, TotalHBS, TotalHS, TotalLHS, TotalBVBS} = + lists:foldl(AccMemFun, {0, undefined, 0, 0, 0, 0}, LevelList), + {LevelSize, TotalMem div LevelSize, BiggestMem, + TotalHBS div LevelSize, + TotalHS div LevelSize, + TotalLHS div LevelSize, + TotalBVBS div LevelSize} end. diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index ea2d230..cee6bb6 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -8,8 +8,8 @@ %% -------- Slots --------- %% %% The view is built from sublists referred to as slot. Each slot is up to 128 -%% keys and values in size. Three strategies have been benchmarked for the -%% slot: a skiplist, a gb-tree, four blocks of flat lists with an index. +%% keys and values in size. Three strategies have been benchmarked for the +%% slot: a skiplist, a gb-tree, four blocks of flat lists with an index. %% %% Skiplist: %% build and serialise slot - 3233 microseconds @@ -26,8 +26,8 @@ %% de-deserialise and check * 128 - 6746 microseconds %% flatten back to list - 187 microseconds %% -%% The negative side of using Indexed Blocks is the storage of the index. In -%% the original implementation this was stored on fadvised disk (the index in +%% The negative side of using Indexed Blocks is the storage of the index. In +%% the original implementation this was stored on fadvised disk (the index in %% this case was a rice-encoded view of which block the object is in). In this %% implementation it is cached in memory -requiring 2-bytes per key to be kept %% in memory. @@ -43,15 +43,15 @@ %% %% -------- Summary --------- %% -%% Each file has a summary - which is the 128 keys at the top of each slot in +%% Each file has a summary - which is the 128 keys at the top of each slot in %% a skiplist, with some basic metadata about the slot stored as the value. %% %% The summary is stored seperately to the slots (within the same file). %% %% -------- CRC Checks --------- %% -%% Every attempt to either read a summary or a slot off disk will also include -%% a CRC check. If the CRC check fails non-presence is assumed (the data +%% Every attempt to either read a summary or a slot off disk will also include +%% a CRC check. If the CRC check fails non-presence is assumed (the data %% within is assumed to be entirely lost). The data can be recovered by either %% using a recoverable strategy in transaction log compaction, and triggering %% the transaction log replay; or by using a higher level for of anti-entropy @@ -60,16 +60,7 @@ -module(leveled_sst). --behaviour(gen_fsm). - --ifdef(fsm_deprecated). --compile({nowarn_deprecated_function, - [{gen_fsm, start_link, 3}, - {gen_fsm, sync_send_event, 2}, - {gen_fsm, sync_send_event, 3}, - {gen_fsm, send_event, 2}, - {gen_fsm, send_all_state_event, 2}]}). --endif. +-behaviour(gen_statem). -include("include/leveled.hrl"). @@ -102,38 +93,37 @@ -define(HIBERNATE_TIMEOUT, 60000). -endif. +-define(START_OPTS, [{hibernate_after, ?HIBERNATE_TIMEOUT}]). + -include_lib("eunit/include/eunit.hrl"). -export([init/1, - handle_sync_event/4, - handle_event/3, - handle_info/3, - terminate/3, - code_change/4, - format_status/2, - starting/2, - starting/3, - reader/2, - reader/3, - delete_pending/2, - delete_pending/3]). + callback_mode/0, + terminate/3, + code_change/4, + format_status/2]). + +%% states +-export([starting/3, + reader/3, + delete_pending/3]). -export([sst_new/6, - sst_newmerge/8, - sst_newlevelzero/7, - sst_open/4, - sst_get/2, - sst_get/3, - sst_getsqn/3, - sst_expandpointer/5, - sst_getmaxsequencenumber/1, - sst_setfordelete/2, - sst_clear/1, - sst_checkready/1, - sst_switchlevels/2, - sst_deleteconfirmed/1, - sst_gettombcount/1, - sst_close/1]). + sst_newmerge/8, + sst_newlevelzero/7, + sst_open/4, + sst_get/2, + sst_get/3, + sst_getsqn/3, + sst_expandpointer/5, + sst_getmaxsequencenumber/1, + sst_setfordelete/2, + sst_clear/1, + sst_checkready/1, + sst_switchlevels/2, + sst_deleteconfirmed/1, + sst_gettombcount/1, + sst_close/1]). -ifdef(TEST). @@ -159,27 +149,27 @@ %% The summary record is persisted as part of the sile format %% Any chnage to this record will mean the change cannot be rolled back --type press_method() +-type press_method() :: lz4|native|none. --type range_endpoint() +-type range_endpoint() :: all|leveled_codec:ledger_key(). --type slot_pointer() +-type slot_pointer() :: {pointer, pid(), integer(), range_endpoint(), range_endpoint()}. --type sst_pointer() +-type sst_pointer() % Used in sst_new - :: {next, - leveled_pmanifest:manifest_entry(), + :: {next, + leveled_pmanifest:manifest_entry(), range_endpoint()}. --type sst_closed_pointer() - % used in expand_list_by_pointer +-type sst_closed_pointer() + % used in expand_list_by_pointer % (close point is added by maybe_expand_pointer - :: {next, - leveled_pmanifest:manifest_entry(), + :: {next, + leveled_pmanifest:manifest_entry(), range_endpoint(), range_endpoint()}. --type expandable_pointer() +-type expandable_pointer() :: slot_pointer()|sst_pointer()|sst_closed_pointer(). --type expanded_pointer() +-type expanded_pointer() :: leveled_codec:ledger_kv()|expandable_pointer(). -type binaryslot_element() :: {tuple(), tuple()}|{binary(), integer(), tuple(), tuple()}. @@ -187,7 +177,7 @@ :: false| {sets, sets:set(non_neg_integer())}| {list, list(non_neg_integer())}. --type sst_options() +-type sst_options() :: #sst_options{}. -type binary_slot() :: {binary(), binary(), list(integer()), leveled_codec:ledger_key()}. @@ -213,7 +203,7 @@ %% see Issue 52. Handling within the SST process may lead to contention and %% extra copying. Files at the top of the tree yield, those lower down don't. --record(state, +-record(state, {summary, handle :: file:fd() | undefined, penciller :: pid() | undefined | false, @@ -251,8 +241,8 @@ %%%============================================================================ -spec sst_open(string(), string(), sst_options(), level()) - -> {ok, pid(), - {leveled_codec:ledger_key(), leveled_codec:ledger_key()}, + -> {ok, pid(), + {leveled_codec:ledger_key(), leveled_codec:ledger_key()}, binary()}. %% @doc %% Open an SST file at a given path and filename. The first and last keys @@ -263,51 +253,48 @@ %% %% The filename should include the file extension. sst_open(RootPath, Filename, OptsSST, Level) -> - {ok, Pid} = gen_fsm:start_link(?MODULE, [], []), - case gen_fsm:sync_send_event(Pid, - {sst_open, - RootPath, Filename, OptsSST, Level}, - infinity) of + {ok, Pid} = gen_statem:start_link(?MODULE, [], ?START_OPTS), + case gen_statem:call( + Pid, {sst_open, RootPath, Filename, OptsSST, Level}, infinity) of {ok, {SK, EK}, Bloom} -> {ok, Pid, {SK, EK}, Bloom} end. --spec sst_new(string(), string(), level(), - list(leveled_codec:ledger_kv()), - integer(), sst_options()) - -> {ok, pid(), - {leveled_codec:ledger_key(), leveled_codec:ledger_key()}, +-spec sst_new(string(), string(), level(), + list(leveled_codec:ledger_kv()), + integer(), sst_options()) + -> {ok, pid(), + {leveled_codec:ledger_key(), leveled_codec:ledger_key()}, binary()}. %% @doc %% Start a new SST file at the assigned level passing in a list of Key, Value %% pairs. This should not be used for basement levels or unexpanded Key/Value %% lists as merge_lists will not be called. sst_new(RootPath, Filename, Level, KVList, MaxSQN, OptsSST) -> - sst_new(RootPath, Filename, Level, - KVList, MaxSQN, OptsSST, ?INDEX_MODDATE). + sst_new( + RootPath, Filename, Level, KVList, MaxSQN, OptsSST, ?INDEX_MODDATE). sst_new(RootPath, Filename, Level, KVList, MaxSQN, OptsSST, IndexModDate) -> - {ok, Pid} = gen_fsm:start_link(?MODULE, [], []), + {ok, Pid} = gen_statem:start_link(?MODULE, [], ?START_OPTS), PressMethod0 = compress_level(Level, OptsSST#sst_options.press_method), MaxSlots0 = maxslots_level(Level, OptsSST#sst_options.max_sstslots), OptsSST0 = - OptsSST#sst_options{press_method = PressMethod0, - max_sstslots = MaxSlots0}, + OptsSST#sst_options{ + press_method = PressMethod0, max_sstslots = MaxSlots0}, {[], [], SlotList, FK, _CountOfTombs} = merge_lists(KVList, OptsSST0, IndexModDate), - case gen_fsm:sync_send_event(Pid, - {sst_new, - RootPath, - Filename, - Level, - {SlotList, FK}, - MaxSQN, - OptsSST0, - IndexModDate, - not_counted, - self()}, - infinity) of + case gen_statem:call(Pid, {sst_new, + RootPath, + Filename, + Level, + {SlotList, FK}, + MaxSQN, + OptsSST0, + IndexModDate, + not_counted, + self()}, + infinity) of {ok, {SK, EK}, Bloom} -> {ok, Pid, {SK, EK}, Bloom} end. @@ -341,54 +328,53 @@ sst_newmerge(RootPath, Filename, KVL1, KVL2, IsBasement, Level, MaxSQN, OptsSST, ?INDEX_MODDATE, ?TOMB_COUNT). -sst_newmerge(RootPath, Filename, - KVL1, KVL2, IsBasement, Level, +sst_newmerge(RootPath, Filename, + KVL1, KVL2, IsBasement, Level, MaxSQN, OptsSST, IndexModDate, TombCount) -> PressMethod0 = compress_level(Level, OptsSST#sst_options.press_method), MaxSlots0 = maxslots_level(Level, OptsSST#sst_options.max_sstslots), OptsSST0 = OptsSST#sst_options{press_method = PressMethod0, max_sstslots = MaxSlots0}, - {Rem1, Rem2, SlotList, FK, CountOfTombs} = + {Rem1, Rem2, SlotList, FK, CountOfTombs} = merge_lists(KVL1, KVL2, {IsBasement, Level}, OptsSST0, IndexModDate, TombCount), case SlotList of [] -> empty; _ -> - {ok, Pid} = gen_fsm:start_link(?MODULE, [], []), - case gen_fsm:sync_send_event(Pid, - {sst_new, - RootPath, - Filename, - Level, - {SlotList, FK}, - MaxSQN, - OptsSST0, - IndexModDate, - CountOfTombs, - self()}, - infinity) of + {ok, Pid} = gen_statem:start_link(?MODULE, [], ?START_OPTS), + case gen_statem:call(Pid, {sst_new, + RootPath, + Filename, + Level, + {SlotList, FK}, + MaxSQN, + OptsSST0, + IndexModDate, + CountOfTombs, + self()}, + infinity) of {ok, {SK, EK}, Bloom} -> {ok, Pid, {{Rem1, Rem2}, SK, EK}, Bloom} end end. -spec sst_newlevelzero(string(), string(), - integer(), + integer(), fun((pos_integer(), leveled_penciller:levelzero_returnfun()) -> ok)| - list(), - pid()|undefined, - integer(), + list(), + pid()|undefined, + integer(), sst_options()) -> {ok, pid(), noreply}. %% @doc %% Start a new file at level zero. At this level the file size is not fixed - -%% it will be as big as the input. Also the KVList is not passed in, it is +%% it will be as big as the input. Also the KVList is not passed in, it is %% fetched slot by slot using the FetchFun -sst_newlevelzero(RootPath, Filename, +sst_newlevelzero(RootPath, Filename, Slots, Fetcher, Penciller, MaxSQN, OptsSST) -> PressMethod0 = compress_level(0, OptsSST#sst_options.press_method), @@ -396,24 +382,23 @@ sst_newlevelzero(RootPath, Filename, OptsSST0 = OptsSST#sst_options{press_method = PressMethod0, max_sstslots = MaxSlots0}, - {ok, Pid} = gen_fsm:start_link(?MODULE, [], []), - % Initiate the file into the "starting" state - ok = gen_fsm:sync_send_event(Pid, - {sst_newlevelzero, - RootPath, - Filename, - Penciller, - MaxSQN, - OptsSST0, - ?INDEX_MODDATE}, - infinity), - ok = - case is_list(Fetcher) of - true -> - gen_fsm:send_event(Pid, {complete_l0startup, Fetcher}); - false -> + {ok, Pid} = gen_statem:start_link(?MODULE, [], ?START_OPTS), + %% Initiate the file into the "starting" state + ok = gen_statem:call(Pid, {sst_newlevelzero, + RootPath, + Filename, + Penciller, + MaxSQN, + OptsSST0, + ?INDEX_MODDATE}, + infinity), + ok = + case Fetcher of + FetchSlots when is_list(Fetcher) -> + gen_statem:cast(Pid, {complete_l0startup, FetchSlots}); + _ -> % Fetcher is a function - gen_fsm:send_event(Pid, {sst_returnslot, none, Fetcher, Slots}) + gen_statem:cast(Pid, {sst_returnslot, none, Fetcher, Slots}) % Start the fetch loop (async). Having the fetch loop running % on async message passing means that the SST file can now be % closed while the fetch loop is still completing @@ -436,7 +421,7 @@ sst_get(Pid, LedgerKey) -> %% Return a Key, Value pair matching a Key or not_present if the Key is not in %% the store (with the magic hash precalculated). sst_get(Pid, LedgerKey, Hash) -> - gen_fsm:sync_send_event(Pid, {get_kv, LedgerKey, Hash}, infinity). + gen_statem:call(Pid, {get_kv, LedgerKey, Hash, undefined}, infinity). -spec sst_getsqn(pid(), leveled_codec:ledger_key(), @@ -445,16 +430,16 @@ sst_get(Pid, LedgerKey, Hash) -> %% Return a SQN for the key or not_present if the key is not in %% the store (with the magic hash precalculated). sst_getsqn(Pid, LedgerKey, Hash) -> - gen_fsm:sync_send_event(Pid, {get_sqn, LedgerKey, Hash}, infinity). + gen_statem:call(Pid, {get_kv, LedgerKey, Hash, fun sqn_only/1}, infinity). -spec sst_getmaxsequencenumber(pid()) -> integer(). %% @doc %% Get the maximume sequence number for this SST file sst_getmaxsequencenumber(Pid) -> - gen_fsm:sync_send_event(Pid, get_maxsequencenumber, infinity). + gen_statem:call(Pid, get_maxsequencenumber, infinity). --spec sst_expandpointer(expandable_pointer(), - list(expandable_pointer()), +-spec sst_expandpointer(expandable_pointer(), + list(expandable_pointer()), pos_integer(), leveled_codec:segment_list(), non_neg_integer()) @@ -478,7 +463,7 @@ sst_expandpointer(Pointer, MorePointers, ScanWidth, SegmentList, LowLastMod) -> %% on it have finished). No polling will be done if the Penciller pid %% is 'false' sst_setfordelete(Pid, Penciller) -> - gen_fsm:sync_send_event(Pid, {set_for_delete, Penciller}, infinity). + gen_statem:call(Pid, {set_for_delete, Penciller}, infinity). -spec sst_gettombcount(pid()) -> non_neg_integer()|not_counted. %% @doc @@ -487,31 +472,31 @@ sst_setfordelete(Pid, Penciller) -> %% could also be because the file is L0 (which aren't counted as being chosen %% for merge is inevitable) sst_gettombcount(Pid) -> - gen_fsm:sync_send_event(Pid, get_tomb_count, infinity). + gen_statem:call(Pid, get_tomb_count, infinity). -spec sst_clear(pid()) -> ok. %% @doc %% For this file to be closed and deleted sst_clear(Pid) -> - gen_fsm:sync_send_event(Pid, {set_for_delete, false}, infinity), - gen_fsm:sync_send_event(Pid, close). + gen_statem:call(Pid, {set_for_delete, false}, infinity), + gen_statem:call(Pid, close). -spec sst_deleteconfirmed(pid()) -> ok. %% @doc %% Allows a penciller to confirm to a SST file that it can be cleared, as it %% is no longer in use sst_deleteconfirmed(Pid) -> - gen_fsm:send_event(Pid, close). + gen_statem:cast(Pid, close). --spec sst_checkready(pid()) -> {ok, string(), - leveled_codec:ledger_key(), +-spec sst_checkready(pid()) -> {ok, string(), + leveled_codec:ledger_key(), leveled_codec:ledger_key()}. %% @doc %% If a file has been set to be built, check that it has been built. Returns %% the filename and the {startKey, EndKey} for the manifest. sst_checkready(Pid) -> %% Only used in test - gen_fsm:sync_send_event(Pid, background_complete). + gen_statem:call(Pid, background_complete). -spec sst_switchlevels(pid(), pos_integer()) -> ok. %% @doc @@ -520,45 +505,56 @@ sst_checkready(Pid) -> %% file, so don't want all the startup state to be held on memory - want to %% proactively drop it sst_switchlevels(Pid, NewLevel) -> - gen_fsm:send_event(Pid, {switch_levels, NewLevel}). + gen_statem:cast(Pid, {switch_levels, NewLevel}). -spec sst_close(pid()) -> ok. %% @doc %% Close the file sst_close(Pid) -> - gen_fsm:sync_send_event(Pid, close). + gen_statem:call(Pid, close). %%%============================================================================ -%%% gen_server callbacks +%%% gen_statem callbacks %%%============================================================================ +callback_mode() -> + state_functions. + init([]) -> {ok, starting, #state{}}. -starting({sst_open, RootPath, Filename, OptsSST, Level}, _From, State) -> +starting({call, From}, + {sst_open, RootPath, Filename, OptsSST, Level}, + State) -> leveled_log:save(OptsSST#sst_options.log_options), Monitor = OptsSST#sst_options.monitor, - {UpdState, Bloom} = + {UpdState, Bloom} = read_file(Filename, State#state{root_path=RootPath}, OptsSST#sst_options.pagecache_level >= Level), Summary = UpdState#state.summary, - {reply, - {ok, {Summary#summary.first_key, Summary#summary.last_key}, Bloom}, + {next_state, reader, UpdState#state{ - level = Level, fetch_cache = new_cache(Level), monitor = Monitor}}; -starting({sst_new, - RootPath, Filename, Level, - {SlotList, FirstKey}, MaxSQN, - OptsSST, IdxModDate, CountOfTombs, StartingPID}, _From, State) -> + level = Level, fetch_cache = new_cache(Level), monitor = Monitor}, + [{reply, From, + {ok, + {Summary#summary.first_key, Summary#summary.last_key}, + Bloom} + }]}; +starting({call, From}, + {sst_new, + RootPath, Filename, Level, + {SlotList, FirstKey}, MaxSQN, + OptsSST, IdxModDate, CountOfTombs, StartingPID}, + State) -> SW = os:timestamp(), leveled_log:save(OptsSST#sst_options.log_options), Monitor = OptsSST#sst_options.monitor, PressMethod = OptsSST#sst_options.press_method, - {Length, SlotIndex, BlockEntries, SlotsBin, Bloom} = + {Length, SlotIndex, BlockEntries, SlotsBin, Bloom} = build_all_slots(SlotList), {_, BlockIndex, HighModDate} = update_blockindex_cache(true, @@ -566,14 +562,14 @@ starting({sst_new, new_blockindex_cache(Length), undefined, IdxModDate), - SummaryBin = + SummaryBin = build_table_summary(SlotIndex, Level, FirstKey, Length, MaxSQN, Bloom, CountOfTombs), - ActualFilename = + ActualFilename = write_file(RootPath, Filename, SummaryBin, SlotsBin, PressMethod, IdxModDate, CountOfTombs), YBQ = Level =< 2, - {UpdState, Bloom} = + {UpdState, Bloom} = read_file(ActualFilename, State#state{root_path=RootPath, yield_blockquery=YBQ}, OptsSST#sst_options.pagecache_level >= Level), @@ -581,8 +577,7 @@ starting({sst_new, leveled_log:log_timer( sst08, [ActualFilename, Level, Summary#summary.max_sqn], SW), erlang:send_after(?STARTUP_TIMEOUT, self(), start_complete), - {reply, - {ok, {Summary#summary.first_key, Summary#summary.last_key}, Bloom}, + {next_state, reader, UpdState#state{ blockindex_cache = BlockIndex, @@ -590,25 +585,32 @@ starting({sst_new, starting_pid = StartingPID, level = Level, fetch_cache = new_cache(Level), - monitor = Monitor}}; -starting({sst_newlevelzero, RootPath, Filename, + monitor = Monitor}, + [{reply, + From, + {ok, {Summary#summary.first_key, Summary#summary.last_key}, Bloom} + }]}; +starting({call, From}, {sst_newlevelzero, RootPath, Filename, Penciller, MaxSQN, - OptsSST, IdxModDate}, _From, State) -> - DeferredStartupTuple = + OptsSST, IdxModDate}, State) -> + DeferredStartupTuple = {RootPath, Filename, Penciller, MaxSQN, OptsSST, IdxModDate}, - {reply, ok, starting, + {next_state, starting, State#state{ deferred_startup_tuple = DeferredStartupTuple, level = 0, - fetch_cache = new_cache(0)}}; -starting(close, _From, State) -> - % No file should have been created, so nothing to close. - {stop, normal, ok, State}. - -starting({complete_l0startup, Slots}, State) -> - starting(complete_l0startup, State#state{new_slots = Slots}); -starting(complete_l0startup, State) -> + fetch_cache = new_cache(0)}, + [{reply, From, ok}]}; +starting({call, From}, close, State) -> + %% No file should have been created, so nothing to close. + {stop_and_reply, normal, [{reply, From, ok}], State}; + +starting(cast, {complete_l0startup, Slots}, State) -> + {keep_state, + State#state{new_slots = Slots}, + [{next_event, cast, complete_l0startup}]}; +starting(cast, complete_l0startup, State) -> {RootPath, Filename, Penciller, MaxSQN, OptsSST, IdxModDate} = State#state.deferred_startup_tuple, SW0 = os:timestamp(), @@ -619,7 +621,7 @@ starting(complete_l0startup, State) -> FetchFun = fun(Slot) -> lists:nth(Slot, FetchedSlots) end, KVList = leveled_pmem:to_list(length(FetchedSlots), FetchFun), Time0 = timer:now_diff(os:timestamp(), SW0), - + SW1 = os:timestamp(), {[], [], SlotList, FirstKey, _CountOfTombs} = merge_lists(KVList, OptsSST, IdxModDate), @@ -635,18 +637,18 @@ starting(complete_l0startup, State) -> undefined, IdxModDate), Time2 = timer:now_diff(os:timestamp(), SW2), - + SW3 = os:timestamp(), - SummaryBin = + SummaryBin = build_table_summary(SlotIndex, 0, FirstKey, SlotCount, MaxSQN, Bloom, not_counted), Time3 = timer:now_diff(os:timestamp(), SW3), - + SW4 = os:timestamp(), - ActualFilename = + ActualFilename = write_file(RootPath, Filename, SummaryBin, SlotsBin, PressMethod, IdxModDate, not_counted), - {UpdState, Bloom} = + {UpdState, Bloom} = read_file(ActualFilename, State#state{root_path=RootPath, yield_blockquery=true, @@ -657,7 +659,7 @@ starting(complete_l0startup, State) -> true), Summary = UpdState#state.summary, Time4 = timer:now_diff(os:timestamp(), SW4), - + leveled_log:log_timer( sst08, [ActualFilename, 0, Summary#summary.max_sqn], SW0), leveled_log:log(sst11, [Time0, Time1, Time2, Time3, Time4]), @@ -680,39 +682,45 @@ starting(complete_l0startup, State) -> blockindex_cache = BlockIndex, high_modified_date = HighModDate, monitor = Monitor}}; -starting({sst_returnslot, FetchedSlot, FetchFun, SlotCount}, State) -> - Self = self(), - FetchedSlots = +starting(cast, {sst_returnslot, FetchedSlot, FetchFun, SlotCount}, State) -> + FetchedSlots = case FetchedSlot of none -> []; _ -> [FetchedSlot|State#state.new_slots] - end, + end, case length(FetchedSlots) == SlotCount of true -> - gen_fsm:send_event(Self, complete_l0startup), - {next_state, - starting, - % Reverse the slots so that they are back in the expected - % order - State#state{new_slots = lists:reverse(FetchedSlots)}}; + {keep_state, + %% Reverse the slots so that they are back in the expected + %% order + State#state{new_slots = lists:reverse(FetchedSlots)}, + [{next_event, cast, complete_l0startup}]}; false -> + Self = self(), ReturnFun = fun(NextSlot) -> - gen_fsm:send_event(Self, + gen_statem:cast(Self, {sst_returnslot, NextSlot, FetchFun, SlotCount}) end, FetchFun(length(FetchedSlots) + 1, ReturnFun), - {next_state, - starting, + {keep_state, State#state{new_slots = FetchedSlots}} end. -reader({get_sqn, LedgerKey, Hash}, _From, State) -> + +reader({call, From}, {get_kv, LedgerKey, Hash, Filter}, State) -> % Get a KV value and potentially take sample timings - {Result, _BIC, _HMD, _FC} = + Monitor = + case Filter of + undefined -> + State#state.monitor; + _ -> + {no_monitor, 0} + end, + {KeyValue, BIC, HMD, FC} = fetch( LedgerKey, Hash, State#state.summary, @@ -724,44 +732,40 @@ reader({get_sqn, LedgerKey, Hash}, _From, State) -> State#state.fetch_cache, State#state.handle, State#state.level, - {no_monitor, 0}), - {reply, sqn_only(Result), reader, State, ?HIBERNATE_TIMEOUT}; -reader({get_kv, LedgerKey, Hash}, _From, State) -> - % Get a KV value and potentially take sample timings - {Result, BIC, HMD, FC} = - fetch( - LedgerKey, Hash, - State#state.summary, - State#state.compression_method, - State#state.high_modified_date, - State#state.index_moddate, - State#state.filter_fun, - State#state.blockindex_cache, - State#state.fetch_cache, - State#state.handle, - State#state.level, - State#state.monitor), + Monitor), + Result = + case Filter of + undefined -> + KeyValue; + F -> + F(KeyValue) + end, case {BIC, HMD, FC} of {no_update, no_update, no_update} -> - {reply, Result, reader, State}; + {keep_state_and_data, [{reply, From, Result}]}; {no_update, no_update, FC} -> - {reply, Result, reader, State#state{fetch_cache = FC}}; + {keep_state, + State#state{fetch_cache = FC}, + [{reply, From, Result}]}; + {BIC, undefined, no_update} -> + {keep_state, + State#state{blockindex_cache = BIC}, + [{reply, From, Result}]}; {BIC, HMD, no_update} -> - {reply, - Result, - reader, - State#state{ - blockindex_cache = BIC, high_modified_date = HMD}} + {keep_state, + State#state{blockindex_cache = BIC, high_modified_date = HMD}, + [hibernate, {reply, From, Result}]} end; -reader({get_kvrange, StartKey, EndKey, ScanWidth, SegList, LowLastMod}, - _From, State) -> +reader({call, From}, + {get_kvrange, StartKey, EndKey, ScanWidth, SegList, LowLastMod}, + State) -> ReadNeeded = check_modified(State#state.high_modified_date, LowLastMod, State#state.index_moddate), {NeedBlockIdx, SlotsToFetchBinList, SlotsToPoint} = case ReadNeeded of - true -> + true -> fetch_range(StartKey, EndKey, ScanWidth, SegList, LowLastMod, State); @@ -770,23 +774,22 @@ reader({get_kvrange, StartKey, EndKey, ScanWidth, SegList, LowLastMod}, end, PressMethod = State#state.compression_method, IdxModDate = State#state.index_moddate, - + case State#state.yield_blockquery of true -> - {reply, - {yield, - SlotsToFetchBinList, - SlotsToPoint, - PressMethod, - IdxModDate}, - reader, - State}; + {keep_state_and_data, + [{reply, + From, + {yield, + SlotsToFetchBinList, + SlotsToPoint, + PressMethod, + IdxModDate} + }]}; false -> - {L, FoundBIC} = - binaryslot_reader(SlotsToFetchBinList, - PressMethod, - IdxModDate, - SegList), + {L, FoundBIC} = + binaryslot_reader( + SlotsToFetchBinList, PressMethod, IdxModDate, SegList), {UpdateCache, BlockIdxC0, HighModDate} = update_blockindex_cache(NeedBlockIdx, FoundBIC, @@ -795,184 +798,70 @@ reader({get_kvrange, StartKey, EndKey, ScanWidth, SegList, LowLastMod}, State#state.index_moddate), case UpdateCache of true -> - {reply, - L ++ SlotsToPoint, - reader, + {keep_state, State#state{ blockindex_cache = BlockIdxC0, - high_modified_date = HighModDate}}; + high_modified_date = HighModDate}, + [{reply, From, L ++ SlotsToPoint}]}; false -> - {reply, L ++ SlotsToPoint, reader, State} + {keep_state_and_data, + [hibernate, {reply, From, L ++ SlotsToPoint}]} end end; -reader({get_slots, SlotList, SegList, LowLastMod}, _From, State) -> +reader({call, From}, {get_slots, SlotList, SegList, LowLastMod}, State) -> PressMethod = State#state.compression_method, IdxModDate = State#state.index_moddate, - {NeedBlockIdx, SlotBins} = - read_slots(State#state.handle, - SlotList, + {NeedBlockIdx, SlotBins} = + read_slots(State#state.handle, + SlotList, {SegList, LowLastMod, State#state.blockindex_cache}, State#state.compression_method, State#state.index_moddate), - {reply, - {NeedBlockIdx, SlotBins, PressMethod, IdxModDate}, - reader, - State}; -reader(get_maxsequencenumber, _From, State) -> + {keep_state_and_data, + [{reply, From, {NeedBlockIdx, SlotBins, PressMethod, IdxModDate}}]}; +reader({call, From}, get_maxsequencenumber, State) -> Summary = State#state.summary, - {reply, Summary#summary.max_sqn, reader, State}; -reader({set_for_delete, Penciller}, _From, State) -> + {keep_state_and_data, + [{reply, From, Summary#summary.max_sqn}]}; +reader({call, From}, {set_for_delete, Penciller}, State) -> leveled_log:log(sst06, [State#state.filename]), - {reply, - ok, + {next_state, delete_pending, State#state{penciller=Penciller}, - ?DELETE_TIMEOUT}; -reader(background_complete, _From, State) -> + [{reply, From,ok}, ?DELETE_TIMEOUT]}; +reader({call, From}, background_complete, State) -> Summary = State#state.summary, - {reply, - {ok, - State#state.filename, - Summary#summary.first_key, - Summary#summary.last_key}, - reader, - State}; -reader(get_tomb_count, _From, State) -> - {reply, State#state.tomb_count, reader, State}; -reader(close, _From, State) -> + {keep_state_and_data, + [{reply, + From, + {ok, + State#state.filename, + Summary#summary.first_key, + Summary#summary.last_key} + }]}; +reader({call, From}, get_tomb_count, State) -> + {keep_state_and_data, + [{reply, From, State#state.tomb_count}]}; +reader({call, From}, close, State) -> ok = file:close(State#state.handle), - {stop, normal, ok, State}. + {stop_and_reply, normal, [{reply, From, ok}], State}; -reader(timeout, State) -> - FreshFetchCache = new_cache(State#state.level), - Summary = State#state.summary, - FreshBlockIndexCache = new_blockindex_cache(Summary#summary.size), - {next_state, - reader, - State#state{ - fetch_cache = FreshFetchCache, - blockindex_cache = FreshBlockIndexCache}, - hibernate}; -reader({switch_levels, NewLevel}, State) -> +reader(cast, {switch_levels, NewLevel}, State) -> FreshCache = new_cache(NewLevel), - {next_state, - reader, - State#state{ - level = NewLevel, - fetch_cache = FreshCache}, - hibernate}. - - -delete_pending({get_sqn, LedgerKey, Hash}, _From, State) -> - % Get a KV value and potentially take sample timings - {Result, _BIC, _HMD, _FC} = - fetch( - LedgerKey, Hash, - State#state.summary, - State#state.compression_method, - State#state.high_modified_date, - State#state.index_moddate, - State#state.filter_fun, - State#state.blockindex_cache, - State#state.fetch_cache, - State#state.handle, - State#state.level, - {no_monitor, 0}), - {reply, sqn_only(Result), delete_pending, State, ?DELETE_TIMEOUT}; -delete_pending({get_kv, LedgerKey, Hash}, _From, State) -> - {Result, _BIC, _HMD, _FC} = - fetch( - LedgerKey, Hash, - State#state.summary, - State#state.compression_method, - State#state.high_modified_date, - State#state.index_moddate, - State#state.filter_fun, - State#state.blockindex_cache, - State#state.fetch_cache, - State#state.handle, - State#state.level, - {no_monitor, 0}), - {reply, Result, delete_pending, State, ?DELETE_TIMEOUT}; -delete_pending({get_kvrange, StartKey, EndKey, ScanWidth, SegList, LowLastMod}, - _From, State) -> - {_NeedBlockIdx, SlotsToFetchBinList, SlotsToPoint} = - fetch_range(StartKey, EndKey, ScanWidth, SegList, LowLastMod, State), - % Always yield as about to clear and de-reference - PressMethod = State#state.compression_method, - IdxModDate = State#state.index_moddate, - {reply, - {yield, SlotsToFetchBinList, SlotsToPoint, PressMethod, IdxModDate}, - delete_pending, - State, - ?DELETE_TIMEOUT}; -delete_pending({get_slots, SlotList, SegList, LowLastMod}, _From, State) -> - PressMethod = State#state.compression_method, - IdxModDate = State#state.index_moddate, - {_NeedBlockIdx, SlotBins} = - read_slots(State#state.handle, - SlotList, - {SegList, LowLastMod, State#state.blockindex_cache}, - PressMethod, - IdxModDate), - {reply, - {false, SlotBins, PressMethod, IdxModDate}, - delete_pending, - State, - ?DELETE_TIMEOUT}; -delete_pending(close, _From, State) -> - leveled_log:log(sst07, [State#state.filename]), - ok = file:close(State#state.handle), - ok = file:delete(filename:join(State#state.root_path, - State#state.filename)), - {stop, normal, ok, State}. - -delete_pending(timeout, State) -> - case State#state.penciller of - false -> - ok = leveled_sst:sst_deleteconfirmed(self()); - PCL -> - FN = State#state.filename, - ok = leveled_penciller:pcl_confirmdelete(PCL, FN, self()) - end, - % If the next thing is another timeout - may be long-running snapshot, so - % back-off - {next_state, delete_pending, State, leveled_rand:uniform(10) * ?DELETE_TIMEOUT}; -delete_pending(close, State) -> - leveled_log:log(sst07, [State#state.filename]), - ok = file:close(State#state.handle), - ok = file:delete(filename:join(State#state.root_path, - State#state.filename)), - {stop, normal, State}. - -handle_sync_event(_Msg, _From, StateName, State) -> - {reply, undefined, StateName, State}. - -handle_event({update_blockindex_cache, BIC}, StateName, State) -> - {_, BlockIndexCache, HighModDate} = - update_blockindex_cache(true, - BIC, - State#state.blockindex_cache, - State#state.high_modified_date, - State#state.index_moddate), - {next_state, - StateName, - State#state{blockindex_cache = BlockIndexCache, - high_modified_date = HighModDate}}. - -handle_info(_Msg, delete_pending, State) -> - % Ignore messages when pending delete. The message may have interrupted - % the delete timeout, so timeout straight away - {next_state, delete_pending, State, 0}; -handle_info(bic_complete, StateName, State) -> + {keep_state, + State#state{level = NewLevel, fetch_cache = FreshCache}, + [hibernate]}; +reader(info, {update_blockindex_cache, BIC}, State) -> + handle_update_blockindex_cache(BIC, State); +reader(info, bic_complete, State) -> % The block index cache is complete, so the memory footprint should be % relatively stable from this point. Hibernate to help minimise % fragmentation leveled_log:log(sst14, [State#state.filename]), - {next_state, StateName, State, hibernate}; -handle_info(start_complete, StateName, State) -> + {keep_state_and_data, [hibernate]}; +reader(info, start_complete, State) -> % The SST file will be started by a clerk, but the clerk may be shut down % prior to the manifest being updated about the existence of this SST file. % If there is no activity after startup, check the clerk is still alive and @@ -982,11 +871,110 @@ handle_info(start_complete, StateName, State) -> % part of the store case is_process_alive(State#state.starting_pid) of true -> - {next_state, StateName, State}; + {keep_state_and_data, []}; false -> - {stop, normal, State} + {stop, normal} end. + +delete_pending({call, From}, {get_kv, LedgerKey, Hash, Filter}, State) -> + {KeyValue, _BIC, _HMD, _FC} = + fetch( + LedgerKey, Hash, + State#state.summary, + State#state.compression_method, + State#state.high_modified_date, + State#state.index_moddate, + State#state.filter_fun, + State#state.blockindex_cache, + State#state.fetch_cache, + State#state.handle, + State#state.level, + {no_monitor, 0}), + Result = + case Filter of + undefined -> + KeyValue; + F -> + F(KeyValue) + end, + {keep_state_and_data, [{reply, From, Result}, ?DELETE_TIMEOUT]}; +delete_pending( + {call, From}, + {get_kvrange, StartKey, EndKey, ScanWidth, SegList, LowLastMod}, + State) -> + {_NeedBlockIdx, SlotsToFetchBinList, SlotsToPoint} = + fetch_range(StartKey, EndKey, ScanWidth, SegList, LowLastMod, State), + % Always yield as about to clear and de-reference + PressMethod = State#state.compression_method, + IdxModDate = State#state.index_moddate, + {keep_state_and_data, + [{reply, From, + {yield, + SlotsToFetchBinList, + SlotsToPoint, + PressMethod, + IdxModDate}}, + ?DELETE_TIMEOUT]}; +delete_pending( + {call, From}, + {get_slots, SlotList, SegList, LowLastMod}, + State) -> + PressMethod = State#state.compression_method, + IdxModDate = State#state.index_moddate, + {_NeedBlockIdx, SlotBins} = + read_slots(State#state.handle, + SlotList, + {SegList, LowLastMod, State#state.blockindex_cache}, + PressMethod, + IdxModDate), + {keep_state_and_data, + [{reply, From, {false, SlotBins, PressMethod, IdxModDate}}, + ?DELETE_TIMEOUT]}; +delete_pending({call, From}, close, State) -> + leveled_log:log(sst07, [State#state.filename]), + ok = file:close(State#state.handle), + ok = + file:delete( + filename:join(State#state.root_path, State#state.filename)), + {stop_and_reply, normal, [{reply, From, ok}], State}; +delete_pending(cast, close, State) -> + leveled_log:log(sst07, [State#state.filename]), + ok = file:close(State#state.handle), + ok = + file:delete( + filename:join(State#state.root_path, State#state.filename)), + {stop, normal, State}; + +delete_pending(info, _Event, _State) -> + % Ignore messages when pending delete. The message may have interrupted + % the delete timeout, so timeout straight away + {keep_state_and_data, [0]}; +delete_pending(timeout, _, State) -> + case State#state.penciller of + false -> + ok = leveled_sst:sst_deleteconfirmed(self()); + PCL -> + FN = State#state.filename, + ok = leveled_penciller:pcl_confirmdelete(PCL, FN, self()) + end, + % If the next thing is another timeout - may be long-running snapshot, so + % back-off + {keep_state_and_data, [leveled_rand:uniform(10) * ?DELETE_TIMEOUT]}. + +handle_update_blockindex_cache(BIC, State) -> + {_, BlockIndexCache, HighModDate} = + update_blockindex_cache(true, + BIC, + State#state.blockindex_cache, + State#state.high_modified_date, + State#state.index_moddate), + {keep_state, + State#state{ + blockindex_cache = BlockIndexCache, + high_modified_date = HighModDate}}. + + terminate(normal, delete_pending, _State) -> ok; terminate(Reason, _StateName, State) -> @@ -995,19 +983,19 @@ terminate(Reason, _StateName, State) -> code_change(_OldVsn, StateName, State, _Extra) -> {ok, StateName, State}. -format_status(normal, [_PDict, State]) -> +format_status(normal, [_PDict, _, State]) -> State; -format_status(terminate, [_PDict, State]) -> - State#state{blockindex_cache = redacted, - fetch_cache = redacted}. +format_status(terminate, [_PDict, _, State]) -> + State#state{ + blockindex_cache = redacted, fetch_cache = redacted}. %%%============================================================================ %%% External Functions %%%============================================================================ --spec expand_list_by_pointer(expandable_pointer(), - list(expandable_pointer()), +-spec expand_list_by_pointer(expandable_pointer(), + list(expandable_pointer()), pos_integer()) -> list(expanded_pointer()). %% @doc @@ -1025,20 +1013,20 @@ expand_list_by_pointer(Pointer, Tail, Width) -> expand_list_by_pointer(Pointer, Tail, Width, SegList) -> expand_list_by_pointer(Pointer, Tail, Width, SegList, 0). --spec expand_list_by_pointer(expandable_pointer(), - list(expandable_pointer()), +-spec expand_list_by_pointer(expandable_pointer(), + list(expandable_pointer()), pos_integer(), leveled_codec:segment_list(), non_neg_integer()) -> list(expanded_pointer()). %% @doc %% With filters (as described in expand_list_by_pointer/3 -expand_list_by_pointer({pointer, SSTPid, Slot, StartKey, EndKey}, +expand_list_by_pointer({pointer, SSTPid, Slot, StartKey, EndKey}, Tail, Width, SegList, LowLastMod) -> FoldFun = fun(X, {Pointers, Remainder}) -> case length(Pointers) of - L when L < Width -> + L when L < Width -> case X of {pointer, SSTPid, S, SK, EK} -> {Pointers ++ [{pointer, S, SK, EK}], Remainder}; @@ -1056,23 +1044,23 @@ expand_list_by_pointer({pointer, SSTPid, Slot, StartKey, EndKey}, SegList, LowLastMod), lists:append(ExpPointers, AccTail); -expand_list_by_pointer({next, ManEntry, StartKey, EndKey}, +expand_list_by_pointer({next, ManEntry, StartKey, EndKey}, Tail, Width, SegList, LowLastMod) -> SSTPid = ManEntry#manifest_entry.owner, leveled_log:log(sst10, [SSTPid, is_process_alive(SSTPid)]), - ExpPointer = sst_getfilteredrange(SSTPid, + ExpPointer = sst_getfilteredrange(SSTPid, StartKey, - EndKey, + EndKey, Width, SegList, LowLastMod), ExpPointer ++ Tail. --spec sst_getkvrange(pid(), - range_endpoint(), - range_endpoint(), - integer()) +-spec sst_getkvrange(pid(), + range_endpoint(), + range_endpoint(), + integer()) -> list(leveled_codec:ledger_kv()|slot_pointer()). %% @doc %% Get a range of {Key, Value} pairs as a list between StartKey and EndKey @@ -1080,15 +1068,15 @@ expand_list_by_pointer({next, ManEntry, StartKey, EndKey}, %% will be placed on the tail of the resulting list if results expand beyond %% the Scan Width sst_getkvrange(Pid, StartKey, EndKey, ScanWidth) -> - sst_getfilteredrange(Pid, StartKey, EndKey, ScanWidth, false, 0). + sst_getfilteredrange(Pid, StartKey, EndKey, ScanWidth, false, 0). --spec sst_getfilteredrange(pid(), - range_endpoint(), - range_endpoint(), +-spec sst_getfilteredrange(pid(), + range_endpoint(), + range_endpoint(), integer(), leveled_codec:segment_list(), - non_neg_integer()) + non_neg_integer()) -> list(leveled_codec:ledger_kv()|slot_pointer()). %% @doc %% Get a range of {Key, Value} pairs as a list between StartKey and EndKey @@ -1099,21 +1087,20 @@ sst_getkvrange(Pid, StartKey, EndKey, ScanWidth) -> %% To make the range open-ended (either to start, end or both) the all atom %% can be used in place of the Key tuple. %% -%% A segment list can also be passed, which inidcates a subset of segment -%% hashes of interest in the query. +%% A segment list can also be passed, which inidcates a subset of segment +%% hashes of interest in the query. %% %% TODO: Optimise this so that passing a list of segments that tune to the -%% same hash is faster - perhaps provide an exportable function in +%% same hash is faster - perhaps provide an exportable function in %% leveled_tictac sst_getfilteredrange(Pid, StartKey, EndKey, ScanWidth, SegList, LowLastMod) -> SegList0 = tune_seglist(SegList), - case gen_fsm:sync_send_event(Pid, - {get_kvrange, - StartKey, EndKey, - ScanWidth, SegList0, LowLastMod}, - infinity) of + case gen_statem:call(Pid, {get_kvrange, + StartKey, EndKey, + ScanWidth, SegList0, LowLastMod}, + infinity) of {yield, SlotsToFetchBinList, SlotsToPoint, PressMethod, IdxModDate} -> - {L, _BIC} = + {L, _BIC} = binaryslot_reader(SlotsToFetchBinList, PressMethod, IdxModDate, SegList0), L ++ SlotsToPoint; @@ -1125,13 +1112,13 @@ sst_getfilteredrange(Pid, StartKey, EndKey, ScanWidth, SegList, LowLastMod) -> -> list(leveled_codec:ledger_kv()). %% @doc %% Get a list of slots by their ID. The slot will be converted from the binary -%% to term form outside of the FSM loop, this is to stop the copying of the +%% to term form outside of the FSM loop, this is to stop the copying of the %% converted term to the calling process. sst_getslots(Pid, SlotList) -> sst_getfilteredslots(Pid, SlotList, false, 0). --spec sst_getfilteredslots(pid(), - list(slot_pointer()), +-spec sst_getfilteredslots(pid(), + list(slot_pointer()), leveled_codec:segment_list(), non_neg_integer()) -> list(leveled_codec:ledger_kv()). @@ -1139,21 +1126,20 @@ sst_getslots(Pid, SlotList) -> %% Get a list of slots by their ID. The slot will be converted from the binary %% to term form outside of the FSM loop %% -%% A list of 16-bit integer Segment IDs can be passed to filter the keys -%% returned (not precisely - with false results returned in addition). Use +%% A list of 16-bit integer Segment IDs can be passed to filter the keys +%% returned (not precisely - with false results returned in addition). Use %% false as a SegList to not filter. %% An integer can be provided which gives a floor for the LastModified Date %% of the object, if the object is to be covered by the query sst_getfilteredslots(Pid, SlotList, SegList, LowLastMod) -> SegL0 = tune_seglist(SegList), - {NeedBlockIdx, SlotBins, PressMethod, IdxModDate} = - gen_fsm:sync_send_event(Pid, - {get_slots, SlotList, SegL0, LowLastMod}, - infinity), + {NeedBlockIdx, SlotBins, PressMethod, IdxModDate} = + gen_statem:call( + Pid, {get_slots, SlotList, SegL0, LowLastMod}, infinity), {L, BIC} = binaryslot_reader(SlotBins, PressMethod, IdxModDate, SegL0), case NeedBlockIdx of true -> - gen_fsm:send_all_state_event(Pid, {update_blockindex_cache, BIC}); + erlang:send(Pid, {update_blockindex_cache, BIC}); false -> ok end, @@ -1187,7 +1173,7 @@ find_pos(_BinRem, _Hash, PosList, _Count) -> PosList. --spec member_check(non_neg_integer(), +-spec member_check(non_neg_integer(), non_neg_integer()| {list, list(non_neg_integer())}| {sets, sets:set(non_neg_integer())}) -> boolean(). @@ -1277,7 +1263,7 @@ tune_hash(SegHash) -> %% @doc %% Only 15 bits of the hash is ever interesting tune_seglist(SegList) -> - case is_list(SegList) of + case is_list(SegList) of true -> SL0 = lists:usort(lists:map(fun tune_hash/1, SegList)), case length(SL0) > ?USE_SET_FOR_SPEED of @@ -1324,11 +1310,11 @@ updatebic_foldfun(HMDRequired) -> {AccCount, Cache, AccHMD} end end. - + -spec update_blockindex_cache( boolean(), list({integer(), binary()}), blockindex_cache(), non_neg_integer()|undefined, - boolean()) -> + boolean()) -> {boolean(), blockindex_cache(), non_neg_integer()|undefined}. update_blockindex_cache(true, Entries, BIC, HighModDate, IdxModDate) -> case {element(1, BIC), array:size(element(2, BIC))} of @@ -1366,7 +1352,7 @@ check_modified(_, _, _) -> true. -spec fetch( - leveled_codec:ledger_key(), + leveled_codec:ledger_key(), leveled_codec:segment_hash(), sst_summary(), press_method(), @@ -1377,7 +1363,7 @@ check_modified(_, _, _) -> fetch_cache(), file:fd(), leveled_pmanifest:lsm_level(), - leveled_monitor:monitor()) + leveled_monitor:monitor()) -> {not_present|leveled_codec:ledger_kv(), blockindex_cache()|no_update, non_neg_integer()|undefined|no_update, @@ -1395,11 +1381,11 @@ fetch(LedgerKey, Hash, lookup_slot(LedgerKey, Summary#summary.index, FilterFun), SlotID = Slot#slot_index_value.slot_id, CachedBlockIdx = array:get(SlotID - 1, element(2, BIC)), - - case extract_header(CachedBlockIdx, IndexModDate) of + + case extract_header(CachedBlockIdx, IndexModDate) of none -> SlotBin = read_slot(Handle, Slot), - {Result, Header} = + {Result, Header} = binaryslot_get( SlotBin, LedgerKey, Hash, PressMethod, IndexModDate), {_UpdateState, BIC0, HMD0} = @@ -1419,25 +1405,25 @@ fetch(LedgerKey, Hash, {Result, BIC0, HMD0, no_update}; {BlockLengths, _LMD, PosBin} -> PosList = find_pos(PosBin, extract_hash(Hash), [], 0), - case PosList of + case PosList of [] -> maybelog_fetch_timing(Monitor, Level, not_found, SW0), {not_present, no_update, no_update, no_update}; _ -> CacheHash = cache_hash(Hash, Level), - case fetch_from_cache(CacheHash, FetchCache) of + case fetch_from_cache(CacheHash, FetchCache) of {LedgerKey, V} -> maybelog_fetch_timing( Monitor, Level, fetch_cache, SW0), {{LedgerKey, V}, no_update, no_update, no_update}; _ -> StartPos = Slot#slot_index_value.start_position, - Result = + Result = check_blocks(PosList, {Handle, StartPos}, BlockLengths, byte_size(PosBin), - LedgerKey, + LedgerKey, PressMethod, IndexModDate, not_present), @@ -1454,24 +1440,24 @@ fetch(LedgerKey, Hash, maybelog_fetch_timing( Monitor, Level, slot_cachedblock, SW0), {Result, - no_update, no_update, FetchCache0} + no_update, no_update, FetchCache0} end end - end + end end. -spec fetch_range(tuple(), tuple(), integer(), - leveled_codec:segment_list(), non_neg_integer(), + leveled_codec:segment_list(), non_neg_integer(), sst_state()) -> {boolean(), list(), list()}. %% @doc -%% Fetch the contents of the SST file for a given key range. This will +%% Fetch the contents of the SST file for a given key range. This will %% pre-fetch some results, and append pointers for additional results. %% -%% A filter can be provided based on the Segment ID (usable for hashable +%% A filter can be provided based on the Segment ID (usable for hashable %% objects not no_lookup entries) to accelerate the query if the 5-arity -%% version is used +%% version is used fetch_range(StartKey, EndKey, ScanWidth, SegList, LowLastMod, State) -> Summary = State#state.summary, Handle = State#state.handle, @@ -1483,8 +1469,8 @@ fetch_range(StartKey, EndKey, ScanWidth, SegList, LowLastMod, State) -> State#state.filter_fun), Self = self(), SL = length(Slots), - - ExpandedSlots = + + ExpandedSlots = case SL of 1 -> [Slot] = Slots, @@ -1495,7 +1481,7 @@ fetch_range(StartKey, EndKey, ScanWidth, SegList, LowLastMod, State) -> [{pointer, Self, Slot, StartKey, all}] end; N -> - {LSlot, MidSlots, RSlot} = + {LSlot, MidSlots, RSlot} = case N of 2 -> [Slot1, Slot2] = Slots, @@ -1520,7 +1506,7 @@ fetch_range(StartKey, EndKey, ScanWidth, SegList, LowLastMod, State) -> [{pointer, Self, RSlot, all, all}] end end, - {SlotsToFetch, SlotsToPoint} = + {SlotsToFetch, SlotsToPoint} = case ScanWidth of SW when SW >= SL -> {ExpandedSlots, []}; @@ -1528,9 +1514,9 @@ fetch_range(StartKey, EndKey, ScanWidth, SegList, LowLastMod, State) -> lists:split(ScanWidth, ExpandedSlots) end, - {NeededBlockIdx, SlotsToFetchBinList} = - read_slots(Handle, - SlotsToFetch, + {NeededBlockIdx, SlotsToFetchBinList} = + read_slots(Handle, + SlotsToFetch, {SegList, LowLastMod, State#state.blockindex_cache}, State#state.compression_method, State#state.index_moddate), @@ -1565,7 +1551,7 @@ write_file(RootPath, Filename, SummaryBin, SlotsBin, false -> ok end, - + ok = leveled_util:safe_rename(filename:join(RootPath, PendingName), filename:join(RootPath, FinalName), < - {Handle, FileVersion, SummaryBin} = + {Handle, FileVersion, SummaryBin} = open_reader( filename:join(State#state.root_path, Filename), LoadPageCache), @@ -1600,11 +1586,11 @@ read_file(Filename, State, LoadPageCache) -> Bloom}. gen_fileversion(PressMethod, IdxModDate, CountOfTombs) -> - % Native or none can be treated the same once written, as reader - % does not need to know as compression info will be in header of the + % Native or none can be treated the same once written, as reader + % does not need to know as compression info will be in header of the % block - Bit1 = - case PressMethod of + Bit1 = + case PressMethod of lz4 -> 1; native -> 0; none -> 0 @@ -1616,7 +1602,7 @@ gen_fileversion(PressMethod, IdxModDate, CountOfTombs) -> false -> 0 end, - Bit3 = + Bit3 = case CountOfTombs of not_counted -> 0; @@ -1626,8 +1612,8 @@ gen_fileversion(PressMethod, IdxModDate, CountOfTombs) -> Bit1 + Bit2 + Bit3. imp_fileversion(VersionInt, State) -> - UpdState0 = - case VersionInt band 1 of + UpdState0 = + case VersionInt band 1 of 0 -> State#state{compression_method = native}; 1 -> @@ -1650,8 +1636,8 @@ imp_fileversion(VersionInt, State) -> open_reader(Filename, LoadPageCache) -> {ok, Handle} = file:open(Filename, [binary, raw, read]), {ok, Lengths} = file:pread(Handle, 0, 9), - <> = Lengths, case LoadPageCache of true -> @@ -1669,10 +1655,10 @@ build_table_summary(SlotIndex, _Level, FirstKey, last_key = LastKey, size = SlotCount, max_sqn = MaxSQN}, - SummBin0 = - term_to_binary({Summary, Bloom, lists:reverse(SlotIndex)}, + SummBin0 = + term_to_binary({Summary, Bloom, lists:reverse(SlotIndex)}, ?BINARY_SETTINGS), - + SummBin = case CountOfTombs of not_counted -> @@ -1684,7 +1670,7 @@ build_table_summary(SlotIndex, _Level, FirstKey, SummCRC = hmac(SummBin), <>. --spec read_table_summary(binary(), not_counted|non_neg_integer()) -> +-spec read_table_summary(binary(), not_counted|non_neg_integer()) -> {sst_summary(), leveled_ebloom:bloom(), list(tuple()), @@ -1711,7 +1697,7 @@ read_table_summary(BinWithCheck, TombCount) -> build_all_slots(SlotList) -> SlotCount = length(SlotList), - {SlotIndex, BlockIndex, SlotsBin, HashLists} = + {SlotIndex, BlockIndex, SlotsBin, HashLists} = build_all_slots(SlotList, 9, 1, @@ -1753,13 +1739,13 @@ generate_filenames(RootFilename) -> FP_NOEXT = filename:basename(RootFilename, Ext), {filename:join(DN, FP_NOEXT) ++ ".pnd", filename:join(DN, FP_NOEXT) ++ ".sst"} - end. + end. -spec serialise_block(any(), press_method()) -> binary(). %% @doc %% Convert term to binary -%% Function split out to make it easier to experiment with different +%% Function split out to make it easier to experiment with different %% compression methods. Also, perhaps standardise applictaion of CRC %% checks serialise_block(Term, lz4) -> @@ -1779,14 +1765,14 @@ serialise_block(Term, none) -> -spec deserialise_block(binary(), press_method()) -> any(). %% @doc %% Convert binary to term -%% Function split out to make it easier to experiment with different -%% compression methods. +%% Function split out to make it easier to experiment with different +%% compression methods. %% -%% If CRC check fails we treat all the data as missing +%% If CRC check fails we treat all the data as missing deserialise_block(Bin, PressMethod) when byte_size(Bin) > 4 -> BinS = byte_size(Bin) - 4, <> = Bin, - case hmac(TermBin) of + case hmac(TermBin) of CRC32 -> deserialise_checkedblock(TermBin, PressMethod); _ -> @@ -1852,7 +1838,7 @@ get_filterfun( key_prefix_filter(N, Prefix); _ -> fun key_filter/1 - + end; get_filterfun(_FirstKey, _LastKey) -> fun null_filter/1. @@ -1937,20 +1923,20 @@ lookup_slots(StartKey, EndKey, Tree, FilterFun) -> %%%============================================================================ %% Implementing a slot has gone through numerous iterations. One of the most -%% critical considerations has been the cost of the binary_to_term and +%% critical considerations has been the cost of the binary_to_term and %% term_to_binary calls for different sizes of slots and different data types. %% %% Microbenchmarking indicated that flat lists were the fastest at sst build %% time. However, the lists need scanning at query time - and so give longer %% lookups. Bigger slots did better at term_to_binary time. However %% binary_to_term is an often repeated task, and this is better with smaller -%% slots. +%% slots. %% -%% The outcome has been to divide the slot into four small blocks to minimise -%% the binary_to_term time. A binary index is provided for the slot for all +%% The outcome has been to divide the slot into four small blocks to minimise +%% the binary_to_term time. A binary index is provided for the slot for all %% Keys that are directly fetchable (i.e. standard keys not index keys). %% -%% The division and use of a list saves about 100 microseconds per fetch when +%% The division and use of a list saves about 100 microseconds per fetch when %% compared to using a 128-member gb:tree. %% %% The binary index is cacheable and doubles as a not_present filter, as it is @@ -1973,9 +1959,9 @@ accumulate_positions({K, V}, {PosBinAcc, NoHashCount, HashAcc, LMDAcc}) -> {_SQN, H1, LMD} = leveled_codec:strip_to_indexdetails({K, V}), LMDAcc0 = take_max_lastmoddate(LMD, LMDAcc), PosH1 = extract_hash(H1), - case is_integer(PosH1) of + case is_integer(PosH1) of true -> - case NoHashCount of + case NoHashCount of 0 -> {<<1:1/integer, PosH1:15/integer,PosBinAcc/binary>>, 0, @@ -1986,9 +1972,9 @@ accumulate_positions({K, V}, {PosBinAcc, NoHashCount, HashAcc, LMDAcc}) -> % and so at read time should count NHC + 1 NHC = N - 1, {<<1:1/integer, - PosH1:15/integer, + PosH1:15/integer, 0:1/integer, - NHC:7/integer, + NHC:7/integer, PosBinAcc/binary>>, 0, [H1|HashAcc], @@ -2000,7 +1986,7 @@ accumulate_positions({K, V}, {PosBinAcc, NoHashCount, HashAcc, LMDAcc}) -> -spec take_max_lastmoddate(leveled_codec:last_moddate(), - leveled_codec:last_moddate()) -> + leveled_codec:last_moddate()) -> leveled_codec:last_moddate(). %% @doc %% Get the last modified date. If no Last Modified Date on any object, can't @@ -2021,16 +2007,16 @@ take_max_lastmoddate(LMD, LMDAcc) -> %% Generate the serialised slot to be used when storing this sublist of keys %% and values generate_binary_slot(Lookup, KVL, PressMethod, IndexModDate, BuildTimings0) -> - + SW0 = os:timestamp(), - - {HashL, PosBinIndex, LMD} = + + {HashL, PosBinIndex, LMD} = case Lookup of lookup -> InitAcc = {<<>>, 0, [], 0}, - {PosBinIndex0, NHC, HashL0, LMD0} = + {PosBinIndex0, NHC, HashL0, LMD0} = lists:foldr(fun accumulate_positions/2, InitAcc, KVL), - PosBinIndex1 = + PosBinIndex1 = case NHC of 0 -> PosBinIndex0; @@ -2042,9 +2028,9 @@ generate_binary_slot(Lookup, KVL, PressMethod, IndexModDate, BuildTimings0) -> no_lookup -> {[], <<0:1/integer, 127:7/integer>>, 0} end, - + BuildTimings1 = update_buildtimings(SW0, BuildTimings0, slot_hashlist), - SW1 = os:timestamp(), + SW1 = os:timestamp(), {SideBlockSize, MidBlockSize} = case Lookup of @@ -2054,19 +2040,19 @@ generate_binary_slot(Lookup, KVL, PressMethod, IndexModDate, BuildTimings0) -> ?NOLOOK_BLOCKSIZE end, - {B1, B2, B3, B4, B5} = - case length(KVL) of + {B1, B2, B3, B4, B5} = + case length(KVL) of L when L =< SideBlockSize -> {serialise_block(KVL, PressMethod), - <<0:0>>, - <<0:0>>, + <<0:0>>, + <<0:0>>, <<0:0>>, <<0:0>>}; L when L =< 2 * SideBlockSize -> {KVLA, KVLB} = lists:split(SideBlockSize, KVL), {serialise_block(KVLA, PressMethod), serialise_block(KVLB, PressMethod), - <<0:0>>, + <<0:0>>, <<0:0>>, <<0:0>>}; L when L =< (2 * SideBlockSize + MidBlockSize) -> @@ -2099,10 +2085,10 @@ generate_binary_slot(Lookup, KVL, PressMethod, IndexModDate, BuildTimings0) -> end, BuildTimings2 = update_buildtimings(SW1, BuildTimings1, slot_serialise), - SW2 = os:timestamp(), + SW2 = os:timestamp(), - B1P = - case IndexModDate of + B1P = + case IndexModDate of true -> byte_size(PosBinIndex) + ?BLOCK_LENGTHS_LENGTH + ?LMD_LENGTH; false -> @@ -2114,29 +2100,29 @@ generate_binary_slot(Lookup, KVL, PressMethod, IndexModDate, BuildTimings0) -> B3L = byte_size(B3), B4L = byte_size(B4), B5L = byte_size(B5), - Header = - case IndexModDate of + Header = + case IndexModDate of true -> - <>; false -> - <> end, CheckH = hmac(Header), SlotBin = <>, - + {LastKey, _LV} = lists:last(KVL), BuildTimings3 = update_buildtimings(SW2, BuildTimings2, slot_finish), @@ -2144,11 +2130,11 @@ generate_binary_slot(Lookup, KVL, PressMethod, IndexModDate, BuildTimings0) -> {{Header, SlotBin, HashL, LastKey}, BuildTimings3}. --spec check_blocks(list(integer()), +-spec check_blocks(list(integer()), binary()|{file:io_device(), integer()}, binary(), integer(), - leveled_codec:ledger_key()|false, + leveled_codec:ledger_key()|false, %% if false the acc is a list, and if true %% Acc will be initially not_present, and may %% result in a {K, V} tuple @@ -2169,28 +2155,42 @@ check_blocks([], _BlockPointer, _BlockLengths, _PosBinLength, check_blocks([Pos|Rest], BlockPointer, BlockLengths, PosBinLength, LedgerKeyToCheck, PressMethod, IdxModDate, Acc) -> {BlockNumber, BlockPos} = revert_position(Pos), - BlockBin = - read_block(BlockPointer, - BlockLengths, + BlockBin = + read_block(BlockPointer, + BlockLengths, PosBinLength, BlockNumber, additional_offset(IdxModDate)), - R = fetchfrom_rawblock(BlockPos, deserialise_block(BlockBin, PressMethod)), - case {R, LedgerKeyToCheck} of + Result = spawn_check_block(BlockPos, BlockBin, PressMethod), + case {Result, LedgerKeyToCheck} of {{K, V}, K} -> {K, V}; {{K, V}, false} -> - check_blocks(Rest, BlockPointer, - BlockLengths, PosBinLength, + check_blocks(Rest, BlockPointer, + BlockLengths, PosBinLength, LedgerKeyToCheck, PressMethod, IdxModDate, [{K, V}|Acc]); _ -> - check_blocks(Rest, BlockPointer, - BlockLengths, PosBinLength, + check_blocks(Rest, BlockPointer, + BlockLengths, PosBinLength, LedgerKeyToCheck, PressMethod, IdxModDate, Acc) end. +-spec spawn_check_block(non_neg_integer(), binary(), press_method()) + -> not_present|leveled_codec:ledger_kv(). +spawn_check_block(BlockPos, BlockBin, PressMethod) -> + Parent = self(), + Pid = + spawn_link( + fun() -> check_block(Parent, BlockPos, BlockBin, PressMethod) end + ), + receive {checked_block, Pid, R} -> R end. + +check_block(From, BlockPos, BlockBin, PressMethod) -> + R = fetchfrom_rawblock(BlockPos, deserialise_block(BlockBin, PressMethod)), + From ! {checked_block, self(), R}. + -spec additional_offset(boolean()) -> pos_integer(). %% @doc %% 4-byte CRC, 4-byte pos, 4-byte CRC, 5x4 byte lengths, 4 byte LMD @@ -2224,18 +2224,18 @@ read_slot(Handle, Slot) -> pointer_mapfun(Pointer) -> - {Slot, SK, EK} = - case Pointer of + {Slot, SK, EK} = + case Pointer of {pointer, _Pid, Slot0, SK0, EK0} -> {Slot0, SK0, EK0}; {pointer, Slot0, SK0, EK0} -> {Slot0, SK0, EK0} end, - {Slot#slot_index_value.start_position, + {Slot#slot_index_value.start_position, Slot#slot_index_value.length, Slot#slot_index_value.slot_id, - SK, + SK, EK}. @@ -2248,26 +2248,26 @@ pointer_mapfun(Pointer) -> %% @doc %% Return a function that can pull individual slot binaries from a binary %% covering multiple slots -binarysplit_mapfun(MultiSlotBin, StartPos) -> - fun({SP, L, ID, SK, EK}) -> +binarysplit_mapfun(MultiSlotBin, StartPos) -> + fun({SP, L, ID, SK, EK}) -> Start = SP - StartPos, <<_Pre:Start/binary, SlotBin:L/binary, _Post/binary>> = MultiSlotBin, {SlotBin, ID, SK, EK} end. --spec read_slots(file:io_device(), list(), +-spec read_slots(file:io_device(), list(), {false|list(), non_neg_integer(), blockindex_cache()}, - press_method(), boolean()) -> + press_method(), boolean()) -> {boolean(), list(binaryslot_element())}. %% @doc -%% The reading of sots will return a list of either 2-tuples containing -%% {K, V} pairs - or 3-tuples containing {Binary, SK, EK}. The 3 tuples +%% The reading of sots will return a list of either 2-tuples containing +%% {K, V} pairs - or 3-tuples containing {Binary, SK, EK}. The 3 tuples %% can be exploded into lists of {K, V} pairs using the binaryslot_reader/4 %% function %% -%% Reading slots is generally unfiltered, but in the sepcial case when -%% querting across slots when only matching segment IDs are required the +%% Reading slots is generally unfiltered, but in the sepcial case when +%% querting across slots when only matching segment IDs are required the %% BlockIndexCache can be used %% %% Note that false positives will be passed through. It is important that @@ -2279,27 +2279,27 @@ read_slots(Handle, SlotList, {false, 0, _BlockIndexCache}, % No list of segments passed or useful Low LastModified Date % Just read slots in SlotList {false, read_slotlist(SlotList, Handle)}; -read_slots(Handle, SlotList, {SegList, LowLastMod, BlockIndexCache}, +read_slots(Handle, SlotList, {SegList, LowLastMod, BlockIndexCache}, PressMethod, IdxModDate) -> % List of segments passed so only {K, V} pairs matching those segments - % should be returned. This required the {K, V} pair to have been added - % with the appropriate hash - if the pair were added with no_lookup as + % should be returned. This required the {K, V} pair to have been added + % with the appropriate hash - if the pair were added with no_lookup as % the hash value this will fail unexpectedly. - BinMapFun = + BinMapFun = fun(Pointer, {NeededBlockIdx, Acc}) -> {SP, _L, ID, SK, EK} = pointer_mapfun(Pointer), CachedHeader = array:get(ID - 1, element(2, BlockIndexCache)), case extract_header(CachedHeader, IdxModDate) of none -> % If there is an attempt to use the seg list query and the - % index block cache isn't cached for any part this may be + % index block cache isn't cached for any part this may be % slower as each slot will be read in turn {true, Acc ++ read_slotlist([Pointer], Handle)}; {BlockLengths, LMD, BlockIdx} -> - % If there is a BlockIndex cached then we can use it to - % check to see if any of the expected segments are - % present without lifting the slot off disk. Also the - % fact that we know position can be used to filter out + % If there is a BlockIndex cached then we can use it to + % check to see if any of the expected segments are + % present without lifting the slot off disk. Also the + % fact that we know position can be used to filter out % other keys % % Note that LMD will be 0 if the indexing of last mod @@ -2321,14 +2321,14 @@ read_slots(Handle, SlotList, {SegList, LowLastMod, BlockIndexCache}, read_slotlist([Pointer], Handle)}; _SL -> % Need to find just the right keys - PositionList = + PositionList = find_pos(BlockIdx, SegList, [], 0), % Note check_blocks should return [] if % PositionList is empty (which it may be) KVL = check_blocks(PositionList, - {Handle, SP}, - BlockLengths, + {Handle, SP}, + BlockLengths, byte_size(BlockIdx), false, PressMethod, @@ -2343,7 +2343,7 @@ read_slots(Handle, SlotList, {SegList, LowLastMod, BlockIndexCache}, Acc ++ lists:filter(FilterFun, KVL)} end end - end + end end, lists:foldl(BinMapFun, {false, []}, SlotList). @@ -2366,19 +2366,19 @@ read_slotlist(SlotList, Handle) -> lists:map(binarysplit_mapfun(MultiSlotBin, StartPos), LengthList). --spec binaryslot_reader(list(binaryslot_element()), +-spec binaryslot_reader(list(binaryslot_element()), press_method(), boolean(), - leveled_codec:segment_list()) - -> {list({tuple(), tuple()}), + leveled_codec:segment_list()) + -> {list({tuple(), tuple()}), list({integer(), binary()})}. %% @doc -%% Read the binary slots converting them to {K, V} pairs if they were not +%% Read the binary slots converting them to {K, V} pairs if they were not %% already {K, V} pairs. If they are already {K, V} pairs it is assumed %% that they have already been range checked before extraction. %% %% Keys which are still to be extracted from the slot, are accompanied at -%% this function by the range against which the keys need to be checked. +%% this function by the range against which the keys need to be checked. %% This range is passed with the slot to binaryslot_trimmedlist which should %% open the slot block by block, filtering individual keys where the endpoints %% of the block are outside of the range, and leaving blocks already proven to @@ -2387,48 +2387,48 @@ binaryslot_reader(SlotBinsToFetch, PressMethod, IdxModDate, SegList) -> % Two accumulators are added. % One to collect the list of keys and values found in the binary slots % (subject to range filtering if the slot is still deserialised at this - % stage. + % stage. % The second accumulator extracts the header information from the slot, so % that the cache can be built for that slot. This is used by the handling % of get_kvreader calls. This means that slots which are only used in % range queries can still populate their block_index caches (on the FSM % loop state), and those caches can be used for future queries. - binaryslot_reader(SlotBinsToFetch, + binaryslot_reader(SlotBinsToFetch, PressMethod, IdxModDate, SegList, [], []). binaryslot_reader([], _PressMethod, _IdxModDate, _SegList, Acc, BIAcc) -> {Acc, BIAcc}; -binaryslot_reader([{SlotBin, ID, SK, EK}|Tail], +binaryslot_reader([{SlotBin, ID, SK, EK}|Tail], PressMethod, IdxModDate, SegList, Acc, BIAcc) -> % The start key and end key here, may not the start key and end key the % application passed into the query. If the slot is known to lie entirely % inside the range, on either of both sides, the SK and EK may be % substituted for the 'all' key work to indicate there is no need for % entries in this slot to be trimmed from either or both sides. - {TrimmedL, BICache} = - binaryslot_trimmedlist(SlotBin, - SK, EK, + {TrimmedL, BICache} = + binaryslot_trimmedlist(SlotBin, + SK, EK, PressMethod, IdxModDate, SegList), - binaryslot_reader(Tail, + binaryslot_reader(Tail, PressMethod, IdxModDate, SegList, Acc ++ TrimmedL, [{ID, BICache}|BIAcc]); -binaryslot_reader([{K, V}|Tail], +binaryslot_reader([{K, V}|Tail], PressMethod, IdxModDate, SegList, Acc, BIAcc) -> % These entries must already have been filtered for membership inside any % range used in the query. - binaryslot_reader(Tail, + binaryslot_reader(Tail, PressMethod, IdxModDate, SegList, Acc ++ [{K, V}], BIAcc). - + read_length_list(Handle, LengthList) -> StartPos = element(1, lists:nth(1, LengthList)), - EndPos = element(1, lists:last(LengthList)) + EndPos = element(1, lists:last(LengthList)) + element(2, lists:last(LengthList)), {ok, MultiSlotBin} = file:pread(Handle, StartPos, EndPos - StartPos), {MultiSlotBin, StartPos}. @@ -2440,7 +2440,7 @@ read_length_list(Handle, LengthList) -> %% Helper for extracting the binaries from the header ignoring the missing LMD %% if LMD is not indexed extract_header(none, _IdxModDate) -> - none; % used when the block cache has returned none + none; % used when the block cache has returned none extract_header(Header, true) -> BL = ?BLOCK_LENGTHS_LENGTH, <> = Header, @@ -2451,13 +2451,13 @@ extract_header(Header, false) -> {BlockLengths, 0, PosBinIndex}. binaryslot_get(FullBin, Key, Hash, PressMethod, IdxModDate) -> - case crc_check_slot(FullBin) of + case crc_check_slot(FullBin) of {Header, Blocks} -> {BlockLengths, _LMD, PosBinIndex} = extract_header(Header, IdxModDate), PosList = find_pos(PosBinIndex, - extract_hash(Hash), - [], + extract_hash(Hash), + [], 0), {fetch_value(PosList, BlockLengths, Blocks, Key, PressMethod), Header}; @@ -2467,9 +2467,9 @@ binaryslot_get(FullBin, Key, Hash, PressMethod, IdxModDate) -> end. binaryslot_tolist(FullBin, PressMethod, IdxModDate) -> - BlockFetchFun = + BlockFetchFun = fun(Length, {Acc, Bin}) -> - case Length of + case Length of 0 -> {Acc, Bin}; _ -> @@ -2478,8 +2478,8 @@ binaryslot_tolist(FullBin, PressMethod, IdxModDate) -> end end, - {Out, _Rem} = - case crc_check_slot(FullBin) of + {Out, _Rem} = + case crc_check_slot(FullBin) of {Header, Blocks} -> {BlockLengths, _LMD, _PosBinIndex} = extract_header(Header, IdxModDate), @@ -2504,9 +2504,9 @@ binaryslot_trimmedlist(FullBin, StartKey, EndKey, PressMethod, IdxModDate, SegList) -> LTrimFun = fun({K, _V}) -> K < StartKey end, RTrimFun = fun({K, _V}) -> not leveled_codec:endkey_passed(EndKey, K) end, - BlockCheckFun = + BlockCheckFun = fun(Block, {Acc, Continue}) -> - case {Block, Continue} of + case {Block, Continue} of {<<>>, _} -> {Acc, false}; {_, true} -> @@ -2517,19 +2517,19 @@ binaryslot_trimmedlist(FullBin, StartKey, EndKey, false -> Block end, - case fetchends_rawblock(BlockList) of + case fetchends_rawblock(BlockList) of {_, LastKey} when StartKey > LastKey -> %% This includes the case when LastKey is %% not_present due to corruption in the BlockList %% as tuple is > not_present. {Acc, true}; {_, LastKey} -> - {_LDrop, RKeep} = lists:splitwith(LTrimFun, + {_LDrop, RKeep} = lists:splitwith(LTrimFun, BlockList), - case leveled_codec:endkey_passed(EndKey, + case leveled_codec:endkey_passed(EndKey, LastKey) of true -> - {LKeep, _RDrop} + {LKeep, _RDrop} = lists:splitwith(RTrimFun, RKeep), {Acc ++ LKeep, false}; false -> @@ -2540,7 +2540,7 @@ binaryslot_trimmedlist(FullBin, StartKey, EndKey, {Acc, false} end end, - + case {crc_check_slot(FullBin), SegList} of % It will be more effecient to check a subset of blocks. To work out % the best subset we always look in the middle block of 5, and based on @@ -2561,7 +2561,7 @@ binaryslot_trimmedlist(FullBin, StartKey, EndKey, <> = Blocks, - BlocksToCheck = + BlocksToCheck = blocks_required({StartKey, EndKey}, [Block1, Block2, MidBlock, Block4, Block5], PressMethod), @@ -2575,12 +2575,12 @@ binaryslot_trimmedlist(FullBin, StartKey, EndKey, KVL = check_blocks(PosList, FullBin, BlockLengths, - byte_size(BlockIdx), - false, + byte_size(BlockIdx), + false, PressMethod, IdxModDate, []), - {KVL, Header}; + {KVL, Header}; {crc_wonky, _} -> {[], none} end. @@ -2623,15 +2623,15 @@ filter_blocks_required({MidFirst, MidLast}, {_StartKey, EndKey}, crc_check_slot(FullBin) -> - <> = FullBin, - PosBL0 = min(PosBL, byte_size(FullBin) - 12), + PosBL0 = min(PosBL, byte_size(FullBin) - 12), % If the position has been bit-flipped to beyond the maximum possible % length, use the maximum possible length <> = Rest, - case {hmac(Header), hmac(PosBL0)} of + case {hmac(Header), hmac(PosBL0)} of {CRC32H, CRC32PBL} -> {Header, Blocks}; _ -> @@ -2675,11 +2675,11 @@ fetch_value([Pos|Rest], BlockLengths, Blocks, Key, PressMethod) -> {BlockNumber, BlockPos} = revert_position(Pos), {Offset, Length} = block_offsetandlength(BlockLengths, BlockNumber), <<_Pre:Offset/binary, Block:Length/binary, _Rest/binary>> = Blocks, - RawBlock = deserialise_block(Block, PressMethod), - case fetchfrom_rawblock(BlockPos, RawBlock) of + R = fetchfrom_rawblock(BlockPos, deserialise_block(Block, PressMethod)), + case R of {K, V} when K == Key -> {K, V}; - _ -> + _ -> fetch_value(Rest, BlockLengths, Blocks, Key, PressMethod) end. @@ -2761,7 +2761,7 @@ revert_position(Pos) -> %% there are matching keys then the highest sequence number must be chosen and %% any lower sequence numbers should be compacted out of existence --spec merge_lists(list(), sst_options(), boolean()) +-spec merge_lists(list(), sst_options(), boolean()) -> {list(), list(), list(binary_slot()), tuple()|null, non_neg_integer()|not_counted}. %% @doc @@ -2780,31 +2780,38 @@ merge_lists(KVList1, SSTOpts, IdxModDate) -> split_lists([], SlotLists, 0, _PressMethod, _IdxModDate) -> lists:reverse(SlotLists); split_lists(LastPuff, SlotLists, 0, PressMethod, IdxModDate) -> - {SlotD, _} = - generate_binary_slot(lookup, LastPuff, PressMethod, IdxModDate, no_timing), + {SlotD, _} = + generate_binary_slot( + lookup, LastPuff, PressMethod, IdxModDate, no_timing), lists:reverse([SlotD|SlotLists]); split_lists(KVList1, SlotLists, N, PressMethod, IdxModDate) -> {Slot, KVListRem} = lists:split(?LOOK_SLOTSIZE, KVList1), - {SlotD, _} = + {SlotD, _} = generate_binary_slot(lookup, Slot, PressMethod, IdxModDate, no_timing), split_lists(KVListRem, [SlotD|SlotLists], N - 1, PressMethod, IdxModDate). --spec merge_lists(list(), list(), tuple(), sst_options(), - boolean(), boolean()) -> - {list(), list(), list(binary_slot()), tuple()|null, - non_neg_integer()}. +-spec merge_lists( + list(expanded_pointer()), + list(expanded_pointer()), + {boolean(), non_neg_integer()}, + sst_options(), boolean(), boolean()) -> + {list(expanded_pointer()), + list(expanded_pointer()), + list(binary_slot()), + leveled_codec:ledger_key()|null, + non_neg_integer()}. %% @doc -%% Merge lists when merging across more than one file. KVLists that are +%% Merge lists when merging across more than one file. KVLists that are %% provided may include pointers to fetch more Keys/Values from the source %% file -merge_lists(KVList1, KVList2, LevelInfo, SSTOpts, - IndexModDate, SaveTombCount) -> +merge_lists( + KVList1, KVList2, LevelInfo, SSTOpts, IndexModDate, SaveTombCount) -> InitTombCount = case SaveTombCount of true -> 0; false -> not_counted end, - merge_lists(KVList1, KVList2, - LevelInfo, - [], null, 0, + merge_lists(KVList1, KVList2, + LevelInfo, + [], null, 0, SSTOpts#sst_options.max_sstslots, SSTOpts#sst_options.press_method, IndexModDate, @@ -2823,25 +2830,25 @@ merge_lists(KVList1, KVList2, LevelInfo, SSTOpts, press_method(), boolean(), non_neg_integer()|not_counted, - build_timings()) -> + build_timings()) -> {list(expanded_pointer()), list(expanded_pointer()), list(binary_slot()), leveled_codec:ledger_key()|null, non_neg_integer()|not_counted}. -merge_lists(KVL1, KVL2, LI, SlotList, FirstKey, MaxSlots, MaxSlots, +merge_lists(KVL1, KVL2, LI, SlotList, FirstKey, MaxSlots, MaxSlots, _PressMethod, _IdxModDate, CountOfTombs, T0) -> - % This SST file is full, move to complete file, and return the + % This SST file is full, move to complete file, and return the % remainder log_buildtimings(T0, LI), {KVL1, KVL2, lists:reverse(SlotList), FirstKey, CountOfTombs}; merge_lists([], [], LI, SlotList, FirstKey, _SlotCount, _MaxSlots, _PressMethod, _IdxModDate, CountOfTombs, T0) -> - % the source files are empty, complete the file + % the source files are empty, complete the file log_buildtimings(T0, LI), {[], [], lists:reverse(SlotList), FirstKey, CountOfTombs}; merge_lists(KVL1, KVL2, LI, SlotList, FirstKey, SlotCount, MaxSlots, PressMethod, IdxModDate, CountOfTombs, T0) -> - % Form a slot by merging the two lists until the next 128 K/V pairs have + % Form a slot by merging the two lists until the next 128 K/V pairs have % been determined SW = os:timestamp(), {KVRem1, KVRem2, Slot, FK0} = @@ -2864,7 +2871,7 @@ merge_lists(KVL1, KVL2, LI, SlotList, FirstKey, SlotCount, MaxSlots, {Lookup, KVL} -> % Convert the list of KVs for the slot into a binary, and related % metadata - {SlotD, T2} = + {SlotD, T2} = generate_binary_slot(Lookup, KVL, PressMethod, IdxModDate, T1), merge_lists(KVRem1, KVRem2, @@ -2880,16 +2887,16 @@ merge_lists(KVL1, KVL2, LI, SlotList, FirstKey, SlotCount, MaxSlots, end. --spec count_tombs(list(leveled_codec:ledger_kv()), - non_neg_integer()|not_counted) -> - non_neg_integer()|not_counted. +-spec count_tombs( + list(leveled_codec:ledger_kv()), non_neg_integer()|not_counted) -> + non_neg_integer()|not_counted. %% @doc %% Count the tombstones in a list of KVs count_tombs(_KVL, not_counted) -> not_counted; count_tombs(KVL, InitCount) -> - FoldFun = - fun(KV, Count) -> + FoldFun = + fun(KV, Count) -> case leveled_codec:strip_to_statusonly(KV) of tomb -> Count + 1; @@ -2910,7 +2917,7 @@ count_tombs(KVL, InitCount) -> {lookup|no_lookup, list(leveled_codec:ledger_kv())}, leveled_codec:ledger_key()}. %% @doc -%% Merge together Key Value lists to provide an ordered slot of KVs +%% Merge together Key Value lists to provide an ordered slot of KVs form_slot([], [], _LI, Type, _Size, Slot, FK) -> {[], [], {Type, lists:reverse(Slot)}, FK}; form_slot(KVList1, KVList2, _LI, lookup, ?LOOK_SLOTSIZE, Slot, FK) -> @@ -2965,7 +2972,7 @@ form_slot(KVList1, KVList2, {IsBasement, TS}, no_lookup, Size, Slot, FK) -> [{TopK, TopV}|Slot], FK0) end - end; + end; {skipped_key, Rem1, Rem2} -> form_slot(Rem1, Rem2, {IsBasement, TS}, no_lookup, Size, Slot, FK) end. @@ -3030,26 +3037,26 @@ maybe_expand_pointer([{next, ManEntry, StartKey}|Tail]) -> ?MERGE_SCANWIDTH); maybe_expand_pointer(List) -> List. - + %%%============================================================================ %%% Timing Functions %%%============================================================================ --spec update_buildtimings(erlang:timestamp(), build_timings(), atom()) - -> build_timings(). +-spec update_buildtimings( + erlang:timestamp(), build_timings(), atom()) -> build_timings(). %% @doc %% -%% Timings taken from the build of a SST file. +%% Timings taken from the build of a SST file. %% -%% There is no sample window, but the no_timing status is still used for +%% There is no sample window, but the no_timing status is still used for %% level zero files where we're not breaking down the build time in this way. update_buildtimings(_SW, no_timing, _Stage) -> no_timing; update_buildtimings(SW, Timings, Stage) -> Timer = timer:now_diff(os:timestamp(), SW), - case Stage of + case Stage of slot_hashlist -> HLT = Timings#build_timings.slot_hashlist + Timer, Timings#build_timings{slot_hashlist = HLT}; @@ -3070,7 +3077,7 @@ update_buildtimings(SW, Timings, Stage) -> %% Log out the time spent during the merge lists part of the SST build log_buildtimings(Timings, LI) -> leveled_log:log( - sst13, + sst13, [Timings#build_timings.fold_toslot, Timings#build_timings.slot_hashlist, Timings#build_timings.slot_serialise, @@ -3151,10 +3158,10 @@ generate_indexkeys(Count, IndexList) -> generate_indexkey(Term, Count) -> IndexSpecs = [{add, "t1_int", Term}], - leveled_codec:idx_indexspecs(IndexSpecs, - "Bucket", - "Key" ++ integer_to_list(Count), - Count, + leveled_codec:idx_indexspecs(IndexSpecs, + "Bucket", + "Key" ++ integer_to_list(Count), + Count, infinity). tombcount_test() -> @@ -3178,7 +3185,7 @@ tombcount_tester(Level) -> end, KVL1 = lists:map(FlipToTombFun, KL1), KVL2 = lists:map(FlipToTombFun, KL2), - CountTombFun = + CountTombFun = fun({_K, V}, Acc) -> case element(2, V) of tomb -> @@ -3190,20 +3197,20 @@ tombcount_tester(Level) -> ExpectedCount = lists:foldl(CountTombFun, 0, KVL1 ++ KVL2), {RP, Filename} = {?TEST_AREA, "tombcount_test"}, - OptsSST = + OptsSST = #sst_options{press_method=native, log_options=leveled_log:get_opts()}, - {ok, SST1, KD, BB} = sst_newmerge(RP, Filename, - KVL1, KVL2, false, Level, + {ok, SST1, _KD, _BB} = sst_newmerge(RP, Filename, + KVL1, KVL2, false, Level, N, OptsSST, false, false), ?assertMatch(not_counted, sst_gettombcount(SST1)), ok = sst_close(SST1), ok = file:delete(filename:join(RP, Filename ++ ".sst")), - {ok, SST2, KD, BB} = sst_newmerge(RP, Filename, - KVL1, KVL2, false, Level, + {ok, SST2, _KD1, _BB1} = sst_newmerge(RP, Filename, + KVL1, KVL2, false, Level, N, OptsSST, false, true), - + ?assertMatch(ExpectedCount, sst_gettombcount(SST2)), ok = sst_close(SST2), ok = file:delete(filename:join(RP, Filename ++ ".sst")). @@ -3320,7 +3327,7 @@ indexed_list_mixedkeys2_test() -> KVL1). indexed_list_allindexkeys_test() -> - Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(150)), + Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(150)), ?LOOK_SLOTSIZE), {{HeaderT, FullBinT, HL, LK}, no_timing} = generate_binary_slot(lookup, Keys, native, true, no_timing), @@ -3328,9 +3335,9 @@ indexed_list_allindexkeys_test() -> generate_binary_slot(lookup, Keys, native, false, no_timing), EmptySlotSize = ?LOOK_SLOTSIZE - 1, LMD = ?FLIPPER32, - ?assertMatch(<<_BL:20/binary, LMD:32/integer, EmptySlotSize:8/integer>>, + ?assertMatch(<<_BL:20/binary, LMD:32/integer, EmptySlotSize:8/integer>>, HeaderT), - ?assertMatch(<<_BL:20/binary, EmptySlotSize:8/integer>>, + ?assertMatch(<<_BL:20/binary, EmptySlotSize:8/integer>>, HeaderF), % SW = os:timestamp(), BinToListT = binaryslot_tolist(FullBinT, native, true), @@ -3340,21 +3347,21 @@ indexed_list_allindexkeys_test() -> % [timer:now_diff(os:timestamp(), SW)]), ?assertMatch(Keys, BinToListT), ?assertMatch({Keys, none}, binaryslot_trimmedlist(FullBinT, - all, all, - native, + all, all, + native, true, false)), ?assertMatch(Keys, BinToListF), ?assertMatch({Keys, none}, binaryslot_trimmedlist(FullBinF, - all, all, - native, + all, all, + native, false, false)). indexed_list_allindexkeys_nolookup_test() -> Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(1000)), ?NOLOOK_SLOTSIZE), - {{Header, FullBin, _HL, _LK}, no_timing} = + {{Header, FullBin, _HL, _LK}, no_timing} = generate_binary_slot(no_lookup, Keys, native, ?INDEX_MODDATE,no_timing), ?assertMatch(<<_BL:20/binary, _LMD:32/integer, 127:8/integer>>, Header), % SW = os:timestamp(), @@ -3363,37 +3370,37 @@ indexed_list_allindexkeys_nolookup_test() -> % "Indexed list flattened in ~w microseconds ~n", % [timer:now_diff(os:timestamp(), SW)]), ?assertMatch(Keys, BinToList), - ?assertMatch({Keys, none}, binaryslot_trimmedlist(FullBin, - all, all, + ?assertMatch({Keys, none}, binaryslot_trimmedlist(FullBin, + all, all, native, ?INDEX_MODDATE, false)). indexed_list_allindexkeys_trimmed_test() -> - Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(150)), + Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(150)), ?LOOK_SLOTSIZE), - {{Header, FullBin, _HL, _LK}, no_timing} = + {{Header, FullBin, _HL, _LK}, no_timing} = generate_binary_slot(lookup, Keys, native, ?INDEX_MODDATE,no_timing), EmptySlotSize = ?LOOK_SLOTSIZE - 1, ?assertMatch(<<_BL:20/binary, _LMD:32/integer, EmptySlotSize:8/integer>>, Header), ?assertMatch({Keys, none}, binaryslot_trimmedlist(FullBin, - {i, - "Bucket", + {i, + "Bucket", {"t1_int", 0}, - null}, - {i, - "Bucket", + null}, + {i, + "Bucket", {"t1_int", 99999}, null}, native, ?INDEX_MODDATE, false)), - + {SK1, _} = lists:nth(10, Keys), {EK1, _} = lists:nth(100, Keys), R1 = lists:sublist(Keys, 10, 91), - {O1, none} = binaryslot_trimmedlist(FullBin, SK1, EK1, + {O1, none} = binaryslot_trimmedlist(FullBin, SK1, EK1, native, ?INDEX_MODDATE, false), ?assertMatch(91, length(O1)), ?assertMatch(R1, O1), @@ -3422,21 +3429,21 @@ indexed_list_mixedkeys_bitflip_test() -> KVL0 = lists:ukeysort(1, generate_randomkeys(1, 50, 1, 4)), KVL1 = lists:sublist(KVL0, 33), Keys = lists:ukeysort(1, generate_indexkeys(60) ++ KVL1), - {{Header, SlotBin, _HL, LK}, no_timing} = + {{Header, SlotBin, _HL, LK}, no_timing} = generate_binary_slot(lookup, Keys, native, ?INDEX_MODDATE, no_timing), - + ?assertMatch(LK, element(1, lists:last(Keys))), - <> = Header, - - TestKey1 = element(1, lists:nth(1, KVL1)), - TestKey2 = element(1, lists:nth(33, KVL1)), + + TestKey1 = element(1, lists:nth(1, KVL1)), + TestKey2 = element(1, lists:nth(33, KVL1)), MH1 = leveled_codec:segment_hash(TestKey1), MH2 = leveled_codec:segment_hash(TestKey2), @@ -3478,7 +3485,7 @@ indexed_list_mixedkeys_bitflip_test() -> {O1, none} = binaryslot_trimmedlist(SlotBin3, SK1, EK1, native, ?INDEX_MODDATE, false), ?assertMatch([], O1), - + SlotBin4 = flip_byte(SlotBin, 0, 20), SlotBin5 = flip_byte(SlotBin, 20, byte_size(Header) - 20 - 12), @@ -3499,7 +3506,7 @@ indexed_list_mixedkeys_bitflip_test() -> flip_byte(Binary, Offset, Length) -> Byte1 = leveled_rand:uniform(Length) + Offset - 1, <> = Binary, - case A of + case A of 0 -> <>; _ -> @@ -3543,11 +3550,11 @@ size_tester(KVL1, KVL2, N) -> io:format(user, "~nStarting ... test with ~w keys ~n", [N]), {RP, Filename} = {?TEST_AREA, "doublesize_test"}, - OptsSST = + OptsSST = #sst_options{press_method=native, log_options=leveled_log:get_opts()}, - {ok, SST1, _KD, _BB} = sst_newmerge(RP, Filename, - KVL1, KVL2, false, ?DOUBLESIZE_LEVEL, + {ok, SST1, _KD, _BB} = sst_newmerge(RP, Filename, + KVL1, KVL2, false, ?DOUBLESIZE_LEVEL, N, OptsSST, false, false), ok = sst_close(SST1), {ok, SST2, _SKEK, Bloom} = @@ -3588,9 +3595,9 @@ merge_tester(NewFunS, NewFunM) -> KVL2 = lists:ukeysort(1, generate_randomkeys(1, N, 1, 20)), KVL3 = lists:ukeymerge(1, KVL1, KVL2), SW0 = os:timestamp(), - {ok, P1, {FK1, LK1}, _Bloom1} = + {ok, P1, {FK1, LK1}, _Bloom1} = NewFunS(?TEST_AREA, "level1_src", 1, KVL1, 6000, native), - {ok, P2, {FK2, LK2}, _Bloom2} = + {ok, P2, {FK2, LK2}, _Bloom2} = NewFunS(?TEST_AREA, "level2_src", 2, KVL2, 3000, native), ExpFK1 = element(1, lists:nth(1, KVL1)), ExpLK1 = element(1, lists:last(KVL1)), @@ -3602,7 +3609,7 @@ merge_tester(NewFunS, NewFunM) -> ?assertMatch(ExpLK2, LK2), ML1 = [{next, #manifest_entry{owner = P1}, FK1}], ML2 = [{next, #manifest_entry{owner = P2}, FK2}], - NewR = + NewR = NewFunM(?TEST_AREA, "level2_merge", ML1, ML2, false, 2, N * 2, native), {ok, P3, {{Rem1, Rem2}, FK3, LK3}, _Bloom3} = NewR, ?assertMatch([], Rem1), @@ -3613,7 +3620,7 @@ merge_tester(NewFunS, NewFunM) -> io:format(user, "Created and merged two files of size ~w in ~w microseconds~n", [N, timer:now_diff(os:timestamp(), SW0)]), - + SW1 = os:timestamp(), lists:foreach(fun({K, V}) -> ?assertMatch({K, V}, sst_get(P3, K)) @@ -3622,14 +3629,14 @@ merge_tester(NewFunS, NewFunM) -> io:format(user, "Checked presence of all ~w objects in ~w microseconds~n", [length(KVL3), timer:now_diff(os:timestamp(), SW1)]), - + ok = sst_close(P1), ok = sst_close(P2), ok = sst_close(P3), ok = file:delete(?TEST_AREA ++ "/level1_src.sst"), ok = file:delete(?TEST_AREA ++ "/level2_src.sst"), ok = file:delete(?TEST_AREA ++ "/level2_merge.sst"). - + simple_persisted_range_test() -> simple_persisted_range_tester(fun testsst_new/6). @@ -3642,35 +3649,35 @@ simple_persisted_range_tester(SSTNewFun) -> {LastKey, _LV} = lists:last(KVList1), {ok, Pid, {FirstKey, LastKey}, _Bloom} = SSTNewFun(RP, Filename, 1, KVList1, length(KVList1), native), - + {o, B, K, null} = LastKey, SK1 = {o, B, K, 0}, EK1 = {o, B, K, 1}, FetchListA1 = sst_getkvrange(Pid, SK1, EK1, 1), ?assertMatch([], FetchListA1), - + SK2 = element(1, lists:nth(127, KVList1)), SK3 = element(1, lists:nth(128, KVList1)), SK4 = element(1, lists:nth(129, KVList1)), SK5 = element(1, lists:nth(130, KVList1)), - + EK2 = element(1, lists:nth(255, KVList1)), EK3 = element(1, lists:nth(256, KVList1)), EK4 = element(1, lists:nth(257, KVList1)), EK5 = element(1, lists:nth(258, KVList1)), - + TestFun = fun({SK, EK}) -> FetchList = sst_getkvrange(Pid, SK, EK, 4), ?assertMatch(SK, element(1, lists:nth(1, FetchList))), ?assertMatch(EK, element(1, lists:last(FetchList))) end, - + TL2 = lists:map(fun(EK) -> {SK2, EK} end, [EK2, EK3, EK4, EK5]), TL3 = lists:map(fun(EK) -> {SK3, EK} end, [EK2, EK3, EK4, EK5]), TL4 = lists:map(fun(EK) -> {SK4, EK} end, [EK2, EK3, EK4, EK5]), TL5 = lists:map(fun(EK) -> {SK5, EK} end, [EK2, EK3, EK4, EK5]), - lists:foreach(TestFun, TL2 ++ TL3 ++ TL4 ++ TL5). + lists:foreach(TestFun, TL2 ++ TL3 ++ TL4 ++ TL5). simple_persisted_rangesegfilter_test() -> @@ -3684,29 +3691,29 @@ simple_persisted_rangesegfilter_tester(SSTNewFun) -> {LastKey, _LV} = lists:last(KVList1), {ok, Pid, {FirstKey, LastKey}, _Bloom} = SSTNewFun(RP, Filename, 1, KVList1, length(KVList1), native), - + SK1 = element(1, lists:nth(124, KVList1)), SK2 = element(1, lists:nth(126, KVList1)), SK3 = element(1, lists:nth(128, KVList1)), SK4 = element(1, lists:nth(130, KVList1)), SK5 = element(1, lists:nth(132, KVList1)), - + EK1 = element(1, lists:nth(252, KVList1)), EK2 = element(1, lists:nth(254, KVList1)), EK3 = element(1, lists:nth(256, KVList1)), EK4 = element(1, lists:nth(258, KVList1)), EK5 = element(1, lists:nth(260, KVList1)), - - GetSegFun = + + GetSegFun = fun(LK) -> extract_hash( leveled_codec:strip_to_segmentonly( lists:keyfind(LK, 1, KVList1))) end, - SegList = + SegList = lists:map(GetSegFun, [SK1, SK2, SK3, SK4, SK5, EK1, EK2, EK3, EK4, EK5]), - + TestFun = fun(StartKey, EndKey, OutList) -> RangeKVL = @@ -3720,34 +3727,34 @@ simple_persisted_rangesegfilter_tester(SSTNewFun) -> end, lists:foreach(CheckOutFun, OutList) end, - - lists:foldl(fun(SK0, Acc) -> + + lists:foldl(fun(SK0, Acc) -> TestFun(SK0, EK1, [EK2, EK3, EK4, EK5] ++ Acc), [SK0|Acc] end, [], [SK1, SK2, SK3, SK4, SK5]), - lists:foldl(fun(SK0, Acc) -> + lists:foldl(fun(SK0, Acc) -> TestFun(SK0, EK2, [EK3, EK4, EK5] ++ Acc), [SK0|Acc] end, [], [SK1, SK2, SK3, SK4, SK5]), - lists:foldl(fun(SK0, Acc) -> + lists:foldl(fun(SK0, Acc) -> TestFun(SK0, EK3, [EK4, EK5] ++ Acc), [SK0|Acc] end, [], [SK1, SK2, SK3, SK4, SK5]), - lists:foldl(fun(SK0, Acc) -> + lists:foldl(fun(SK0, Acc) -> TestFun(SK0, EK4, [EK5] ++ Acc), [SK0|Acc] end, [], [SK1, SK2, SK3, SK4, SK5]), - + ok = sst_clear(Pid). - + additional_range_test() -> @@ -3768,17 +3775,17 @@ additional_range_test() -> [], lists:seq(?NOLOOK_SLOTSIZE + Gap + 1, 2 * ?NOLOOK_SLOTSIZE + Gap)), - {ok, P1, {{Rem1, Rem2}, SK, EK}, _Bloom1} = + {ok, P1, {{Rem1, Rem2}, SK, EK}, _Bloom1} = testsst_new(?TEST_AREA, "range1_src", IK1, IK2, false, 1, 9999, native), ?assertMatch([], Rem1), ?assertMatch([], Rem2), ?assertMatch(SK, element(1, lists:nth(1, IK1))), ?assertMatch(EK, element(1, lists:last(IK2))), - + % Basic test - checking scanwidth R1 = sst_getkvrange(P1, SK, EK, 1), ?assertMatch(?NOLOOK_SLOTSIZE + 1, length(R1)), - QR1 = lists:sublist(R1, ?NOLOOK_SLOTSIZE), + QR1 = lists:sublist(R1, ?NOLOOK_SLOTSIZE), ?assertMatch(IK1, QR1), R2 = sst_getkvrange(P1, SK, EK, 2), ?assertMatch(?NOLOOK_SLOTSIZE * 2, length(R2)), @@ -3786,14 +3793,14 @@ additional_range_test() -> QR3 = lists:sublist(R2, ?NOLOOK_SLOTSIZE + 1, 2 * ?NOLOOK_SLOTSIZE), ?assertMatch(IK1, QR2), ?assertMatch(IK2, QR3), - + % Testing the gap [GapSKV] = generate_indexkey(?NOLOOK_SLOTSIZE + 1, ?NOLOOK_SLOTSIZE + 1), [GapEKV] = generate_indexkey(?NOLOOK_SLOTSIZE + 2, ?NOLOOK_SLOTSIZE + 2), io:format("Gap test between ~p and ~p", [GapSKV, GapEKV]), R3 = sst_getkvrange(P1, element(1, GapSKV), element(1, GapEKV), 1), ?assertMatch([], R3), - + % Testing beyond the range [PastEKV] = generate_indexkey(2 * ?NOLOOK_SLOTSIZE + Gap + 1, 2 * ?NOLOOK_SLOTSIZE + Gap + 1), @@ -3808,24 +3815,27 @@ additional_range_test() -> R6 = sst_getkvrange(P1, element(1, MidREKV), element(1, PastEKV), 2), Exp6 = lists:sublist(IK2, 2, length(IK2)), ?assertMatch(Exp6, R6), - + % Testing at a slot end Slot1EK = element(1, lists:last(IK1)), R7 = sst_getkvrange(P1, SK, Slot1EK, 2), ?assertMatch(IK1, R7). - + % Testing beyond end (should never happen if manifest behaves) % Test blows up anyway % R8 = sst_getkvrange(P1, element(1, PastEKV), element(1, PastEKV), 2), % ?assertMatch([], R8). -simple_switchcache_test() -> +simple_switchcache_test_() -> + {timeout, 60, fun simple_switchcache_tester/0}. + +simple_switchcache_tester() -> {RP, Filename} = {?TEST_AREA, "simple_switchcache_test"}, KVList0 = generate_randomkeys(1, ?LOOK_SLOTSIZE * 2, 1, 20), KVList1 = lists:sublist(lists:ukeysort(1, KVList0), ?LOOK_SLOTSIZE), [{FirstKey, _FV}|_Rest] = KVList1, {LastKey, _LV} = lists:last(KVList1), - {ok, OpenP4, {FirstKey, LastKey}, Bloom} = + {ok, OpenP4, {FirstKey, LastKey}, _Bloom1} = testsst_new(RP, Filename, 4, KVList1, length(KVList1), native), lists:foreach(fun({K, V}) -> ?assertMatch({K, V}, sst_get(OpenP4, K)) @@ -3840,7 +3850,7 @@ simple_switchcache_test() -> ?assertMatch({K, V}, sst_get(OpenP4, K)) end, KVList1), - gen_fsm:send_event(OpenP4, timeout), + timer:sleep(?HIBERNATE_TIMEOUT + 10), lists:foreach(fun({K, V}) -> ?assertMatch({K, V}, sst_get(OpenP4, K)) end, @@ -3848,27 +3858,17 @@ simple_switchcache_test() -> ok = sst_close(OpenP4), OptsSST = #sst_options{press_method=native, log_options=leveled_log:get_opts()}, - {ok, OpenP5, {FirstKey, LastKey}, Bloom} = + {ok, OpenP5, {FirstKey, LastKey}, _Bloom2} = sst_open(RP, Filename ++ ".sst", OptsSST, 5), lists:foreach(fun({K, V}) -> ?assertMatch({K, V}, sst_get(OpenP5, K)) end, KVList1), - lists:foreach(fun({K, V}) -> - ?assertMatch({K, V}, sst_get(OpenP5, K)) - end, - KVList1), - gen_fsm:send_event(OpenP5, timeout), lists:foreach(fun({K, V}) -> ?assertMatch({K, V}, sst_get(OpenP5, K)) end, KVList1), ok = sst_switchlevels(OpenP5, 6), - lists:foreach(fun({K, V}) -> - ?assertMatch({K, V}, sst_get(OpenP5, K)) - end, - KVList1), - gen_fsm:send_event(OpenP5, timeout), lists:foreach(fun({K, V}) -> ?assertMatch({K, V}, sst_get(OpenP5, K)) end, @@ -3878,7 +3878,7 @@ simple_switchcache_test() -> ?assertMatch({K, V}, sst_get(OpenP5, K)) end, KVList1), - gen_fsm:send_event(OpenP5, timeout), + timer:sleep(?HIBERNATE_TIMEOUT + 10), lists:foreach(fun({K, V}) -> ?assertMatch({K, V}, sst_get(OpenP5, K)) end, @@ -3915,7 +3915,7 @@ reader_hibernate_tester() -> KVList1 = lists:ukeysort(1, KVList0), [{FirstKey, FV}|_Rest] = KVList1, {LastKey, _LV} = lists:last(KVList1), - {ok, Pid, {FirstKey, LastKey}, _Bloom} = + {ok, Pid, {FirstKey, LastKey}, _Bloom} = testsst_new(RP, Filename, 1, KVList1, length(KVList1), native), ?assertMatch({FirstKey, FV}, sst_get(Pid, FirstKey)), SQN = leveled_codec:strip_to_seqonly({FirstKey, FV}), @@ -3935,7 +3935,7 @@ delete_pending_tester() -> KVList1 = lists:ukeysort(1, KVList0), [{FirstKey, _FV}|_Rest] = KVList1, {LastKey, _LV} = lists:last(KVList1), - {ok, Pid, {FirstKey, LastKey}, _Bloom} = + {ok, Pid, {FirstKey, LastKey}, _Bloom} = testsst_new(RP, Filename, 1, KVList1, length(KVList1), native), timer:sleep(2000), leveled_sst:sst_setfordelete(Pid, false), @@ -3948,19 +3948,18 @@ fetch_status_test() -> KVList1 = lists:ukeysort(1, KVList0), [{FirstKey, _FV}|_Rest] = KVList1, {LastKey, _LV} = lists:last(KVList1), - {ok, Pid, {FirstKey, LastKey}, _Bloom} = + {ok, Pid, {FirstKey, LastKey}, _Bloom} = testsst_new(RP, Filename, 1, KVList1, length(KVList1), native), - {status, Pid, {module, gen_fsm}, SItemL} = sys:get_status(Pid), + {status, Pid, {module, gen_statem}, SItemL} = sys:get_status(Pid), S = lists:keyfind(state, 1, lists:nth(5, SItemL)), true = is_integer(array:size(S#state.fetch_cache)), true = is_integer(array:size(element(2, S#state.blockindex_cache))), - ST = format_status(terminate, [dict:new(), S]), + ST = format_status(terminate, [dict:new(), starting, S]), ?assertMatch(redacted, ST#state.blockindex_cache), ?assertMatch(redacted, ST#state.fetch_cache), ok = sst_close(Pid), ok = file:delete(filename:join(RP, Filename ++ ".sst")). - simple_persisted_test_() -> {timeout, 60, fun simple_persisted_test_bothformats/0}. @@ -3976,7 +3975,7 @@ simple_persisted_tester(SSTNewFun) -> {LastKey, _LV} = lists:last(KVList1), {ok, Pid, {FirstKey, LastKey}, Bloom} = SSTNewFun(RP, Filename, Level, KVList1, length(KVList1), native), - + B0 = check_binary_references(Pid), SW0 = os:timestamp(), @@ -4029,7 +4028,7 @@ simple_persisted_tester(SSTNewFun) -> end end, FetchedList1 = lists:foldl(FoldFun, [], FetchList1), ?assertMatch(KVList1, FetchedList1), - + {TenthKey, _v10} = lists:nth(10, KVList1), {Three000Key, _v300} = lists:nth(300, KVList1), SubKVList1 = lists:sublist(KVList1, 10, 291), @@ -4039,7 +4038,7 @@ simple_persisted_tester(SSTNewFun) -> FetchedList2 = lists:foldl(FoldFun, [], FetchList2), ?assertMatch(SubKVList1L, length(FetchedList2)), ?assertMatch(SubKVList1, FetchedList2), - + {Eight000Key, V800} = lists:nth(800, KVList1), SubKVListA1 = lists:sublist(KVList1, 10, 791), SubKVListA1L = length(SubKVListA1), @@ -4048,13 +4047,13 @@ simple_persisted_tester(SSTNewFun) -> FetchedListA2 = lists:foldl(FoldFun, [], FetchListA2), ?assertMatch(SubKVListA1L, length(FetchedListA2)), ?assertMatch(SubKVListA1, FetchedListA2), - + FetchListB2 = sst_getkvrange(Pid, TenthKey, Eight000Key, 4), ?assertMatch(pointer, element(1, lists:last(FetchListB2))), FetchedListB2 = lists:foldl(FoldFun, [], FetchListB2), ?assertMatch(SubKVListA1L, length(FetchedListB2)), ?assertMatch(SubKVListA1, FetchedListB2), - + FetchListB3 = sst_getkvrange(Pid, Eight000Key, {o, null, null, null}, @@ -4065,7 +4064,7 @@ simple_persisted_tester(SSTNewFun) -> io:format("Length expected ~w~n", [SubKVListA3L]), ?assertMatch(SubKVListA3L, length(FetchedListB3)), ?assertMatch(SubKVListA3, FetchedListB3), - + io:format("Eight hundredth key ~w~n", [Eight000Key]), FetchListB4 = sst_getkvrange(Pid, Eight000Key, @@ -4073,7 +4072,7 @@ simple_persisted_tester(SSTNewFun) -> 4), FetchedListB4 = lists:foldl(FoldFun, [], FetchListB4), ?assertMatch([{Eight000Key, V800}], FetchedListB4), - + B1 = check_binary_references(Pid), ok = sst_close(Pid), @@ -4096,9 +4095,9 @@ simple_persisted_tester(SSTNewFun) -> B3 = check_binary_references(OpenP), ?assertMatch(0, B2), % Opens with an empty cache ?assertMatch(true, B3 > B2), % Now has headers in cache - ?assertMatch(false, B3 > B0 * 2), + ?assertMatch(false, B3 > B0 * 2), % Not significantly bigger than when created new - ?assertMatch(false, B3 > B1 * 2), + ?assertMatch(false, B3 > B1 * 2), % Not significantly bigger than when created new ok = sst_close(OpenP), @@ -4107,7 +4106,7 @@ simple_persisted_tester(SSTNewFun) -> check_binary_references(Pid) -> garbage_collect(Pid), {binary, BinList} = process_info(Pid, binary), - TotalBinMem = + TotalBinMem = lists:foldl(fun({_R, BM, _RC}, Acc) -> Acc + BM end, 0, BinList), io:format(user, "Total binary memory ~w~n", [TotalBinMem]), TotalBinMem. @@ -4167,37 +4166,33 @@ nonsense_coverage_test() -> ?assertMatch( {ok, reader, #state{}}, code_change(nonsense, reader, #state{}, nonsense)), - ?assertMatch( - {reply, undefined, reader, #state{}}, - handle_sync_event("hello", self(), reader, #state{})), - SampleBin = <<0:128/integer>>, FlippedBin = flip_byte(SampleBin, 0, 16), ?assertMatch(false, FlippedBin == SampleBin). hashmatching_bytreesize_test() -> B = <<"Bucket">>, - V = leveled_head:riak_metadata_to_binary(term_to_binary([{"actor1", 1}]), - <<1:32/integer, + V = leveled_head:riak_metadata_to_binary(term_to_binary([{"actor1", 1}]), + <<1:32/integer, 0:32/integer, 0:32/integer>>), GenKeyFun = fun(X) -> - LK = - {?RIAK_TAG, - B, - list_to_binary("Key" ++ integer_to_list(X)), + LK = + {?RIAK_TAG, + B, + list_to_binary("Key" ++ integer_to_list(X)), null}, - LKV = leveled_codec:generate_ledgerkv(LK, - X, - V, - byte_size(V), + LKV = leveled_codec:generate_ledgerkv(LK, + X, + V, + byte_size(V), {active, infinity}), {_Bucket, _Key, MetaValue, _Hashes, _LastMods} = LKV, {LK, MetaValue} end, KVL = lists:map(GenKeyFun, lists:seq(1, 128)), - {{PosBinIndex1, _FullBin, _HL, _LK}, no_timing} = + {{PosBinIndex1, _FullBin, _HL, _LK}, no_timing} = generate_binary_slot(lookup, KVL, native, ?INDEX_MODDATE, no_timing), check_segment_match(PosBinIndex1, KVL, small), check_segment_match(PosBinIndex1, KVL, medium). @@ -4206,7 +4201,7 @@ hashmatching_bytreesize_test() -> check_segment_match(PosBinIndex1, KVL, TreeSize) -> CheckFun = fun({{_T, B, K, null}, _V}) -> - Seg = + Seg = leveled_tictac:get_segment( leveled_tictac:keyto_segment32(<>), TreeSize), @@ -4216,15 +4211,9 @@ check_segment_match(PosBinIndex1, KVL, TreeSize) -> end, lists:foreach(CheckFun, KVL). -take_max_lastmoddate_test() -> - % TODO: Remove this test - % Temporarily added to make dialyzer happy (until we've made use of last - % modified dates - ?assertMatch(1, take_max_lastmoddate(0, 1)). - stopstart_test() -> - {ok, Pid} = gen_fsm:start_link(?MODULE, [], []), - % check we can close in the starting state. This may happen due to the + {ok, Pid} = gen_statem:start_link(?MODULE, [], ?START_OPTS), + % check we can close in the starting state. This may happen due to the % fetcher on new level zero files working in a loop ok = sst_close(Pid). @@ -4238,7 +4227,7 @@ stop_whenstarter_stopped_testto() -> fun(X, Acc) -> case Acc of false -> false; - true -> + true -> timer:sleep(X), is_process_alive(RP) end @@ -4267,7 +4256,7 @@ corrupted_block_rangetester(PressMethod, TestCount) -> MidBlock = serialise_block(lists:sublist(KVL1, 41, 20), PressMethod), B4 = serialise_block(lists:sublist(KVL1, 61, 20), PressMethod), B5 = serialise_block(lists:sublist(KVL1, 81, 20), PressMethod), - CorruptBlockFun = + CorruptBlockFun = fun(Block) -> case leveled_rand:uniform(10) < 2 of true -> @@ -4276,8 +4265,8 @@ corrupted_block_rangetester(PressMethod, TestCount) -> Block end end, - - CheckFun = + + CheckFun = fun({SK, EK}) -> InputBlocks = lists:map(CorruptBlockFun, [B1, B2, MidBlock, B4, B5]), @@ -4306,28 +4295,28 @@ corrupted_block_fetch_tester(PressMethod) -> KC = 120, KVL1 = lists:ukeysort(1, generate_randomkeys(1, KC, 1, 2)), - {{Header, SlotBin, _HashL, _LastKey}, _BT} = + {{Header, SlotBin, _HashL, _LastKey}, _BT} = generate_binary_slot(lookup, KVL1, PressMethod, false, no_timing), - <> = Header, HS = byte_size(Header), - + <> = SlotBin, - + CorruptB3 = flip_byte(B3, 0 , B3L), CorruptSlotBin = <>, - - CheckFun = + + CheckFun = fun(N, {AccHit, AccMiss}) -> PosL = [min(0, leveled_rand:uniform(N - 2)), N - 1], {LK, LV} = lists:nth(N, KVL1), @@ -4362,7 +4351,7 @@ block_index_cache_test() -> {I, <<0:160/integer, TS:32/integer, 0:32/integer>>} end, lists:seq(1, 8)), - EntriesNoTS = + EntriesNoTS = lists:map(fun(I) -> {I, <<0:160/integer, 0:32/integer>>} end, @@ -4385,7 +4374,7 @@ block_index_cache_test() -> update_blockindex_cache(true, ETSP2, BIC3, undefined, true), {_, BIC4, LMD4} = update_blockindex_cache(true, ETSP2, BIC4, LMD4, true), - + ?assertMatch(none, array:get(0, element(2, BIC0))), ?assertMatch(none, array:get(0, element(2, BIC1))), ?assertMatch(HeaderNoTS, array:get(0, element(2, BIC2))), @@ -4412,10 +4401,10 @@ key_matchesprefix_test() -> {<<"dob_bin">>, <<"19601301">>}, list_to_binary(io_lib:format("~6..0w", [0]))}, {1, {active, infinity}, no_lookup, null}}, - OptsSST = + OptsSST = #sst_options{press_method=native, log_options=leveled_log:get_opts()}, - {ok, P1, {_FK1, _LK1}, _Bloom1} = + {ok, P1, {_FK1, _LK1}, _Bloom1} = sst_new( ?TEST_AREA, FileName, 1, [OddIdxKey|IndexEntries], 6000, OptsSST), IdxRange2 = @@ -4484,9 +4473,9 @@ key_matchesprefix_test() -> <<"19601301">>, null}, {1, {active, infinity}, {100, 100}, null}}, - OptsSST = + OptsSST = #sst_options{press_method=native, log_options=leveled_log:get_opts()}, - {ok, P2, {_FK2, _LK2}, _Bloom2} = + {ok, P2, {_FK2, _LK2}, _Bloom2} = sst_new( ?TEST_AREA, FileName, 1, [OddObjKey|ObjectEntries], 6000, OptsSST), ObjRange2 = @@ -4529,7 +4518,7 @@ key_matchesprefix_test() -> ?assertMatch(500, length(ObjRangeY)), ok = sst_close(P2), ok = file:delete(filename:join(?TEST_AREA, FileName ++ ".sst")). - + range_key_indextermmatch_test() -> FileName = "indextermmatch_test", @@ -4543,12 +4532,12 @@ range_key_indextermmatch_test() -> {1, {active, infinity}, no_lookup, null}} end, IndexEntries = lists:map(IndexKeyFun, lists:seq(1, 500)), - OptsSST = + OptsSST = #sst_options{press_method=native, log_options=leveled_log:get_opts()}, - {ok, P1, {_FK1, _LK1}, _Bloom1} = + {ok, P1, {_FK1, _LK1}, _Bloom1} = sst_new(?TEST_AREA, FileName, 1, IndexEntries, 6000, OptsSST), - + IdxRange1 = sst_getkvrange( P1, @@ -4632,7 +4621,7 @@ range_key_indextermmatch_test() -> ?assertMatch(500, length(IdxRange9)), ok = sst_close(P1), ok = file:delete(filename:join(?TEST_AREA, FileName ++ ".sst")). - + range_key_lestthanprefix_test() -> FileName = "lessthanprefix_test", @@ -4647,14 +4636,14 @@ range_key_lestthanprefix_test() -> {1, {active, infinity}, no_lookup, null}} end, IndexEntries = lists:map(IndexKeyFun, lists:seq(1, 500)), - OptsSST = + OptsSST = #sst_options{press_method=native, log_options=leveled_log:get_opts()}, - {ok, P1, {_FK1, _LK1}, _Bloom1} = + {ok, P1, {_FK1, _LK1}, _Bloom1} = sst_new(?TEST_AREA, FileName, 1, IndexEntries, 6000, OptsSST), - + IndexFileStateSize = size_summary(P1), - + IdxRange1 = sst_getkvrange( P1, @@ -4740,12 +4729,12 @@ range_key_lestthanprefix_test() -> {1, {active, infinity}, {0, 0}, null}} end, ObjectEntries = lists:map(ObjectKeyFun, lists:seq(1, 500)), - OptsSST = + OptsSST = #sst_options{press_method=native, log_options=leveled_log:get_opts()}, - {ok, P2, {_FK2, _LK2}, _Bloom2} = + {ok, P2, {_FK2, _LK2}, _Bloom2} = sst_new(?TEST_AREA, FileName, 1, ObjectEntries, 6000, OptsSST), - + ObjectFileStateSize = size_summary(P2), ObjRange1 = @@ -4825,7 +4814,7 @@ range_key_lestthanprefix_test() -> {1, {active, infinity}, {0, 0}, null, undefined}} end, HeadEntries = lists:map(HeadKeyFun, lists:seq(1, 500)), - {ok, P3, {_FK3, _LK3}, _Bloom3} = + {ok, P3, {_FK3, _LK3}, _Bloom3} = sst_new(?TEST_AREA, FileName, 1, HeadEntries, 6000, OptsSST), HeadFileStateSize = size_summary(P3), @@ -4896,12 +4885,12 @@ range_key_lestthanprefix_test() -> ?assertMatch(100, length(HeadRange8)), ok = sst_close(P3), ok = file:delete(filename:join(?TEST_AREA, FileName ++ ".sst")), - + [_HdO|RestObjectEntries] = ObjectEntries, [_HdI|RestIndexEntries] = IndexEntries, [_Hdh|RestHeadEntries] = HeadEntries, - {ok, P4, {_FK4, _LK4}, _Bloom4} = + {ok, P4, {_FK4, _LK4}, _Bloom4} = sst_new( ?TEST_AREA, FileName, 1, @@ -4911,7 +4900,7 @@ range_key_lestthanprefix_test() -> ok = sst_close(P4), ok = file:delete(filename:join(?TEST_AREA, FileName ++ ".sst")), - {ok, P5, {_FK5, _LK5}, _Bloom5} = + {ok, P5, {_FK5, _LK5}, _Bloom5} = sst_new( ?TEST_AREA, FileName, 1, @@ -4921,7 +4910,7 @@ range_key_lestthanprefix_test() -> ok = sst_close(P5), ok = file:delete(filename:join(?TEST_AREA, FileName ++ ".sst")), - {ok, P6, {_FK6, _LK6}, _Bloom6} = + {ok, P6, {_FK6, _LK6}, _Bloom6} = sst_new( ?TEST_AREA, FileName, 1, @@ -4943,7 +4932,7 @@ print_compare_size(Type, OptimisedSize, UnoptimisedSize) -> [Type, OptimisedSize * 8, UnoptimisedSize * 8]), % Reduced by at least a quarter ?assert(OptimisedSize < (UnoptimisedSize - (UnoptimisedSize div 4))). - + single_key_test() -> FileName = "single_key_test", @@ -4952,23 +4941,23 @@ single_key_test() -> Chunk = leveled_rand:rand_bytes(16), {_B, _K, MV, _H, _LMs} = leveled_codec:generate_ledgerkv(LK, 1, Chunk, 16, infinity), - OptsSST = + OptsSST = #sst_options{press_method=native, log_options=leveled_log:get_opts()}, - {ok, P1, {LK, LK}, _Bloom1} = + {ok, P1, {LK, LK}, _Bloom1} = sst_new(?TEST_AREA, FileName, 1, [{LK, MV}], 6000, OptsSST), ?assertMatch({LK, MV}, sst_get(P1, LK)), ok = sst_close(P1), ok = file:delete(filename:join(?TEST_AREA, FileName ++ ".sst")), IndexSpecs = [{add, Field, <<"20220101">>}], - [{IdxK, IdxV}] = - leveled_codec:idx_indexspecs(IndexSpecs, - <<"Bucket">>, - <<"Key">>, - 1, + [{IdxK, IdxV}] = + leveled_codec:idx_indexspecs(IndexSpecs, + <<"Bucket">>, + <<"Key">>, + 1, infinity), - {ok, P2, {IdxK, IdxK}, _Bloom2} = + {ok, P2, {IdxK, IdxK}, _Bloom2} = sst_new(?TEST_AREA, FileName, 1, [{IdxK, IdxV}], 6000, OptsSST), ?assertMatch( [{IdxK, IdxV}], @@ -4999,17 +4988,17 @@ strange_range_test() -> V = leveled_head:riak_metadata_to_binary( term_to_binary([{"actor1", 1}]), <<1:32/integer, 0:32/integer, 0:32/integer>>), - OptsSST = + OptsSST = #sst_options{press_method=native, log_options=leveled_log:get_opts()}, - + FK = leveled_codec:to_ledgerkey({<<"T0">>, <<"B0">>}, <<"K0">>, ?RIAK_TAG), LK = leveled_codec:to_ledgerkey({<<"T0">>, <<"B0">>}, <<"K02">>, ?RIAK_TAG), EK = leveled_codec:to_ledgerkey({<<"T0">>, <<"B0">>}, <<"K0299">>, ?RIAK_TAG), KL1 = lists:map( - fun(I) -> + fun(I) -> leveled_codec:to_ledgerkey( {<<"T0">>, <<"B0">>}, list_to_binary("K00" ++ integer_to_list(I)), @@ -5018,30 +5007,30 @@ strange_range_test() -> lists:seq(1, 300)), KL2 = lists:map( - fun(I) -> + fun(I) -> leveled_codec:to_ledgerkey( {<<"T0">>, <<"B0">>}, list_to_binary("K02" ++ integer_to_list(I)), ?RIAK_TAG) end, lists:seq(1, 300)), - + GenerateValue = fun(K) -> element( 3, leveled_codec:generate_ledgerkv(K, 1, V, 16, infinity)) end, - KVL = + KVL = lists:ukeysort( 1, lists:map( fun(K) -> {K, GenerateValue(K)} end, [FK] ++ KL1 ++ [LK] ++ KL2)), - - {ok, P1, {FK, EK}, _Bloom1} = + + {ok, P1, {FK, EK}, _Bloom1} = sst_new(?TEST_AREA, FileName, 1, KVL, 6000, OptsSST), - + ?assertMatch(LK, element(1, sst_get(P1, LK))), ?assertMatch(FK, element(1, sst_get(P1, FK))), ok = sst_close(P1), @@ -5051,13 +5040,13 @@ strange_range_test() -> lists:map( fun(I) -> {add, <<"t1_bin">>, integer_to_binary(I)} end, lists:seq(1, 500)), - IdxKVL = - leveled_codec:idx_indexspecs(IndexSpecs, - <<"Bucket">>, - <<"Key">>, - 1, + IdxKVL = + leveled_codec:idx_indexspecs(IndexSpecs, + <<"Bucket">>, + <<"Key">>, + 1, infinity), - {ok, P2, {_FIdxK, _EIdxK}, _Bloom2} = + {ok, P2, {_FIdxK, _EIdxK}, _Bloom2} = sst_new( ?TEST_AREA, FileName, 1, lists:ukeysort(1, IdxKVL), 6000, OptsSST), [{IdxK1, _IdxV1}, {IdxK2, _IdxV2}] = @@ -5076,7 +5065,7 @@ strange_range_test() -> ), ok = sst_close(P2), ok = file:delete(filename:join(?TEST_AREA, FileName ++ ".sst")). - + receive_fun() -> receive @@ -5088,10 +5077,10 @@ receive_fun() -> start_sst_fun(ProcessToInform) -> N = 3000, KVL1 = lists:ukeysort(1, generate_randomkeys(N + 1, N, 1, 20)), - OptsSST = + OptsSST = #sst_options{press_method=native, log_options=leveled_log:get_opts()}, - {ok, P1, {_FK1, _LK1}, _Bloom1} = + {ok, P1, {_FK1, _LK1}, _Bloom1} = sst_new(?TEST_AREA, "level1_src", 1, KVL1, 6000, OptsSST), ProcessToInform ! {sst_pid, P1}. diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index 0f018c0..b163fa8 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -68,6 +68,7 @@ replace_everything(_Config) -> testutil:put_altered_indexed_objects(Book1, BKT, KSpcL1), ok = testutil:check_indexed_objects(Book1, BKT, KSpcL2, V2), compact_and_wait(Book1, 1000), + compact_and_wait(Book1, 1000), {ok, FileList1} = file:list_dir(CompPath), io:format("Number of files after compaction ~w~n", [length(FileList1)]), compact_and_wait(Book1, 1000),