diff --git a/.gitignore b/.gitignore index ae9a11c..667578a 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ cover cover_* .eqc-info leveled_data/* +elp diff --git a/include/leveled.hrl b/include/leveled.hrl index 55b8281..6295b4c 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -70,6 +70,15 @@ %% Inker key type used for tombstones %%%============================================================================ +%%%============================================================================ +%%% Helper Function +%%%============================================================================ + +-define(IS_DEF(Attribute), Attribute =/= undefined). + +-if(?OTP_RELEASE < 26). +-type dynamic() :: any(). +-endif. %%%============================================================================ %%% Shared records @@ -79,13 +88,6 @@ is_basement = false :: boolean(), timestamp :: integer()}). --record(manifest_entry, - {start_key :: tuple() | undefined, - end_key :: tuple() | undefined, - owner :: pid()|list(), - filename :: string() | undefined, - bloom = none :: leveled_ebloom:bloom() | none}). - -record(cdb_options, {max_size :: pos_integer() | undefined, max_count :: pos_integer() | undefined, @@ -129,14 +131,15 @@ singlefile_compactionperc :: float()|undefined, maxrunlength_compactionperc :: float()|undefined, score_onein = 1 :: pos_integer(), - snaptimeout_long :: pos_integer() | undefined, + snaptimeout_long = 60 :: pos_integer(), monitor = {no_monitor, 0} :: leveled_monitor:monitor()}). -record(penciller_options, {root_path :: string() | undefined, sst_options = #sst_options{} :: #sst_options{}, - max_inmemory_tablesize :: integer() | undefined, + max_inmemory_tablesize = ?MIN_PCL_CACHE_SIZE + :: pos_integer(), start_snapshot = false :: boolean(), snapshot_query, bookies_pid :: pid() | undefined, diff --git a/rebar.config b/rebar.config index b76e2c2..1953a96 100644 --- a/rebar.config +++ b/rebar.config @@ -12,6 +12,14 @@ {eunit_opts, [verbose]}. +{project_plugins, [ + {eqwalizer_rebar3, + {git_subdir, + "https://github.com/whatsapp/eqwalizer.git", + {branch, "main"}, + "eqwalizer_rebar3"}} +]}. + {profiles, [{eqc, [{deps, [meck, fqc]}, {erl_opts, [debug_info, {d, 'EQC'}]}, @@ -28,7 +36,8 @@ {deps, [ {lz4, ".*", {git, "https://github.com/nhs-riak/erlang-lz4", {branch, "nhse-develop-3.4"}}}, - {zstd, ".*", {git, "https://github.com/nhs-riak/zstd-erlang", {branch, "nhse-develop"}}} + {zstd, ".*", {git, "https://github.com/nhs-riak/zstd-erlang", {branch, "nhse-develop"}}}, + {eqwalizer_support, {git_subdir, "https://github.com/whatsapp/eqwalizer.git", {branch, "main"}, "eqwalizer_support"}} ]}. {ct_opts, [{dir, ["test/end_to_end"]}]}. diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index a0f8ba7..4aee189 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -148,7 +148,7 @@ min_sqn = infinity :: integer()|infinity, max_sqn = 0 :: integer()}). --record(state, {inker :: pid() | undefined, +-record(state, {inker :: pid() | null, penciller :: pid() | undefined, cache_size :: pos_integer() | undefined, cache_multiple :: pos_integer() | undefined, @@ -359,18 +359,29 @@ ]. -type load_item() :: - {leveled_codec:journal_key_tag()|null, - leveled_codec:ledger_key()|?DUMMY, - non_neg_integer(), any(), integer(), - leveled_codec:journal_keychanges()}. + { + leveled_codec:journal_key_tag()|null, + leveled_codec:primary_key()|?DUMMY, + leveled_codec:sqn(), + dynamic(), + leveled_codec:journal_keychanges(), + integer() + }. -type initial_loadfun() :: fun((leveled_codec:journal_key(), - any(), + dynamic(), non_neg_integer(), - {non_neg_integer(), non_neg_integer(), ledger_cache()}, + {non_neg_integer(), non_neg_integer(), list(load_item())}, fun((any()) -> {binary(), non_neg_integer()})) -> - {loop|stop, list(load_item())}). + {loop|stop, + { + non_neg_integer(), + non_neg_integer(), + list(load_item()) + } + } + ). -export_type([initial_loadfun/0, ledger_cache/0]). @@ -421,7 +432,9 @@ book_start(RootPath, LedgerCacheSize, JournalSize, SyncStrategy) -> %% comments on the open_options() type book_start(Opts) -> - gen_server:start_link(?MODULE, [set_defaults(Opts)], []). + {ok, Bookie} = + gen_server:start_link(?MODULE, [set_defaults(Opts)], []), + {ok, Bookie}. -spec book_plainstart(list(tuple())) -> {ok, pid()}. @@ -429,7 +442,9 @@ book_start(Opts) -> %% @doc %% Start used in tests to start without linking book_plainstart(Opts) -> - gen_server:start(?MODULE, [set_defaults(Opts)], []). + {ok, Bookie} = + gen_server:start(?MODULE, [set_defaults(Opts)], []), + {ok, Bookie}. -spec book_tempput(pid(), leveled_codec:key(), leveled_codec:key(), any(), @@ -448,8 +463,8 @@ book_plainstart(Opts) -> %% reload_strategy. If expired objects are to be compacted entirely, then the %% history of KeyChanges will be lost on reload. -book_tempput(Pid, Bucket, Key, Object, IndexSpecs, Tag, TTL) - when is_integer(TTL) -> +book_tempput( + Pid, Bucket, Key, Object, IndexSpecs, Tag, TTL) when is_integer(TTL) -> book_put(Pid, Bucket, Key, Object, IndexSpecs, Tag, TTL). %% @doc - Standard PUT @@ -613,7 +628,7 @@ book_sqn(Pid, Bucket, Key) -> book_sqn(Pid, Bucket, Key, Tag) -> gen_server:call(Pid, {head, Bucket, Key, Tag, true}, infinity). --spec book_returnfolder(pid(), tuple()) -> {async, fun(() -> term())}. +-spec book_returnfolder(pid(), tuple()) -> {async, fun(() -> dynamic())}. %% @doc Folds over store - deprecated %% The tuple() is a query, and book_returnfolder will return an {async, Folder} @@ -685,13 +700,12 @@ book_returnfolder(Pid, RunnerType) -> FoldAccT :: {FoldFun, Acc}, Range :: {IndexField, Start, End}, TermHandling :: {ReturnTerms, TermRegex}) -> - {async, Runner::fun(() -> term())} + {async, Runner::fun(() -> dynamic())} when Bucket::term(), Key :: term(), StartKey::term(), FoldFun::fun((Bucket, Key | {IndexVal, Key}, Acc) -> Acc), - Key::term(), - Acc::term(), + Acc::dynamic(), IndexField::term(), IndexVal::term(), Start::IndexVal, @@ -728,10 +742,9 @@ book_indexfold(Pid, Bucket, FoldAccT, Range, TermHandling) -> Tag :: leveled_codec:tag(), FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Constraint :: first | all, Bucket :: term(), - Acc :: term(), Runner :: fun(() -> Acc). book_bucketlist(Pid, Tag, FoldAccT, Constraint) -> RunnerType= @@ -758,7 +771,7 @@ book_bucketlist(Pid, Tag, FoldAccT, Constraint) -> Tag :: leveled_codec:tag(), FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), Key :: term(), Runner :: fun(() -> Acc). @@ -772,7 +785,7 @@ book_keylist(Pid, Tag, FoldAccT) -> Tag :: leveled_codec:tag(), FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), Key :: term(), Runner :: fun(() -> Acc). @@ -790,7 +803,7 @@ book_keylist(Pid, Tag, Bucket, FoldAccT) -> Tag :: leveled_codec:tag(), FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), KeyRange :: {StartKey, EndKey} | all, StartKey :: Key, @@ -809,7 +822,7 @@ book_keylist(Pid, Tag, Bucket, KeyRange, FoldAccT) -> Tag :: leveled_codec:tag(), FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), KeyRange :: {StartKey, EndKey} | all, StartKey :: Key, @@ -838,7 +851,7 @@ book_keylist(Pid, Tag, Bucket, KeyRange, FoldAccT, TermRegex) -> Tag :: leveled_codec:tag(), FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Value, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), Key :: term(), Value :: term(), @@ -860,7 +873,7 @@ book_objectfold(Pid, Tag, FoldAccT, SnapPreFold) -> Tag :: leveled_codec:tag(), FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Value, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), Key :: term(), Value :: term(), @@ -884,7 +897,7 @@ book_objectfold(Pid, Tag, FoldAccT, SnapPreFold, Order) -> Tag :: leveled_codec:tag(), FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Value, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), Key :: term(), Value :: term(), @@ -940,7 +953,7 @@ book_objectfold(Pid, Tag, Bucket, Limiter, FoldAccT, SnapPreFold) -> Tag :: leveled_codec:tag(), FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Value, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), Key :: term(), Value :: term(), @@ -975,7 +988,7 @@ book_headfold(Pid, Tag, FoldAccT, JournalCheck, SnapPreFold, SegmentList) -> EndKey :: Key, FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Value, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), Key :: term(), Value :: term(), @@ -1008,7 +1021,7 @@ book_headfold(Pid, Tag, Limiter, FoldAccT, JournalCheck, SnapPreFold, SegmentLis EndKey :: Key, FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Value, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), Key :: term(), Value :: term(), @@ -1040,10 +1053,9 @@ book_headfold(Pid, Tag, all, FoldAccT, JournalCheck, SnapPreFold, SegmentList, LastModRange, MaxObjectCount}, book_returnfolder(Pid, RunnerType). --spec book_snapshot(pid(), - store|ledger, - tuple()|undefined, - boolean()|undefined) -> {ok, pid(), pid()|null}. +-spec book_snapshot( + pid(), store|ledger, tuple()|no_lookup|undefined, boolean()) + -> {ok, pid(), pid()|null}. %% @doc create a snapshot of the store %% @@ -1151,9 +1163,8 @@ book_headstatus(Pid) -> %%% gen_server callbacks %%%============================================================================ --spec init([open_options()]) -> {ok, book_state()}. +-spec init([open_options()]) -> {ok, book_state()}|{stop, atom()}. init([Opts]) -> - leveled_rand:seed(), case {proplists:get_value(snapshot_bookie, Opts), proplists:get_value(root_path, Opts)} of {undefined, undefined} -> @@ -1262,16 +1273,21 @@ init([Opts]) -> end. -handle_call({put, Bucket, Key, Object, IndexSpecs, Tag, TTL, DataSync}, - From, State) when State#state.head_only == false -> - LedgerKey = leveled_codec:to_ledgerkey(Bucket, Key, Tag), +handle_call( + {put, Bucket, Key, Object, IndexSpecs, Tag, TTL, DataSync}, + From, + State) + when State#state.head_only == false , Tag =/= ?HEAD_TAG -> + LedgerKey = leveled_codec:to_objectkey(Bucket, Key, Tag), SWLR = os:timestamp(), SW0 = leveled_monitor:maybe_time(State#state.monitor), - {ok, SQN, ObjSize} = leveled_inker:ink_put(State#state.inker, - LedgerKey, - Object, - {IndexSpecs, TTL}, - DataSync), + {ok, SQN, ObjSize} = + leveled_inker:ink_put( + State#state.inker, + LedgerKey, + Object, + {IndexSpecs, TTL}, + DataSync), {T0, SW1} = leveled_monitor:step_time(SW0), Changes = preparefor_ledgercache( @@ -1302,9 +1318,13 @@ handle_call({mput, ObjectSpecs, TTL}, From, State) {ok, SQN} = leveled_inker:ink_mput(State#state.inker, dummy, {ObjectSpecs, TTL}), Changes = - preparefor_ledgercache(?INKT_MPUT, ?DUMMY, - SQN, null, length(ObjectSpecs), - {ObjectSpecs, TTL}), + preparefor_ledgercache( + ?INKT_MPUT, + ?DUMMY, + SQN, null, + length(ObjectSpecs), + {ObjectSpecs, TTL} + ), Cache0 = addto_ledgercache(Changes, State#state.ledger_cache), case State#state.slow_offer of true -> @@ -1324,7 +1344,7 @@ handle_call({mput, ObjectSpecs, TTL}, From, State) end; handle_call({get, Bucket, Key, Tag}, _From, State) when State#state.head_only == false -> - LedgerKey = leveled_codec:to_ledgerkey(Bucket, Key, Tag), + LedgerKey = leveled_codec:to_objectkey(Bucket, Key, Tag), SW0 = leveled_monitor:maybe_time(State#state.monitor), {H0, _CacheHit} = fetch_head(LedgerKey, @@ -1370,7 +1390,7 @@ handle_call({get, Bucket, Key, Tag}, _From, State) handle_call({head, Bucket, Key, Tag, SQNOnly}, _From, State) when State#state.head_lookup == true -> SW0 = leveled_monitor:maybe_time(State#state.monitor), - LK = leveled_codec:to_ledgerkey(Bucket, Key, Tag), + LK = leveled_codec:to_objectkey(Bucket, Key, Tag), {Head, CacheHit} = fetch_head(LK, State#state.penciller, @@ -1412,7 +1432,7 @@ handle_call({head, Bucket, Key, Tag, SQNOnly}, _From, State) case {LedgerMD, SQNOnly} of {not_found, _} -> not_found; - {_, false} -> + {LedgerMD, false} when LedgerMD =/= null -> {ok, leveled_head:build_head(Tag, LedgerMD)}; {_, true} -> {ok, SQN} @@ -1426,14 +1446,18 @@ handle_call({head, Bucket, Key, Tag, SQNOnly}, _From, State) UpdJrnalCheckFreq -> {reply, Reply, State#state{ink_checking = UpdJrnalCheckFreq}} end; -handle_call({snapshot, SnapType, Query, LongRunning}, _From, State) -> +handle_call( + {snapshot, SnapType, Query, LongRunning}, + _From, + State = #state{penciller = Pcl}) + when is_pid(Pcl) -> % Snapshot the store, specifying if the snapshot should be long running % (i.e. will the snapshot be queued or be required for an extended period % e.g. many minutes) {ok, PclSnap, InkSnap} = snapshot_store( State#state.ledger_cache, - State#state.penciller, + Pcl, State#state.inker, State#state.monitor, SnapType, @@ -1494,9 +1518,11 @@ handle_call(hot_backup, _From, State) when State#state.head_only == false -> bookies_pid = self()}, {ok, Snapshot} = leveled_inker:ink_snapstart(InkerOpts), {reply, {async, BackupFun(Snapshot)}, State}; -handle_call(close, _From, State) -> - leveled_inker:ink_close(State#state.inker), - leveled_penciller:pcl_close(State#state.penciller), +handle_call( + close, _From, State = #state{inker = Inker, penciller = Pcl}) + when is_pid(Inker), is_pid(Pcl) -> + leveled_inker:ink_close(Inker), + leveled_penciller:pcl_close(Pcl), leveled_monitor:monitor_close(element(1, State#state.monitor)), {stop, normal, ok, State}; handle_call(destroy, _From, State=#state{is_snapshot=Snp}) when Snp == false -> @@ -1515,11 +1541,11 @@ handle_call(Msg, _From, State) -> {reply, {unsupported_message, element(1, Msg)}, State}. -handle_cast({log_level, LogLevel}, State) -> - PCL = State#state.penciller, - INK = State#state.inker, - ok = leveled_penciller:pcl_loglevel(PCL, LogLevel), - ok = leveled_inker:ink_loglevel(INK, LogLevel), +handle_cast( + {log_level, LogLevel}, State = #state{inker = Inker, penciller = Pcl}) + when is_pid(Inker), is_pid(Pcl) -> + ok = leveled_penciller:pcl_loglevel(Pcl, LogLevel), + ok = leveled_inker:ink_loglevel(Inker, LogLevel), case element(1, State#state.monitor) of no_monitor -> ok; @@ -1528,11 +1554,11 @@ handle_cast({log_level, LogLevel}, State) -> end, ok = leveled_log:set_loglevel(LogLevel), {noreply, State}; -handle_cast({add_logs, ForcedLogs}, State) -> - PCL = State#state.penciller, - INK = State#state.inker, - ok = leveled_penciller:pcl_addlogs(PCL, ForcedLogs), - ok = leveled_inker:ink_addlogs(INK, ForcedLogs), +handle_cast( + {add_logs, ForcedLogs}, State = #state{inker = Inker, penciller = Pcl}) + when is_pid(Inker), is_pid(Pcl) -> + ok = leveled_penciller:pcl_addlogs(Pcl, ForcedLogs), + ok = leveled_inker:ink_addlogs(Inker, ForcedLogs), case element(1, State#state.monitor) of no_monitor -> ok; @@ -1541,11 +1567,11 @@ handle_cast({add_logs, ForcedLogs}, State) -> end, ok = leveled_log:add_forcedlogs(ForcedLogs), {noreply, State}; -handle_cast({remove_logs, ForcedLogs}, State) -> - PCL = State#state.penciller, - INK = State#state.inker, - ok = leveled_penciller:pcl_removelogs(PCL, ForcedLogs), - ok = leveled_inker:ink_removelogs(INK, ForcedLogs), +handle_cast( + {remove_logs, ForcedLogs}, State = #state{inker = Inker, penciller = Pcl}) + when is_pid(Inker), is_pid(Pcl) -> + ok = leveled_penciller:pcl_removelogs(Pcl, ForcedLogs), + ok = leveled_inker:ink_removelogs(Inker, ForcedLogs), case element(1, State#state.monitor) of no_monitor -> ok; @@ -1661,8 +1687,8 @@ loadqueue_ledgercache(Cache) -> null|pid(), leveled_monitor:monitor(), store|ledger, - undefined|tuple(), - undefined|boolean()) -> + undefined|no_lookup|tuple(), + boolean()) -> {ok, pid(), pid()|null}. %% @doc %% Allow all a snapshot to be created from part of the store, preferably @@ -1679,7 +1705,7 @@ loadqueue_ledgercache(Cache) -> %% lookup is required but the range isn't defined then 'undefined' should be %% passed as the query snapshot_store( - LedgerCache, Penciller, Inker, Monitor, SnapType, Query, LongRunning) -> + LedgerCache, Penciller, Ink, Monitor, SnapType, Query, LongRunning) -> SW0 = leveled_monitor:maybe_time(Monitor), LedgerCacheReady = readycache_forsnapshot(LedgerCache, Query), BookiesMem = {LedgerCacheReady#ledger_cache.loader, @@ -1687,21 +1713,25 @@ snapshot_store( LedgerCacheReady#ledger_cache.min_sqn, LedgerCacheReady#ledger_cache.max_sqn}, PCLopts = - #penciller_options{start_snapshot = true, - source_penciller = Penciller, - snapshot_query = Query, - snapshot_longrunning = LongRunning, - bookies_pid = self(), - bookies_mem = BookiesMem}, + #penciller_options{ + start_snapshot = true, + source_penciller = Penciller, + snapshot_query = Query, + snapshot_longrunning = LongRunning, + bookies_pid = self(), + bookies_mem = BookiesMem + }, {TS0, SW1} = leveled_monitor:step_time(SW0), {ok, LedgerSnapshot} = leveled_penciller:pcl_snapstart(PCLopts), {TS1, _SW2} = leveled_monitor:step_time(SW1), ok = maybelog_snap_timing(Monitor, TS0, TS1), case SnapType of - store -> - InkerOpts = #inker_options{start_snapshot = true, - bookies_pid = self(), - source_inker = Inker}, + store when is_pid(Ink) -> + InkerOpts = + #inker_options{ + start_snapshot = true, + bookies_pid = self(), + source_inker = Ink}, {ok, JournalSnapshot} = leveled_inker:ink_snapstart(InkerOpts), {ok, LedgerSnapshot, JournalSnapshot}; ledger -> @@ -1797,8 +1827,13 @@ set_options(Opts, Monitor) -> ReloadStrategy = leveled_codec:inker_reload_strategy(AltStrategy), PCLL0CacheSize = - max(?MIN_PCL_CACHE_SIZE, - proplists:get_value(max_pencillercachesize, Opts)), + case proplists:get_value(max_pencillercachesize, Opts) of + P0CS when is_integer(P0CS), P0CS > ?MIN_PCL_CACHE_SIZE -> + P0CS; + _ -> + ?MIN_PCL_CACHE_SIZE + end, + RootPath = proplists:get_value(root_path, Opts), JournalFP = filename:join(RootPath, ?JOURNAL_FP), @@ -1896,7 +1931,13 @@ set_options(Opts, Monitor) -> %% previous snapshot should be used instead. Also the snapshot should not be %% closed as part of the post-query activity as the snapshot may be reused, and %% should be manually closed. -return_snapfun(State, SnapType, Query, LongRunning, SnapPreFold) -> +return_snapfun( + State = #state{penciller = Pcl, inker = Ink}, + SnapType, + Query, + LongRunning, + SnapPreFold) + when is_pid(Pcl), is_pid(Ink) -> CloseFun = fun(LS0, JS0) -> fun() -> @@ -1965,9 +2006,9 @@ get_runner(State, {index_query, Constraint, FoldAccT, Range, TermHandling}) -> {B, null} end, StartKey = - leveled_codec:to_ledgerkey(Bucket, ObjKey0, ?IDX_TAG, IdxFld, StartT), + leveled_codec:to_querykey(Bucket, ObjKey0, ?IDX_TAG, IdxFld, StartT), EndKey = - leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, IdxFld, EndT), + leveled_codec:to_querykey(Bucket, null, ?IDX_TAG, IdxFld, EndT), SnapFun = return_snapfun(State, ledger, {StartKey, EndKey}, false, false), leveled_runner:index_query(SnapFun, {StartKey, EndKey, TermHandling}, @@ -2119,8 +2160,15 @@ get_deprecatedrunner(State, PartitionFilter). --spec return_ledger_keyrange(atom(), any(), tuple()|all) -> - {tuple(), tuple(), tuple()|no_lookup}. +-spec return_ledger_keyrange( + atom(), leveled_codec:key(), tuple()|all) + -> + { + leveled_codec:query_key(), + leveled_codec:query_key(), + {leveled_codec:query_key(), + leveled_codec:query_key()}|no_lookup + }. %% @doc %% Convert a range of binary keys into a ledger key range, returning %% {StartLK, EndLK, Query} where Query is to indicate whether the query @@ -2129,16 +2177,16 @@ return_ledger_keyrange(Tag, Bucket, KeyRange) -> {StartKey, EndKey, Snap} = case KeyRange of all -> - {leveled_codec:to_ledgerkey(Bucket, null, Tag), - leveled_codec:to_ledgerkey(Bucket, null, Tag), + {leveled_codec:to_querykey(Bucket, null, Tag), + leveled_codec:to_querykey(Bucket, null, Tag), false}; {StartTerm, <<"$all">>} -> - {leveled_codec:to_ledgerkey(Bucket, StartTerm, Tag), - leveled_codec:to_ledgerkey(Bucket, null, Tag), + {leveled_codec:to_querykey(Bucket, StartTerm, Tag), + leveled_codec:to_querykey(Bucket, null, Tag), false}; {StartTerm, EndTerm} -> - {leveled_codec:to_ledgerkey(Bucket, StartTerm, Tag), - leveled_codec:to_ledgerkey(Bucket, EndTerm, Tag), + {leveled_codec:to_querykey(Bucket, StartTerm, Tag), + leveled_codec:to_querykey(Bucket, EndTerm, Tag), true} end, SnapQuery = @@ -2164,8 +2212,8 @@ maybe_longrunning(SW, Aspect) -> ok end. --spec readycache_forsnapshot(ledger_cache(), tuple()|no_lookup|undefined) - -> ledger_cache(). +-spec readycache_forsnapshot( + ledger_cache(), tuple()|no_lookup|undefined) -> ledger_cache(). %% @doc %% Strip the ledger cach back to only the relevant information needed in %% the query, and to make the cache a snapshot (and so not subject to changes @@ -2303,7 +2351,7 @@ journal_notfound(CheckFrequency, Inker, LK, SQN) -> {boolean(), integer()}. %% @doc Use a function to check if an item is found check_notfound(CheckFrequency, CheckFun) -> - case leveled_rand:uniform(?MAX_KEYCHECK_FREQUENCY) of + case rand:uniform(?MAX_KEYCHECK_FREQUENCY) of X when X =< CheckFrequency -> case CheckFun() of probably -> @@ -2316,28 +2364,32 @@ check_notfound(CheckFrequency, CheckFun) -> end. --spec preparefor_ledgercache(leveled_codec:journal_key_tag()|null, - leveled_codec:ledger_key()|?DUMMY, - non_neg_integer(), any(), integer(), - leveled_codec:journal_keychanges()) - -> {leveled_codec:segment_hash(), - non_neg_integer(), - list(leveled_codec:ledger_kv())}. +-spec preparefor_ledgercache( + leveled_codec:journal_key_tag()|null, + leveled_codec:primary_key()|?DUMMY, + non_neg_integer(), + any(), + integer(), + leveled_codec:journal_keychanges()) + -> {leveled_codec:segment_hash(), + non_neg_integer(), + list(leveled_codec:ledger_kv())}. %% @doc %% Prepare an object and its related key changes for addition to the Ledger %% via the Ledger Cache. -preparefor_ledgercache(?INKT_MPUT, - ?DUMMY, SQN, _O, _S, {ObjSpecs, TTL}) -> +preparefor_ledgercache(?INKT_MPUT, ?DUMMY, SQN, _O, _S, {ObjSpecs, TTL}) -> ObjChanges = leveled_codec:obj_objectspecs(ObjSpecs, SQN, TTL), {no_lookup, SQN, ObjChanges}; -preparefor_ledgercache(?INKT_KEYD, - LedgerKey, SQN, _Obj, _Size, {IdxSpecs, TTL}) -> +preparefor_ledgercache( + ?INKT_KEYD, LedgerKey, SQN, _Obj, _Size, {IdxSpecs, TTL}) + when LedgerKey =/= ?DUMMY -> {Bucket, Key} = leveled_codec:from_ledgerkey(LedgerKey), KeyChanges = leveled_codec:idx_indexspecs(IdxSpecs, Bucket, Key, SQN, TTL), {no_lookup, SQN, KeyChanges}; -preparefor_ledgercache(_InkTag, - LedgerKey, SQN, Obj, Size, {IdxSpecs, TTL}) -> +preparefor_ledgercache( + _InkTag, LedgerKey, SQN, Obj, Size, {IdxSpecs, TTL}) + when LedgerKey =/= ?DUMMY -> {Bucket, Key, MetaValue, {KeyH, _ObjH}, _LastMods} = leveled_codec:generate_ledgerkv(LedgerKey, SQN, Obj, Size, TTL), KeyChanges = @@ -2346,31 +2398,31 @@ preparefor_ledgercache(_InkTag, {KeyH, SQN, KeyChanges}. --spec recalcfor_ledgercache(leveled_codec:journal_key_tag()|null, - leveled_codec:ledger_key()|?DUMMY, - non_neg_integer(), any(), integer(), - leveled_codec:journal_keychanges(), - ledger_cache(), - pid()) - -> {leveled_codec:segment_hash(), - non_neg_integer(), - list(leveled_codec:ledger_kv())}. +-spec recalcfor_ledgercache( + leveled_codec:journal_key_tag()|null, + leveled_codec:primary_key()|?DUMMY, + non_neg_integer(), + binary()|term(), + integer(), + leveled_codec:journal_keychanges(), + ledger_cache(), + pid()) + -> {leveled_codec:segment_hash(), + non_neg_integer(), + list(leveled_codec:ledger_kv())}. %% @doc %% When loading from the journal to the ledger, may hit a key which has the %% `recalc` strategy. Such a key needs to recalculate the key changes by %% comparison with the current state of the ledger, assuming it is a full %% journal entry (i.e. KeyDeltas which may be a result of previously running %% with a retain strategy should be ignored). -recalcfor_ledgercache(InkTag, - _LedgerKey, SQN, _Obj, _Size, {_IdxSpecs, _TTL}, - _LedgerCache, - _Penciller) - when InkTag == ?INKT_MPUT; InkTag == ?INKT_KEYD -> +recalcfor_ledgercache( + InkTag, _LedgerKey, SQN, _Obj, _Size, {_IdxSpecs, _TTL}, _LC, _Pcl) + when InkTag == ?INKT_MPUT; InkTag == ?INKT_KEYD -> {no_lookup, SQN, []}; -recalcfor_ledgercache(_InkTag, - LK, SQN, Obj, Size, {_IgnoreJournalIdxSpecs, TTL}, - LedgerCache, - Penciller) -> +recalcfor_ledgercache( + _InkTag, LK, SQN, Obj, Size, {_Ignore, TTL}, LedgerCache, Penciller) + when LK =/= ?DUMMY -> {Bucket, Key, MetaValue, {KeyH, _ObjH}, _LastMods} = leveled_codec:generate_ledgerkv(LK, SQN, Obj, Size, TTL), OldObject = @@ -2385,9 +2437,16 @@ recalcfor_ledgercache(_InkTag, not_present -> not_present; {LK, LV} -> - leveled_codec:get_metadata(LV) + case leveled_codec:get_metadata(LV) of + MDO when MDO =/= null -> + MDO + end + end, + UpdMetadata = + case leveled_codec:get_metadata(MetaValue) of + MDU when MDU =/= null -> + MDU end, - UpdMetadata = leveled_codec:get_metadata(MetaValue), IdxSpecs = leveled_head:diff_indexspecs(element(1, LK), UpdMetadata, OldMetadata), {KeyH, @@ -2396,11 +2455,12 @@ recalcfor_ledgercache(_InkTag, ++ leveled_codec:idx_indexspecs(IdxSpecs, Bucket, Key, SQN, TTL)}. --spec addto_ledgercache({leveled_codec:segment_hash(), - non_neg_integer(), - list(leveled_codec:ledger_kv())}, - ledger_cache()) - -> ledger_cache(). +-spec addto_ledgercache( + {leveled_codec:segment_hash(), + non_neg_integer(), + list(leveled_codec:ledger_kv())}, + ledger_cache()) + -> ledger_cache(). %% @doc %% Add a set of changes associated with a single sequence number (journal %% update) and key to the ledger cache. If the changes are not to be looked @@ -2412,12 +2472,13 @@ addto_ledgercache({H, SQN, KeyChanges}, Cache) -> min_sqn=min(SQN, Cache#ledger_cache.min_sqn), max_sqn=max(SQN, Cache#ledger_cache.max_sqn)}. --spec addto_ledgercache({integer()|no_lookup, - integer(), - list(leveled_codec:ledger_kv())}, - ledger_cache(), - loader) - -> ledger_cache(). +-spec addto_ledgercache( + {leveled_codec:segment_hash()|no_lookup, + integer(), + list(leveled_codec:ledger_kv())}, + ledger_cache(), + loader) + -> ledger_cache(). %% @doc %% Add a set of changes associated with a single sequence number (journal %% update) to the ledger cache. This is used explicitly when loading the @@ -2469,10 +2530,13 @@ maybepush_ledgercache(MaxCacheSize, MaxCacheMult, Cache, Penciller) -> TimeToPush = maybe_withjitter(CacheSize, MaxCacheSize, MaxCacheMult), if TimeToPush -> - CacheToLoad = {Tab, - Cache#ledger_cache.index, - Cache#ledger_cache.min_sqn, - Cache#ledger_cache.max_sqn}, + CacheToLoad = + { + Tab, + Cache#ledger_cache.index, + Cache#ledger_cache.min_sqn, + Cache#ledger_cache.max_sqn + }, case leveled_penciller:pcl_pushmem(Penciller, CacheToLoad) of ok -> Cache0 = #ledger_cache{}, @@ -2493,7 +2557,7 @@ maybepush_ledgercache(MaxCacheSize, MaxCacheMult, Cache, Penciller) -> %% a push should be maybe_withjitter( CacheSize, MaxCacheSize, MaxCacheMult) when CacheSize > MaxCacheSize -> - R = leveled_rand:uniform(MaxCacheMult * MaxCacheSize), + R = rand:uniform(MaxCacheMult * MaxCacheSize), (CacheSize - MaxCacheSize) > R; maybe_withjitter(_CacheSize, _MaxCacheSize, _MaxCacheMult) -> false. @@ -2515,7 +2579,8 @@ get_loadfun() -> _ -> {VBin, ValSize} = ExtractFun(ValueInJournal), % VBin may already be a term - {Obj, IdxSpecs} = leveled_codec:split_inkvalue(VBin), + {Obj, IdxSpecs} = + leveled_codec:revert_value_from_journal(VBin), case SQN of MaxSQN -> {stop, @@ -2546,11 +2611,14 @@ delete_path(DirPath) -> leveled_monitor:timing(), leveled_monitor:timing(), pos_integer()) -> ok. -maybelog_put_timing(_Monitor, no_timing, no_timing, no_timing, _Size) -> - ok; -maybelog_put_timing({Pid, _StatsFreq}, InkTime, PrepTime, MemTime, Size) -> +maybelog_put_timing( + {Pid, _StatsFreq}, InkTime, PrepTime, MemTime, Size) + when is_pid(Pid), + is_integer(InkTime), is_integer(PrepTime), is_integer(MemTime) -> leveled_monitor:add_stat( - Pid, {bookie_put_update, InkTime, PrepTime, MemTime, Size}). + Pid, {bookie_put_update, InkTime, PrepTime, MemTime, Size}); +maybelog_put_timing(_Monitor, _, _, _, _Size) -> + ok. -spec maybelog_head_timing( leveled_monitor:monitor(), @@ -2558,37 +2626,42 @@ maybelog_put_timing({Pid, _StatsFreq}, InkTime, PrepTime, MemTime, Size) -> leveled_monitor:timing(), boolean(), boolean()) -> ok. -maybelog_head_timing(_Monitor, no_timing, no_timing, _NF, _CH) -> - ok; -maybelog_head_timing({Pid, _StatsFreq}, FetchTime, _, true, _CH) -> - leveled_monitor:add_stat( - Pid, {bookie_head_update, FetchTime, not_found, 0}); -maybelog_head_timing({Pid, _StatsFreq}, FetchTime, RspTime, _NF, CH) -> +maybelog_head_timing({Pid, _StatsFreq}, FetchTime, RspTime, false, CH) + when is_pid(Pid), is_integer(FetchTime), is_integer(RspTime) -> CH0 = case CH of true -> 1; false -> 0 end, leveled_monitor:add_stat( - Pid, {bookie_head_update, FetchTime, RspTime, CH0}). + Pid, {bookie_head_update, FetchTime, RspTime, CH0}); +maybelog_head_timing({Pid, _StatsFreq}, FetchTime, _, true, _CH) + when is_pid(Pid), is_integer(FetchTime) -> + leveled_monitor:add_stat( + Pid, {bookie_head_update, FetchTime, not_found, 0}); +maybelog_head_timing(_Monitor, _, _, _NF, _CH) -> + ok. -spec maybelog_get_timing( leveled_monitor:monitor(), leveled_monitor:timing(), leveled_monitor:timing(), boolean()) -> ok. -maybelog_get_timing(_Monitor, no_timing, no_timing, _NF) -> - ok; -maybelog_get_timing({Pid, _StatsFreq}, HeadTime, _BodyTime, true) -> +maybelog_get_timing({Pid, _StatsFreq}, HeadTime, BodyTime, false) + when is_pid(Pid), is_integer(HeadTime), is_integer(BodyTime) -> + leveled_monitor:add_stat(Pid, {bookie_get_update, HeadTime, BodyTime}); +maybelog_get_timing({Pid, _StatsFreq}, HeadTime, _BodyTime, true) + when is_pid(Pid), is_integer(HeadTime) -> leveled_monitor:add_stat(Pid, {bookie_get_update, HeadTime, not_found}); -maybelog_get_timing({Pid, _StatsFreq}, HeadTime, BodyTime, false) -> - leveled_monitor:add_stat(Pid, {bookie_get_update, HeadTime, BodyTime}). - +maybelog_get_timing(_Monitor, _, _, _NF) -> + ok. -spec maybelog_snap_timing( leveled_monitor:monitor(), leveled_monitor:timing(), leveled_monitor:timing()) -> ok. -maybelog_snap_timing(_Monitor, no_timing, no_timing) -> - ok; -maybelog_snap_timing({Pid, _StatsFreq}, BookieTime, PCLTime) -> - leveled_monitor:add_stat(Pid, {bookie_snap_update, BookieTime, PCLTime}). +maybelog_snap_timing({Pid, _StatsFreq}, BookieTime, PCLTime) + when is_pid(Pid), is_integer(BookieTime), is_integer(PCLTime) -> + leveled_monitor:add_stat(Pid, {bookie_snap_update, BookieTime, PCLTime}); +maybelog_snap_timing(_Monitor, _, _) -> + ok. + %%%============================================================================ %%% Test @@ -2603,27 +2676,32 @@ maybelog_snap_timing({Pid, _StatsFreq}, BookieTime, PCLTime) -> book_returnactors(Pid) -> gen_server:call(Pid, return_actors). - reset_filestructure() -> RootPath = "test/test_area", leveled_inker:clean_testdir(RootPath ++ "/" ++ ?JOURNAL_FP), leveled_penciller:clean_testdir(RootPath ++ "/" ++ ?LEDGER_FP), RootPath. - generate_multiple_objects(Count, KeyNumber) -> generate_multiple_objects(Count, KeyNumber, []). generate_multiple_objects(0, _KeyNumber, ObjL) -> ObjL; generate_multiple_objects(Count, KeyNumber, ObjL) -> - Key = "Key" ++ integer_to_list(KeyNumber), - Value = leveled_rand:rand_bytes(256), - IndexSpec = [{add, "idx1_bin", "f" ++ integer_to_list(KeyNumber rem 10)}], - generate_multiple_objects(Count - 1, - KeyNumber + 1, - ObjL ++ [{Key, Value, IndexSpec}]). - + Key = list_to_binary("Key" ++ integer_to_list(KeyNumber)), + Value = crypto:strong_rand_bytes(256), + IndexSpec = + [ + { + add, <<"idx1_bin">>, + list_to_binary("f" ++ integer_to_list(KeyNumber rem 10)) + } + ], + generate_multiple_objects( + Count - 1, + KeyNumber + 1, + ObjL ++ [{Key, Value, IndexSpec}] + ). shutdown_test_() -> {timeout, 10, fun shutdown_tester/0}. @@ -2648,7 +2726,9 @@ shutdown_tester() -> timer:sleep(2000), ok = leveled_penciller:pcl_close(SnpPCL1), - ok = leveled_inker:ink_close(SnpJrnl1), + case SnpJrnl1 of + P when is_pid(P) -> ok = leveled_inker:ink_close(SnpJrnl1) + end, SW = os:timestamp(), receive ok -> ok end, WaitForShutDown = timer:now_diff(SW, os:timestamp()) div 1000, @@ -2662,83 +2742,105 @@ ttl_test() -> ObjL1 = generate_multiple_objects(100, 1), % Put in all the objects with a TTL in the future Future = leveled_util:integer_now() + 300, - lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, - "Bucket", K, V, S, - ?STD_TAG, - Future) end, - ObjL1), - lists:foreach(fun({K, V, _S}) -> - {ok, V} = book_get(Bookie1, "Bucket", K, ?STD_TAG) - end, - ObjL1), - lists:foreach(fun({K, _V, _S}) -> - {ok, _} = book_head(Bookie1, "Bucket", K, ?STD_TAG) - end, - ObjL1), + lists:foreach( + fun({K, V, S}) -> + ok = + book_tempput(Bookie1, <<"Bucket">>, K, V, S, ?STD_TAG, Future) + end, + ObjL1 + ), + lists:foreach( + fun({K, V, _S}) -> + {ok, V} = book_get(Bookie1, <<"Bucket">>, K, ?STD_TAG) + end, + ObjL1 + ), + lists:foreach( + fun({K, _V, _S}) -> + {ok, _} = book_head(Bookie1, <<"Bucket">>, K, ?STD_TAG) + end, + ObjL1 + ), ObjL2 = generate_multiple_objects(100, 101), Past = leveled_util:integer_now() - 300, - lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, - "Bucket", K, V, S, - ?STD_TAG, - Past) end, - ObjL2), - lists:foreach(fun({K, _V, _S}) -> - not_found = book_get(Bookie1, "Bucket", K, ?STD_TAG) - end, - ObjL2), - lists:foreach(fun({K, _V, _S}) -> - not_found = book_head(Bookie1, "Bucket", K, ?STD_TAG) - end, + lists:foreach( + fun({K, V, S}) -> + ok = book_tempput(Bookie1, <<"Bucket">>, K, V, S, ?STD_TAG, Past) + end, + ObjL2), + lists:foreach( + fun({K, _V, _S}) -> + not_found = book_get(Bookie1, <<"Bucket">>, K, ?STD_TAG) + end, + ObjL2), + lists:foreach( + fun({K, _V, _S}) -> + not_found = book_head(Bookie1, <<"Bucket">>, K, ?STD_TAG) + end, ObjL2), - {async, BucketFolder} = book_returnfolder(Bookie1, - {bucket_stats, "Bucket"}), + {async, BucketFolder} = + book_returnfolder(Bookie1, {bucket_stats, <<"Bucket">>}), {_Size, Count} = BucketFolder(), ?assertMatch(100, Count), FoldKeysFun = fun(_B, Item, FKFAcc) -> FKFAcc ++ [Item] end, - {async, - IndexFolder} = book_returnfolder(Bookie1, - {index_query, - "Bucket", - {FoldKeysFun, []}, - {"idx1_bin", "f8", "f9"}, - {false, undefined}}), + {async, IndexFolder} = + book_returnfolder( + Bookie1, + { + index_query, + <<"Bucket">>, + {FoldKeysFun, []}, + {<<"idx1_bin">>, <<"f8">>, <<"f9">>}, + {false, undefined} + } + ), KeyList = IndexFolder(), ?assertMatch(20, length(KeyList)), {ok, Regex} = re:compile("f8"), - {async, - IndexFolderTR} = book_returnfolder(Bookie1, - {index_query, - "Bucket", - {FoldKeysFun, []}, - {"idx1_bin", "f8", "f9"}, - {true, Regex}}), + {async, IndexFolderTR} = + book_returnfolder( + Bookie1, + { + index_query, + <<"Bucket">>, + {FoldKeysFun, []}, + {<<"idx1_bin">>, <<"f8">>, <<"f9">>}, + {true, Regex}} + ), TermKeyList = IndexFolderTR(), ?assertMatch(10, length(TermKeyList)), ok = book_close(Bookie1), {ok, Bookie2} = book_start([{root_path, RootPath}]), - {async, - IndexFolderTR2} = book_returnfolder(Bookie2, - {index_query, - "Bucket", - {FoldKeysFun, []}, - {"idx1_bin", "f7", "f9"}, - {false, Regex}}), + {async, IndexFolderTR2} = + book_returnfolder( + Bookie2, + { + index_query, + <<"Bucket">>, + {FoldKeysFun, []}, + {<<"idx1_bin">>, <<"f7">>, <<"f9">>}, + {false, Regex}} + ), KeyList2 = IndexFolderTR2(), ?assertMatch(10, length(KeyList2)), - lists:foreach(fun({K, _V, _S}) -> - not_found = book_get(Bookie2, "Bucket", K, ?STD_TAG) - end, - ObjL2), - lists:foreach(fun({K, _V, _S}) -> - not_found = book_head(Bookie2, "Bucket", K, ?STD_TAG) - end, - ObjL2), + lists:foreach( + fun({K, _V, _S}) -> + not_found = book_get(Bookie2, <<"Bucket">>, K, ?STD_TAG) + end, + ObjL2 + ), + lists:foreach( + fun({K, _V, _S}) -> + not_found = book_head(Bookie2, <<"Bucket">>, K, ?STD_TAG) + end, + ObjL2 + ), ok = book_close(Bookie2), reset_filestructure(). @@ -2748,45 +2850,55 @@ hashlist_query_test_() -> hashlist_query_testto() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}]), + {ok, Bookie1} = + book_start( + [ + {root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500} + ] + ), ObjL1 = generate_multiple_objects(1200, 1), % Put in all the objects with a TTL in the future Future = leveled_util:integer_now() + 300, - lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, - "Bucket", K, V, S, - ?STD_TAG, - Future) end, - ObjL1), + lists:foreach( + fun({K, V, S}) -> + ok = book_tempput(Bookie1, <<"Bucket">>, K, V, S, ?STD_TAG, Future) + end, + ObjL1 + ), ObjL2 = generate_multiple_objects(20, 1201), % Put in a few objects with a TTL in the past Past = leveled_util:integer_now() - 300, - lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, - "Bucket", K, V, S, - ?STD_TAG, - Past) end, - ObjL2), + lists:foreach( + fun({K, V, S}) -> + ok = book_tempput(Bookie1, <<"Bucket">>, K, V, S, ?STD_TAG, Past) + end, + ObjL2 + ), % Scan the store for the Bucket, Keys and Hashes - {async, HTFolder} = book_returnfolder(Bookie1, - {hashlist_query, - ?STD_TAG, - false}), + {async, HTFolder} = + book_returnfolder(Bookie1, {hashlist_query, ?STD_TAG, false}), KeyHashList = HTFolder(), - lists:foreach(fun({B, _K, H}) -> - ?assertMatch("Bucket", B), - ?assertMatch(true, is_integer(H)) - end, - KeyHashList), + lists:foreach( + fun({B, _K, H}) -> + ?assertMatch(<<"Bucket">>, B), + ?assertMatch(true, is_integer(H)) + end, + KeyHashList) + , ?assertMatch(1200, length(KeyHashList)), ok = book_close(Bookie1), - {ok, Bookie2} = book_start([{root_path, RootPath}, - {max_journalsize, 200000}, - {cache_size, 500}]), - {async, HTFolder2} = book_returnfolder(Bookie2, - {hashlist_query, - ?STD_TAG, - false}), + {ok, Bookie2} = + book_start( + [ + {root_path, RootPath}, + {max_journalsize, 200000}, + {cache_size, 500} + ] + ), + {async, HTFolder2} = + book_returnfolder(Bookie2, {hashlist_query, ?STD_TAG, false}), L0 = length(KeyHashList), HTR2 = HTFolder2(), ?assertMatch(L0, length(HTR2)), @@ -2800,26 +2912,28 @@ hashlist_query_withjournalcheck_test_() -> hashlist_query_withjournalcheck_testto() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}]), + {ok, Bookie1} = + book_start( + [ + {root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500} + ] + ), ObjL1 = generate_multiple_objects(800, 1), % Put in all the objects with a TTL in the future Future = leveled_util:integer_now() + 300, - lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, - "Bucket", K, V, S, - ?STD_TAG, - Future) end, - ObjL1), - {async, HTFolder1} = book_returnfolder(Bookie1, - {hashlist_query, - ?STD_TAG, - false}), + lists:foreach( + fun({K, V, S}) -> + ok = book_tempput(Bookie1, <<"Bucket">>, K, V, S, ?STD_TAG, Future) + end, + ObjL1 + ), + {async, HTFolder1} = + book_returnfolder(Bookie1, {hashlist_query, ?STD_TAG, false}), KeyHashList = HTFolder1(), - {async, HTFolder2} = book_returnfolder(Bookie1, - {hashlist_query, - ?STD_TAG, - true}), + {async, HTFolder2} = + book_returnfolder(Bookie1, {hashlist_query, ?STD_TAG, true}), ?assertMatch(KeyHashList, HTFolder2()), ok = book_close(Bookie1), reset_filestructure(). @@ -2829,68 +2943,89 @@ foldobjects_vs_hashtree_test_() -> foldobjects_vs_hashtree_testto() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}]), + {ok, Bookie1} = + book_start( + [{ + root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500} + ]), ObjL1 = generate_multiple_objects(800, 1), % Put in all the objects with a TTL in the future Future = leveled_util:integer_now() + 300, - lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, - "Bucket", K, V, S, - ?STD_TAG, - Future) end, - ObjL1), - {async, HTFolder1} = book_returnfolder(Bookie1, - {hashlist_query, - ?STD_TAG, - false}), + lists:foreach( + fun({K, V, S}) -> ok = + book_tempput(Bookie1, <<"Bucket">>, K, V, S, ?STD_TAG, Future) + end, + ObjL1 + ), + {async, HTFolder1} = + book_returnfolder(Bookie1, {hashlist_query, ?STD_TAG, false}), KeyHashList1 = lists:usort(HTFolder1()), - FoldObjectsFun = fun(B, K, V, Acc) -> - [{B, K, erlang:phash2(term_to_binary(V))}|Acc] end, - {async, HTFolder2} = book_returnfolder(Bookie1, - {foldobjects_allkeys, - ?STD_TAG, - FoldObjectsFun, - true}), + FoldObjectsFun = + fun(B, K, V, Acc) -> + [{B, K, erlang:phash2(term_to_binary(V))}|Acc] + end, + {async, HTFolder2} = + book_returnfolder( + Bookie1, + { + foldobjects_allkeys, + ?STD_TAG, + FoldObjectsFun, + true + } + ), KeyHashList2 = HTFolder2(), ?assertMatch(KeyHashList1, lists:usort(KeyHashList2)), FoldHeadsFun = fun(B, K, ProxyV, Acc) -> - {proxy_object, - _MDBin, - _Size, - {FetchFun, Clone, JK}} = binary_to_term(ProxyV), + {proxy_object, _MDBin, _Size, {FetchFun, Clone, JK}} = + binary_to_term(ProxyV), V = FetchFun(Clone, JK), [{B, K, erlang:phash2(term_to_binary(V))}|Acc] end, {async, HTFolder3} = - book_returnfolder(Bookie1, - {foldheads_allkeys, - ?STD_TAG, - FoldHeadsFun, - true, true, false, false, false}), + book_returnfolder( + Bookie1, + { + foldheads_allkeys, + ?STD_TAG, + FoldHeadsFun, + true, + true, + false, + false, + false + } + ), KeyHashList3 = HTFolder3(), ?assertMatch(KeyHashList1, lists:usort(KeyHashList3)), FoldHeadsFun2 = fun(B, K, ProxyV, Acc) -> - {proxy_object, - MD, - _Size1, - _Fetcher} = binary_to_term(ProxyV), + {proxy_object, MD, _Size1, _Fetcher} = binary_to_term(ProxyV), {Hash, _Size0, _UserDefinedMD} = MD, [{B, K, Hash}|Acc] end, {async, HTFolder4} = - book_returnfolder(Bookie1, - {foldheads_allkeys, - ?STD_TAG, - FoldHeadsFun2, - false, false, false, false, false}), + book_returnfolder( + Bookie1, + { + foldheads_allkeys, + ?STD_TAG, + FoldHeadsFun2, + false, + false, + false, + false, + false + } + ), KeyHashList4 = HTFolder4(), ?assertMatch(KeyHashList1, lists:usort(KeyHashList4)), @@ -2908,138 +3043,207 @@ foldobjects_vs_foldheads_bybucket_testto() -> folder_cache_test(CacheSize) -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, CacheSize}]), + {ok, Bookie1} = + book_start( + [{ + root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, CacheSize + }] + ), _ = book_returnactors(Bookie1), ObjL1 = generate_multiple_objects(400, 1), ObjL2 = generate_multiple_objects(400, 1), % Put in all the objects with a TTL in the future Future = leveled_util:integer_now() + 300, - lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, - "BucketA", K, V, S, - ?STD_TAG, - Future) end, - ObjL1), - lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, - "BucketB", K, V, S, - ?STD_TAG, - Future) end, + lists:foreach( + fun({K, V, S}) -> + ok = + book_tempput(Bookie1, <<"BucketA">>, K, V, S, ?STD_TAG, Future) + end, + ObjL1 + ), + lists:foreach( + fun({K, V, S}) -> + ok = + book_tempput(Bookie1, <<"BucketB">>, K, V, S, ?STD_TAG, Future) + end, ObjL2), - FoldObjectsFun = fun(B, K, V, Acc) -> - [{B, K, erlang:phash2(term_to_binary(V))}|Acc] end, + FoldObjectsFun = + fun(B, K, V, Acc) -> + [{B, K, erlang:phash2(term_to_binary(V))}|Acc] + end, {async, HTFolder1A} = - book_returnfolder(Bookie1, - {foldobjects_bybucket, - ?STD_TAG, - "BucketA", - all, - FoldObjectsFun, - false}), + book_returnfolder( + Bookie1, + { + foldobjects_bybucket, + ?STD_TAG, + <<"BucketA">>, + all, + FoldObjectsFun, + false + } + ), KeyHashList1A = HTFolder1A(), {async, HTFolder1B} = - book_returnfolder(Bookie1, - {foldobjects_bybucket, - ?STD_TAG, - "BucketB", - all, - FoldObjectsFun, - true}), + book_returnfolder( + Bookie1, + { + foldobjects_bybucket, + ?STD_TAG, + <<"BucketB">>, + all, + FoldObjectsFun, + true + } + ), KeyHashList1B = HTFolder1B(), - ?assertMatch(false, - lists:usort(KeyHashList1A) == lists:usort(KeyHashList1B)), + ?assertMatch( + false, + lists:usort(KeyHashList1A) == lists:usort(KeyHashList1B) + ), FoldHeadsFun = fun(B, K, ProxyV, Acc) -> - {proxy_object, - _MDBin, - _Size, - {FetchFun, Clone, JK}} = binary_to_term(ProxyV), + {proxy_object, _MDBin, _Size, {FetchFun, Clone, JK}} = + binary_to_term(ProxyV), V = FetchFun(Clone, JK), [{B, K, erlang:phash2(term_to_binary(V))}|Acc] end, {async, HTFolder2A} = - book_returnfolder(Bookie1, - {foldheads_bybucket, - ?STD_TAG, - "BucketA", - all, - FoldHeadsFun, - true, true, - false, false, false}), - KeyHashList2A = HTFolder2A(), + book_returnfolder( + Bookie1, + { + foldheads_bybucket, + ?STD_TAG, + <<"BucketA">>, + all, + FoldHeadsFun, + true, + true, + false, + false,false + } + ), + KeyHashList2A = return_list_result(HTFolder2A), {async, HTFolder2B} = - book_returnfolder(Bookie1, - {foldheads_bybucket, - ?STD_TAG, - "BucketB", - all, - FoldHeadsFun, - true, false, - false, false, false}), - KeyHashList2B = HTFolder2B(), + book_returnfolder( + Bookie1, + { + foldheads_bybucket, + ?STD_TAG, + <<"BucketB">>, + all, + FoldHeadsFun, + true, + false, + false, + false, + false + } + ), + KeyHashList2B = return_list_result(HTFolder2B), - ?assertMatch(true, - lists:usort(KeyHashList1A) == lists:usort(KeyHashList2A)), - ?assertMatch(true, - lists:usort(KeyHashList1B) == lists:usort(KeyHashList2B)), + ?assertMatch( + true, + lists:usort(KeyHashList1A) == lists:usort(KeyHashList2A) + ), + ?assertMatch( + true, + lists:usort(KeyHashList1B) == lists:usort(KeyHashList2B) + ), {async, HTFolder2C} = - book_returnfolder(Bookie1, - {foldheads_bybucket, - ?STD_TAG, - "BucketB", - {"Key", <<"$all">>}, - FoldHeadsFun, - true, false, - false, false, false}), - KeyHashList2C = HTFolder2C(), + book_returnfolder( + Bookie1, + { + foldheads_bybucket, + ?STD_TAG, + <<"BucketB">>, + {<<"Key">>, <<"$all">>}, + FoldHeadsFun, + true, + false, + false, + false, + false + } + ), + KeyHashList2C = return_list_result(HTFolder2C), {async, HTFolder2D} = - book_returnfolder(Bookie1, - {foldheads_bybucket, - ?STD_TAG, - "BucketB", - {"Key", "Keyzzzzz"}, - FoldHeadsFun, - true, true, - false, false, false}), - KeyHashList2D = HTFolder2D(), - ?assertMatch(true, - lists:usort(KeyHashList2B) == lists:usort(KeyHashList2C)), - ?assertMatch(true, - lists:usort(KeyHashList2B) == lists:usort(KeyHashList2D)), + book_returnfolder( + Bookie1, + { + foldheads_bybucket, + ?STD_TAG, + <<"BucketB">>, + {<<"Key">>, <<"Keyzzzzz">>}, + FoldHeadsFun, + true, + true, + false, + false, + false + } + ), + KeyHashList2D = return_list_result(HTFolder2D), + ?assertMatch( + true, + lists:usort(KeyHashList2B) == lists:usort(KeyHashList2C) + ), + ?assertMatch( + true, + lists:usort(KeyHashList2B) == lists:usort(KeyHashList2D) + ), - CheckSplitQueryFun = fun(SplitInt) -> io:format("Testing SplitInt ~w~n", [SplitInt]), - SplitIntEnd = "Key" ++ integer_to_list(SplitInt) ++ "|", - SplitIntStart = "Key" ++ integer_to_list(SplitInt + 1), + SplitIntEnd = + list_to_binary("Key" ++ integer_to_list(SplitInt) ++ "|"), + SplitIntStart = + list_to_binary("Key" ++ integer_to_list(SplitInt + 1)), {async, HTFolder2E} = - book_returnfolder(Bookie1, - {foldheads_bybucket, - ?STD_TAG, - "BucketB", - {"Key", SplitIntEnd}, - FoldHeadsFun, - true, false, - false, false, false}), - KeyHashList2E = HTFolder2E(), + book_returnfolder( + Bookie1, + { + foldheads_bybucket, + ?STD_TAG, + <<"BucketB">>, + {<<"Key">>, SplitIntEnd}, + FoldHeadsFun, + true, + false, + false, + false, + false + } + ), + KeyHashList2E = return_list_result(HTFolder2E), {async, HTFolder2F} = - book_returnfolder(Bookie1, - {foldheads_bybucket, - ?STD_TAG, - "BucketB", - {SplitIntStart, "Key|"}, - FoldHeadsFun, - true, false, - false, false, false}), - KeyHashList2F = HTFolder2F(), - + book_returnfolder( + Bookie1, + { + foldheads_bybucket, + ?STD_TAG, + <<"BucketB">>, + {SplitIntStart, <<"Key|">>}, + FoldHeadsFun, + true, + false, + false, + false, + false + } + ), + KeyHashList2F = return_list_result(HTFolder2F), + ?assertMatch(true, length(KeyHashList2E) > 0), ?assertMatch(true, length(KeyHashList2F) > 0), + io:format("Length of 2B ~w 2E ~w 2F ~w~n", [length(KeyHashList2B), length(KeyHashList2E), @@ -3053,25 +3257,41 @@ folder_cache_test(CacheSize) -> ok = book_close(Bookie1), reset_filestructure(). +-spec return_list_result(fun(() -> list(dynamic()))) -> list(). +return_list_result(FoldFun) -> + case FoldFun() of + HL when is_list(HL) -> + HL + end. + small_cachesize_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 1}]), + {ok, Bookie1} = + book_start( + [{ + root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 1} + ]), ok = leveled_bookie:book_close(Bookie1). is_empty_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}]), + {ok, Bookie1} = + book_start( + [{ + root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500} + ]), % Put in an object with a TTL in the future Future = leveled_util:integer_now() + 300, ?assertMatch(true, leveled_bookie:book_isempty(Bookie1, ?STD_TAG)), - ok = book_tempput(Bookie1, - <<"B">>, <<"K">>, {value, <<"V">>}, [], - ?STD_TAG, Future), + ok = + book_tempput( + Bookie1, <<"B">>, <<"K">>, {value, <<"V">>}, [], ?STD_TAG, Future + ), ?assertMatch(false, leveled_bookie:book_isempty(Bookie1, ?STD_TAG)), ?assertMatch(true, leveled_bookie:book_isempty(Bookie1, ?RIAK_TAG)), @@ -3079,13 +3299,17 @@ is_empty_test() -> is_empty_headonly_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}, - {head_only, no_lookup}]), + {ok, Bookie1} = + book_start( + [ + {root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500}, + {head_only, no_lookup} + ]), ?assertMatch(true, book_isempty(Bookie1, ?HEAD_TAG)), ObjSpecs = - [{add, <<"B1">>, <<"K1">>, <<1:8/integer>>, 100}, + [{add, <<"B1">>, <<"K1">>, <<1:8/integer>>, {size, 100}}, {remove, <<"B1">>, <<"K1">>, <<0:8/integer>>, null}], ok = book_mput(Bookie1, ObjSpecs), ?assertMatch(false, book_isempty(Bookie1, ?HEAD_TAG)), @@ -3098,30 +3322,32 @@ undefined_rootpath_test() -> ?assertMatch({error, no_root_path}, R), error_logger:tty(true). - foldkeys_headonly_test() -> - foldkeys_headonly_tester(5000, 25, "BucketStr"), + foldkeys_headonly_tester(5000, 25, <<"BucketStr">>), foldkeys_headonly_tester(2000, 25, <<"B0">>). - foldkeys_headonly_tester(ObjectCount, BlockSize, BStr) -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}, - {head_only, no_lookup}]), + {ok, Bookie1} = + book_start( + [{root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500}, + {head_only, no_lookup}] + ), GenObjSpecFun = fun(I) -> Key = I rem 6, - {add, BStr, <>, integer_to_list(I), I} + {add, BStr, <>, <>, null} end, ObjSpecs = lists:map(GenObjSpecFun, lists:seq(1, ObjectCount)), ObjSpecBlocks = - lists:map(fun(I) -> - lists:sublist(ObjSpecs, I * BlockSize + 1, BlockSize) - end, - lists:seq(0, ObjectCount div BlockSize - 1)), + lists:map( + fun(I) -> + lists:sublist(ObjSpecs, I * BlockSize + 1, BlockSize) + end, + lists:seq(0, ObjectCount div BlockSize - 1)), lists:map(fun(Block) -> book_mput(Bookie1, Block) end, ObjSpecBlocks), ?assertMatch(false, book_isempty(Bookie1, ?HEAD_TAG)), @@ -3130,98 +3356,101 @@ foldkeys_headonly_tester(ObjectCount, BlockSize, BStr) -> ?HEAD_TAG, BStr, {fun(_B, {K, SK}, Acc) -> [{K, SK}|Acc] end, []} }, - {async, Folder1} = book_returnfolder(Bookie1, FolderT), - Key_SKL1 = lists:reverse(Folder1()), + Key_SKL_Compare = - lists:usort(lists:map(fun({add, _B, K, SK, _V}) -> {K, SK} end, ObjSpecs)), - ?assertMatch(Key_SKL_Compare, Key_SKL1), + lists:usort( + lists:map( + fun({add, _B, K, SK, _V}) -> {K, SK} end, ObjSpecs + ) + ), + {async, Folder1} = book_returnfolder(Bookie1, FolderT), + case Folder1() of + Key_SKL1 when is_list(Key_SKL1) -> + ?assertMatch(Key_SKL_Compare, lists:reverse(Key_SKL1)) + end, ok = book_close(Bookie1), - {ok, Bookie2} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}, - {head_only, no_lookup}]), + {ok, Bookie2} = + book_start( + [ + {root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500}, + {head_only, no_lookup} + ]), + {async, Folder2} = book_returnfolder(Bookie2, FolderT), - Key_SKL2 = lists:reverse(Folder2()), - ?assertMatch(Key_SKL_Compare, Key_SKL2), + case Folder2() of + Key_SKL2 when is_list(Key_SKL2) -> + ?assertMatch(Key_SKL_Compare, lists:reverse(Key_SKL2)) + end, ok = book_close(Bookie2). is_empty_stringkey_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}]), + {ok, Bookie1} = + book_start( + [ + {root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500} + ]), ?assertMatch(true, book_isempty(Bookie1, ?STD_TAG)), Past = leveled_util:integer_now() - 300, ?assertMatch(true, leveled_bookie:book_isempty(Bookie1, ?STD_TAG)), - ok = book_tempput(Bookie1, - "B", "K", {value, <<"V">>}, [], - ?STD_TAG, Past), - ok = book_put(Bookie1, - "B", "K0", {value, <<"V">>}, [], - ?STD_TAG), + ok = + book_tempput( + Bookie1, <<"B">>, <<"K">>, {value, <<"V">>}, [], ?STD_TAG, Past + ), + ok = book_put(Bookie1, <<"B">>, <<"K0">>, {value, <<"V">>}, [], ?STD_TAG), ?assertMatch(false, book_isempty(Bookie1, ?STD_TAG)), ok = book_close(Bookie1). scan_table_test() -> - K1 = leveled_codec:to_ledgerkey(<<"B1">>, - <<"K1">>, - ?IDX_TAG, - <<"F1-bin">>, - <<"AA1">>), - K2 = leveled_codec:to_ledgerkey(<<"B1">>, - <<"K2">>, - ?IDX_TAG, - <<"F1-bin">>, - <<"AA1">>), - K3 = leveled_codec:to_ledgerkey(<<"B1">>, - <<"K3">>, - ?IDX_TAG, - <<"F1-bin">>, - <<"AB1">>), - K4 = leveled_codec:to_ledgerkey(<<"B1">>, - <<"K4">>, - ?IDX_TAG, - <<"F1-bin">>, - <<"AA2">>), - K5 = leveled_codec:to_ledgerkey(<<"B2">>, - <<"K5">>, - ?IDX_TAG, - <<"F1-bin">>, - <<"AA2">>), + K1 = + leveled_codec:to_objectkey( + <<"B1">>, <<"K1">>, ?IDX_TAG, <<"F1-bin">>, <<"AA1">>), + K2 = + leveled_codec:to_objectkey( + <<"B1">>, <<"K2">>, ?IDX_TAG, <<"F1-bin">>, <<"AA1">>), + K3 = + leveled_codec:to_objectkey( + <<"B1">>, <<"K3">>, ?IDX_TAG, <<"F1-bin">>, <<"AB1">>), + K4 = + leveled_codec:to_objectkey( + <<"B1">>, <<"K4">>, ?IDX_TAG, <<"F1-bin">>, <<"AA2">>), + K5 = + leveled_codec:to_objectkey( + <<"B2">>, <<"K5">>, ?IDX_TAG, <<"F1-bin">>, <<"AA2">>), Tab0 = ets:new(mem, [ordered_set]), - SK_A0 = leveled_codec:to_ledgerkey(<<"B1">>, - null, - ?IDX_TAG, - <<"F1-bin">>, - <<"AA0">>), - EK_A9 = leveled_codec:to_ledgerkey(<<"B1">>, - null, - ?IDX_TAG, - <<"F1-bin">>, - <<"AA9">>), + SK_A0 = + leveled_codec:to_querykey( + <<"B1">>, null, ?IDX_TAG, <<"F1-bin">>, <<"AA0">>), + EK_A9 = + leveled_codec:to_querykey( + <<"B1">>, null, ?IDX_TAG, <<"F1-bin">>, <<"AA9">>), Empty = {[], infinity, 0}, - ?assertMatch(Empty, - scan_table(Tab0, SK_A0, EK_A9)), + ?assertMatch(Empty, scan_table(Tab0, SK_A0, EK_A9)), ets:insert(Tab0, [{K1, {1, active, no_lookup, null}}]), - ?assertMatch({[{K1, _}], 1, 1}, - scan_table(Tab0, SK_A0, EK_A9)), + ?assertMatch({[{K1, _}], 1, 1}, scan_table(Tab0, SK_A0, EK_A9)), ets:insert(Tab0, [{K2, {2, active, no_lookup, null}}]), - ?assertMatch({[{K1, _}, {K2, _}], 1, 2}, - scan_table(Tab0, SK_A0, EK_A9)), + ?assertMatch({[{K1, _}, {K2, _}], 1, 2}, scan_table(Tab0, SK_A0, EK_A9)), ets:insert(Tab0, [{K3, {3, active, no_lookup, null}}]), - ?assertMatch({[{K1, _}, {K2, _}], 1, 2}, - scan_table(Tab0, SK_A0, EK_A9)), + ?assertMatch({[{K1, _}, {K2, _}], 1, 2}, scan_table(Tab0, SK_A0, EK_A9)), ets:insert(Tab0, [{K4, {4, active, no_lookup, null}}]), - ?assertMatch({[{K1, _}, {K2, _}, {K4, _}], 1, 4}, - scan_table(Tab0, SK_A0, EK_A9)), + ?assertMatch( + {[{K1, _}, {K2, _}, {K4, _}], 1, 4}, + scan_table(Tab0, SK_A0, EK_A9) + ), ets:insert(Tab0, [{K5, {5, active, no_lookup, null}}]), - ?assertMatch({[{K1, _}, {K2, _}, {K4, _}], 1, 4}, - scan_table(Tab0, SK_A0, EK_A9)). + ?assertMatch( + {[{K1, _}, {K2, _}, {K4, _}], 1, 4}, + scan_table(Tab0, SK_A0, EK_A9) + ). longrunning_test() -> SW = os:timestamp(), @@ -3229,31 +3458,36 @@ longrunning_test() -> ok = maybe_longrunning(SW, put). coverage_cheat_test() -> - {noreply, _State0} = handle_info(timeout, #state{}), - {ok, _State1} = code_change(null, #state{}, null). + DummyState = #state{inker = list_to_pid("<0.101.0>")}, + {noreply, _State0} = handle_info(timeout, DummyState), + {ok, _State1} = code_change(null, DummyState, null). erase_journal_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 50000}, - {cache_size, 100}]), + {ok, Bookie1} = + book_start( + [ + {root_path, RootPath}, + {max_journalsize, 50000}, + {cache_size, 100} + ]), ObjL1 = generate_multiple_objects(500, 1), % Put in all the objects with a TTL in the future lists:foreach( fun({K, V, S}) -> - ok = book_put(Bookie1, "Bucket", K, V, S, ?STD_TAG) + ok = book_put(Bookie1, <<"Bucket">>, K, V, S, ?STD_TAG) end, ObjL1), lists:foreach( fun({K, V, _S}) -> - {ok, V} = book_get(Bookie1, "Bucket", K, ?STD_TAG) + {ok, V} = book_get(Bookie1, <<"Bucket">>, K, ?STD_TAG) end, ObjL1), CheckHeadFun = fun(Book) -> fun({K, _V, _S}, Acc) -> - case book_head(Book, "Bucket", K, ?STD_TAG) of + case book_head(Book, <<"Bucket">>, K, ?STD_TAG) of {ok, _Head} -> Acc; not_found -> Acc + 1 end @@ -3265,70 +3499,73 @@ erase_journal_test() -> ok = book_close(Bookie1), io:format("Bookie closed - clearing Journal~n"), leveled_inker:clean_testdir(RootPath ++ "/" ++ ?JOURNAL_FP), - {ok, Bookie2} = book_start([{root_path, RootPath}, - {max_journalsize, 5000}, - {cache_size, 100}]), + {ok, Bookie2} = + book_start([ + {root_path, RootPath}, + {max_journalsize, 5000}, + {cache_size, 100}] + ), HeadsNotFound2 = lists:foldl(CheckHeadFun(Bookie2), 0, ObjL1), ?assertMatch(500, HeadsNotFound2), ok = book_destroy(Bookie2). sqnorder_fold_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}]), - ok = book_put(Bookie1, - <<"B">>, <<"K1">>, {value, <<"V1">>}, [], - ?STD_TAG), - ok = book_put(Bookie1, - <<"B">>, <<"K2">>, {value, <<"V2">>}, [], - ?STD_TAG), + {ok, Bookie1} = + book_start([ + {root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500}] + ), + ok = book_put(Bookie1, <<"B">>, <<"K1">>, {value, <<"V1">>}, [], ?STD_TAG), + ok = book_put(Bookie1, <<"B">>, <<"K2">>, {value, <<"V2">>}, [], ?STD_TAG), FoldObjectsFun = fun(B, K, V, Acc) -> Acc ++ [{B, K, V}] end, {async, ObjFPre} = - book_objectfold(Bookie1, - ?STD_TAG, {FoldObjectsFun, []}, true, sqn_order), + book_objectfold( + Bookie1, ?STD_TAG, {FoldObjectsFun, []}, true, sqn_order), {async, ObjFPost} = - book_objectfold(Bookie1, - ?STD_TAG, {FoldObjectsFun, []}, false, sqn_order), + book_objectfold( + Bookie1, ?STD_TAG, {FoldObjectsFun, []}, false, sqn_order), - ok = book_put(Bookie1, - <<"B">>, <<"K3">>, {value, <<"V3">>}, [], - ?STD_TAG), + ok = book_put(Bookie1, <<"B">>, <<"K3">>, {value, <<"V3">>}, [], ?STD_TAG), ObjLPre = ObjFPre(), - ?assertMatch([{<<"B">>, <<"K1">>, {value, <<"V1">>}}, - {<<"B">>, <<"K2">>, {value, <<"V2">>}}], ObjLPre), + ?assertMatch( + [{<<"B">>, <<"K1">>, {value, <<"V1">>}}, + {<<"B">>, <<"K2">>, {value, <<"V2">>}}], + ObjLPre + ), ObjLPost = ObjFPost(), - ?assertMatch([{<<"B">>, <<"K1">>, {value, <<"V1">>}}, - {<<"B">>, <<"K2">>, {value, <<"V2">>}}, - {<<"B">>, <<"K3">>, {value, <<"V3">>}}], ObjLPost), + ?assertMatch( + [{<<"B">>, <<"K1">>, {value, <<"V1">>}}, + {<<"B">>, <<"K2">>, {value, <<"V2">>}}, + {<<"B">>, <<"K3">>, {value, <<"V3">>}}], + ObjLPost + ), ok = book_destroy(Bookie1). sqnorder_mutatefold_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}]), - ok = book_put(Bookie1, - <<"B">>, <<"K1">>, {value, <<"V1">>}, [], - ?STD_TAG), - ok = book_put(Bookie1, - <<"B">>, <<"K1">>, {value, <<"V2">>}, [], - ?STD_TAG), + {ok, Bookie1} = + book_start([ + {root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500} + ]), + ok = book_put(Bookie1, <<"B">>, <<"K1">>, {value, <<"V1">>}, [], ?STD_TAG), + ok = book_put(Bookie1, <<"B">>, <<"K1">>, {value, <<"V2">>}, [], ?STD_TAG), FoldObjectsFun = fun(B, K, V, Acc) -> Acc ++ [{B, K, V}] end, {async, ObjFPre} = - book_objectfold(Bookie1, - ?STD_TAG, {FoldObjectsFun, []}, true, sqn_order), + book_objectfold( + Bookie1, ?STD_TAG, {FoldObjectsFun, []}, true, sqn_order), {async, ObjFPost} = - book_objectfold(Bookie1, - ?STD_TAG, {FoldObjectsFun, []}, false, sqn_order), + book_objectfold( + Bookie1, ?STD_TAG, {FoldObjectsFun, []}, false, sqn_order), - ok = book_put(Bookie1, - <<"B">>, <<"K1">>, {value, <<"V3">>}, [], - ?STD_TAG), + ok = book_put(Bookie1, <<"B">>, <<"K1">>, {value, <<"V3">>}, [], ?STD_TAG), ObjLPre = ObjFPre(), ?assertMatch([{<<"B">>, <<"K1">>, {value, <<"V2">>}}], ObjLPre), @@ -3340,19 +3577,22 @@ sqnorder_mutatefold_test() -> check_notfound_test() -> ProbablyFun = fun() -> probably end, MissingFun = fun() -> missing end, - MinFreq = lists:foldl(fun(_I, Freq) -> - {false, Freq0} = - check_notfound(Freq, ProbablyFun), - Freq0 - end, - 100, - lists:seq(1, 5000)), - % 5000 as needs to be a lot as doesn't decrement - % when random interval is not hit + MinFreq = + lists:foldl( + fun(_I, Freq) -> + {false, Freq0} = check_notfound(Freq, ProbablyFun), + Freq0 + end, + 100, + lists:seq(1, 5000)), + % 5000 as needs to be a lot as doesn't decrement + % when random interval is not hit ?assertMatch(?MIN_KEYCHECK_FREQUENCY, MinFreq), - ?assertMatch({true, ?MAX_KEYCHECK_FREQUENCY}, - check_notfound(?MAX_KEYCHECK_FREQUENCY, MissingFun)), + ?assertMatch( + {true, ?MAX_KEYCHECK_FREQUENCY}, + check_notfound(?MAX_KEYCHECK_FREQUENCY, MissingFun) + ), ?assertMatch({false, 0}, check_notfound(0, MissingFun)). diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 6c1f6a6..d53f8e9 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -94,15 +94,11 @@ hashtable_calc/2]). -define(DWORD_SIZE, 8). --define(WORD_SIZE, 4). -define(MAX_FILE_SIZE, 3221225472). -define(BINARY_MODE, false). -define(BASE_POSITION, 2048). -define(WRITE_OPS, [binary, raw, read, write]). --define(PENDING_ROLL_WAIT, 30). -define(DELETE_TIMEOUT, 10000). --define(TIMING_SAMPLECOUNTDOWN, 5000). --define(TIMING_SAMPLESIZE, 100). -define(GETPOS_FACTOR, 8). -define(MAX_OBJECT_SIZE, 1000000000). % 1GB but really should be much smaller than this @@ -111,18 +107,24 @@ -record(state, {hashtree, last_position :: integer() | undefined, + % defined when writing, not required once rolled last_key = empty, current_count = 0 :: non_neg_integer(), hash_index = {} :: tuple(), filename :: string() | undefined, - handle :: file:fd() | undefined, - max_size :: pos_integer() | undefined, - max_count :: pos_integer() | undefined, + % defined when starting + handle :: file:io_device() | undefined, + % defined when starting + max_size :: pos_integer(), + max_count :: pos_integer(), binary_mode = false :: boolean(), delete_point = 0 :: integer(), inker :: pid() | undefined, + % undefined until delete_pending deferred_delete = false :: boolean(), - waste_path :: string() | undefined, + waste_path :: string()|undefined, + % undefined has functional meaning + % - no sending to waste on delete sync_strategy = none, log_options = leveled_log:get_opts() :: leveled_log:log_options(), @@ -133,8 +135,11 @@ -type hashtable_index() :: tuple(). -type file_location() :: integer()|eof. -type filter_fun() :: - fun((any(), binary(), integer(), any(), fun((binary()) -> any())) -> - {stop|loop, any()}). + fun((any(), + binary(), + integer(), + term()|{term(), term()}, + fun((binary()) -> any())) -> {stop|loop, any()}). -export_type([filter_fun/0]). @@ -265,11 +270,11 @@ cdb_getpositions(Pid, SampleSize) -> cdb_getpositions_fromidx(Pid, FC, Index, Acc) end end, - RandFun = fun(X) -> {leveled_rand:uniform(), X} end, + RandFun = fun(X) -> {rand:uniform(), X} end, SeededL = lists:map(RandFun, lists:seq(0, 255)), SortedL = lists:keysort(1, SeededL), PosList0 = lists:foldl(FoldFun, [], SortedL), - P1 = leveled_rand:uniform(max(1, length(PosList0) - S0)), + P1 = rand:uniform(max(1, length(PosList0) - S0)), lists:sublist(lists:sort(PosList0), P1, S0) end. @@ -367,7 +372,7 @@ cdb_scan(Pid, FilterFun, InitAcc, StartPosition) -> {cdb_scan, FilterFun, InitAcc, StartPosition}, infinity). --spec cdb_lastkey(pid()) -> any(). +-spec cdb_lastkey(pid()) -> leveled_codec:journal_key()|empty. %% @doc %% Get the last key to be added to the file (which will have the highest %% sequence number) @@ -487,38 +492,49 @@ starting({call, From}, {open_reader, Filename, LastKey}, State) -> {next_state, reader, State0, [{reply, From, ok}, hibernate]}. -writer({call, From}, {get_kv, Key}, State) -> +writer( + {call, From}, {get_kv, Key}, State = #state{handle =IO}) + when ?IS_DEF(IO) -> {keep_state_and_data, [{reply, From, get_mem( Key, - State#state.handle, + IO, State#state.hashtree, State#state.binary_mode)}]}; -writer({call, From}, {key_check, Key}, State) -> +writer( + {call, From}, {key_check, Key}, State = #state{handle =IO}) + when ?IS_DEF(IO) -> {keep_state_and_data, [{reply, From, get_mem( Key, - State#state.handle, + IO, State#state.hashtree, State#state.binary_mode, loose_presence)}]}; -writer({call, From}, {put_kv, Key, Value, Sync}, State) -> +writer( + {call, From}, + {put_kv, Key, Value, Sync}, + State = #state{last_position = LP, handle = IO}) + when ?IS_DEF(last_position), ?IS_DEF(IO) -> NewCount = State#state.current_count + 1, case NewCount >= State#state.max_count of true -> {keep_state_and_data, [{reply, From, roll}]}; false -> - Result = put(State#state.handle, - Key, - Value, - {State#state.last_position, State#state.hashtree}, - State#state.binary_mode, - State#state.max_size, - State#state.last_key == empty), + Result = + put( + IO, + Key, + Value, + {LP, State#state.hashtree}, + State#state.binary_mode, + State#state.max_size, + State#state.last_key == empty + ), case Result of roll -> %% Key and value could not be written @@ -545,7 +561,11 @@ writer({call, From}, {put_kv, Key, Value, Sync}, State) -> end; writer({call, From}, {mput_kv, []}, _State) -> {keep_state_and_data, [{reply, From, ok}]}; -writer({call, From}, {mput_kv, KVList}, State) -> +writer( + {call, From}, + {mput_kv, KVList}, + State = #state{last_position = LP, handle = IO}) + when ?IS_DEF(last_position), ?IS_DEF(IO) -> NewCount = State#state.current_count + length(KVList), TooMany = NewCount >= State#state.max_count, NotEmpty = State#state.current_count > 0, @@ -553,11 +573,14 @@ writer({call, From}, {mput_kv, KVList}, State) -> true -> {keep_state_and_data, [{reply, From, roll}]}; false -> - Result = mput(State#state.handle, - KVList, - {State#state.last_position, State#state.hashtree}, - State#state.binary_mode, - State#state.max_size), + Result = + mput( + IO, + KVList, + {LP, State#state.hashtree}, + State#state.binary_mode, + State#state.max_size + ), case Result of roll -> %% Keys and values could not be written @@ -573,38 +596,46 @@ writer({call, From}, {mput_kv, KVList}, State) -> [{reply, From, ok}]} end end; -writer({call, From}, cdb_complete, State) -> - NewName = determine_new_filename(State#state.filename), +writer( + {call, From}, cdb_complete, State = #state{filename = FN}) + when ?IS_DEF(FN) -> + NewName = determine_new_filename(FN), ok = close_file(State#state.handle, State#state.hashtree, State#state.last_position), - ok = rename_for_read(State#state.filename, NewName), + ok = rename_for_read(FN, NewName), {stop_and_reply, normal, [{reply, From, {ok, NewName}}]}; writer({call, From}, Event, State) -> handle_sync_event(Event, From, State); -writer(cast, cdb_roll, State) -> +writer( + cast, cdb_roll, State = #state{last_position = LP}) + when ?IS_DEF(LP) -> ok = leveled_iclerk:clerk_hashtablecalc( - State#state.hashtree, State#state.last_position, self()), + State#state.hashtree, LP, self()), {next_state, rolling, State}. -rolling({call, From}, {get_kv, Key}, State) -> +rolling( + {call, From}, {get_kv, Key}, State = #state{handle = IO}) + when ?IS_DEF(IO) -> {keep_state_and_data, [{reply, From, get_mem( Key, - State#state.handle, + IO, State#state.hashtree, State#state.binary_mode)}]}; -rolling({call, From}, {key_check, Key}, State) -> +rolling( + {call, From}, {key_check, Key}, State = #state{handle = IO}) + when ?IS_DEF(IO) -> {keep_state_and_data, [{reply, From, get_mem( Key, - State#state.handle, + IO, State#state.hashtree, State#state.binary_mode, loose_presence)}]}; @@ -612,15 +643,19 @@ rolling({call, From}, {get_positions, _SampleSize, _Index, SampleAcc}, _State) -> {keep_state_and_data, [{reply, From, SampleAcc}]}; -rolling({call, From}, {return_hashtable, IndexList, HashTreeBin}, State) -> +rolling( + {call, From}, + {return_hashtable, IndexList, HashTreeBin}, + State = #state{filename = FN}) + when ?IS_DEF(FN) -> SW = os:timestamp(), Handle = State#state.handle, {ok, BasePos} = file:position(Handle, State#state.last_position), - NewName = determine_new_filename(State#state.filename), + NewName = determine_new_filename(FN), ok = perform_write_hash_tables(Handle, HashTreeBin, BasePos), ok = write_top_index_table(Handle, BasePos, IndexList), file:close(Handle), - ok = rename_for_read(State#state.filename, NewName), + ok = rename_for_read(FN, NewName), leveled_log:log(cdb03, [NewName]), ets:delete(State#state.hashtree), {NewHandle, Index, LastKey} = @@ -646,13 +681,17 @@ rolling(cast, {delete_pending, ManSQN, Inker}, State) -> {keep_state, State#state{delete_point=ManSQN, inker=Inker, deferred_delete=true}}. -reader({call, From}, {get_kv, Key}, State) -> +reader( + {call, From}, {get_kv, Key}, State = #state{handle = IO}) + when ?IS_DEF(IO) -> Result = - get_withcache(State#state.handle, - Key, - State#state.hash_index, - State#state.binary_mode, - State#state.monitor), + get_withcache( + IO, + Key, + State#state.hash_index, + State#state.binary_mode, + State#state.monitor + ), {keep_state_and_data, [{reply, From, Result}]}; reader({call, From}, {key_check, Key}, State) -> Result = @@ -673,8 +712,11 @@ reader({call, From}, {get_positions, SampleSize, Index, Acc}, State) -> {keep_state_and_data, [{reply, From, lists:sublist(UpdAcc, SampleSize)}]} end; -reader({call, From}, {direct_fetch, PositionList, Info}, State) -> - H = State#state.handle, +reader( + {call, From}, + {direct_fetch, PositionList, Info}, + State = #state{handle = IO}) + when ?IS_DEF(IO) -> FilterFalseKey = fun(Tpl) -> case element(1, Tpl) of @@ -687,20 +729,23 @@ reader({call, From}, {direct_fetch, PositionList, Info}, State) -> case Info of key_only -> - FM = lists:filtermap( + FM = + lists:filtermap( fun(P) -> - FilterFalseKey(extract_key(H, P)) end, - PositionList), + FilterFalseKey(extract_key(IO, P)) + end, + PositionList + ), MapFun = fun(T) -> element(1, T) end, {keep_state_and_data, [{reply, From, lists:map(MapFun, FM)}]}; key_size -> - FilterFun = fun(P) -> FilterFalseKey(extract_key_size(H, P)) end, + FilterFun = fun(P) -> FilterFalseKey(extract_key_size(IO, P)) end, {keep_state_and_data, [{reply, From, lists:filtermap(FilterFun, PositionList)}]}; key_value_check -> BM = State#state.binary_mode, - MapFun = fun(P) -> extract_key_value_check(H, P, BM) end, + MapFun = fun(P) -> extract_key_value_check(IO, P, BM) end, % direct_fetch will occur in batches, so it doesn't make sense to % hibernate the process that is likely to be used again. However, % a significant amount of unused binary references may have @@ -709,12 +754,13 @@ reader({call, From}, {direct_fetch, PositionList, Info}, State) -> garbage_collect(), {keep_state_and_data, []} end; -reader({call, From}, cdb_complete, State) -> - leveled_log:log(cdb05, [State#state.filename, reader, cdb_ccomplete]), - ok = file:close(State#state.handle), - {stop_and_reply, - normal, - [{reply, From, {ok, State#state.filename}}], +reader( + {call, From}, cdb_complete, State = #state{filename = FN, handle = IO}) + when ?IS_DEF(FN), ?IS_DEF(IO) -> + leveled_log:log(cdb05, [FN, reader, cdb_ccomplete]), + ok = file:close(IO), + {stop_and_reply, normal, + [{reply, From, {ok, FN}}], State#state{handle=undefined}}; reader({call, From}, check_hashtable, _State) -> {keep_state_and_data, [{reply, From, true}]}; @@ -731,69 +777,77 @@ reader(cast, clerk_complete, _State) -> {keep_state_and_data, [hibernate]}. -delete_pending({call, From}, {get_kv, Key}, State) -> +delete_pending( + {call, From}, {get_kv, Key}, State = #state{handle = IO}) + when ?IS_DEF(IO) -> Result = - get_withcache(State#state.handle, - Key, - State#state.hash_index, - State#state.binary_mode, - State#state.monitor), + get_withcache( + IO, + Key, + State#state.hash_index, + State#state.binary_mode, + State#state.monitor + ), {keep_state_and_data, [{reply, From, Result}, ?DELETE_TIMEOUT]}; -delete_pending({call, From}, {key_check, Key}, State) -> +delete_pending( + {call, From}, {key_check, Key}, State = #state{handle = IO}) + when ?IS_DEF(IO) -> Result = - get_withcache(State#state.handle, - Key, - State#state.hash_index, - loose_presence, - State#state.binary_mode, - {no_monitor, 0}), + get_withcache( + IO, + Key, + State#state.hash_index, + loose_presence, + State#state.binary_mode, + {no_monitor, 0} + ), {keep_state_and_data, [{reply, From, Result}, ?DELETE_TIMEOUT]}; -delete_pending({call, From}, cdb_close, State) -> - leveled_log:log(cdb05, [State#state.filename, delete_pending, cdb_close]), - close_pendingdelete(State#state.handle, - State#state.filename, - State#state.waste_path), +delete_pending( + {call, From}, cdb_close, State = #state{handle = IO, filename = FN}) + when ?IS_DEF(FN), ?IS_DEF(IO) -> + leveled_log:log(cdb05, [FN, delete_pending, cdb_close]), + close_pendingdelete(IO, FN, State#state.waste_path), {stop_and_reply, normal, [{reply, From, ok}]}; -delete_pending(cast, delete_confirmed, State=#state{delete_point=ManSQN}) -> - leveled_log:log(cdb04, [State#state.filename, ManSQN]), - close_pendingdelete(State#state.handle, - State#state.filename, - State#state.waste_path), - {stop, normal}; -delete_pending(cast, destroy, State) -> - leveled_log:log(cdb05, [State#state.filename, delete_pending, destroy]), - close_pendingdelete(State#state.handle, - State#state.filename, - State#state.waste_path), +delete_pending( + cast, delete_confirmed, State = #state{handle = IO, filename = FN}) + when ?IS_DEF(FN), ?IS_DEF(IO) -> + leveled_log:log(cdb04, [FN, State#state.delete_point]), + close_pendingdelete(IO, FN, State#state.waste_path), {stop, normal}; delete_pending( - timeout, _, State=#state{delete_point=ManSQN}) when ManSQN > 0 -> + cast, destroy, State = #state{handle = IO, filename = FN}) + when ?IS_DEF(FN), ?IS_DEF(IO) -> + leveled_log:log(cdb05, [FN, delete_pending, destroy]), + close_pendingdelete(IO, FN, State#state.waste_path), + {stop, normal}; +delete_pending( + timeout, _, State=#state{delete_point=ManSQN, handle = IO, filename = FN}) + when ManSQN > 0, ?IS_DEF(FN), ?IS_DEF(IO) -> case is_process_alive(State#state.inker) of true -> ok = - leveled_inker:ink_confirmdelete(State#state.inker, - ManSQN, - self()), + leveled_inker:ink_confirmdelete( + State#state.inker, ManSQN, self()), {keep_state_and_data, [?DELETE_TIMEOUT]}; false -> - leveled_log:log(cdb04, [State#state.filename, ManSQN]), - close_pendingdelete(State#state.handle, - State#state.filename, - State#state.waste_path), + leveled_log:log(cdb04, [FN, ManSQN]), + close_pendingdelete(IO, FN, State#state.waste_path), {stop, normal} end. -handle_sync_event({cdb_scan, FilterFun, Acc, StartPos}, From, State) -> - {ok, EndPos0} = file:position(State#state.handle, eof), +handle_sync_event( + {cdb_scan, FilterFun, Acc, StartPos}, From, State = #state{handle = IO}) + when ?IS_DEF(IO) -> + {ok, EndPos0} = file:position(IO, eof), {ok, StartPos0} = case StartPos of undefined -> - file:position(State#state.handle, ?BASE_POSITION); + file:position(IO, ?BASE_POSITION); StartPos -> {ok, StartPos} end, - file:position(State#state.handle, StartPos0), + file:position(IO, StartPos0), MaybeEnd = (check_last_key(State#state.last_key) == empty) or (StartPos0 >= (EndPos0 - ?DWORD_SIZE)), @@ -802,11 +856,13 @@ handle_sync_event({cdb_scan, FilterFun, Acc, StartPos}, From, State) -> true -> {eof, Acc}; false -> - scan_over_file(State#state.handle, - StartPos0, - FilterFun, - Acc, - State#state.last_key) + scan_over_file( + IO, + StartPos0, + FilterFun, + Acc, + State#state.last_key + ) end, % The scan may have created a lot of binary references, clear up the % reference counters for this process here manually. The cdb process @@ -821,20 +877,26 @@ handle_sync_event({cdb_scan, FilterFun, Acc, StartPos}, From, State) -> {keep_state_and_data, []}; handle_sync_event(cdb_lastkey, From, State) -> {keep_state_and_data, [{reply, From, State#state.last_key}]}; -handle_sync_event(cdb_firstkey, From, State) -> - {ok, EOFPos} = file:position(State#state.handle, eof), - FilterFun = fun(Key, _V, _P, _O, _Fun) -> {stop, Key} end, +handle_sync_event( + cdb_firstkey, From, State = #state{handle = IO}) + when ?IS_DEF(IO) -> + {ok, EOFPos} = file:position(IO, eof), FirstKey = case EOFPos of ?BASE_POSITION -> empty; _ -> - file:position(State#state.handle, ?BASE_POSITION), - {_Pos, FirstScanKey} = scan_over_file(State#state.handle, - ?BASE_POSITION, - FilterFun, - empty, - State#state.last_key), + FindFirstKeyFun = + fun(Key, _V, _P, _O, _Fun) -> {stop, Key} end, + file:position(IO, ?BASE_POSITION), + {_Pos, FirstScanKey} = + scan_over_file( + IO, + ?BASE_POSITION, + FindFirstKeyFun, + empty, + State#state.last_key + ), FirstScanKey end, {keep_state_and_data, [{reply, From, FirstKey}]}; @@ -861,14 +923,15 @@ handle_sync_event({put_cachedscore, Score}, From, State) -> {keep_state, State#state{cached_score = {Score,os:timestamp()}}, [{reply, From, ok}]}; -handle_sync_event(cdb_close, From, State) -> - file:close(State#state.handle), +handle_sync_event( + cdb_close, From, _State = #state{handle = IO}) + when ?IS_DEF(IO) -> + file:close(IO), {stop_and_reply, normal, [{reply, From, ok}]}. terminate(_Reason, _StateName, _State) -> ok. - code_change(_OldVsn, StateName, State, _Extra) -> {ok, StateName, State}. @@ -877,7 +940,6 @@ code_change(_OldVsn, StateName, State, _Extra) -> %%% External functions %%%============================================================================ - finished_rolling(CDB) -> RollerFun = fun(Sleep, FinishedRolling) -> @@ -908,9 +970,9 @@ close_pendingdelete(Handle, Filename, WasteFP) -> undefined -> ok = file:delete(Filename); WasteFP -> - Components = filename:split(Filename), - NewName = WasteFP ++ lists:last(Components), - file:rename(Filename, NewName) + FN = filename:basename(Filename), + NewName = filename:join(WasteFP, FN), + ok = file:rename(Filename, NewName) end; false -> % This may happen when there has been a destroy while files are @@ -1179,7 +1241,7 @@ find_lastkey(Handle, IndexCache) -> _ -> {ok, _} = file:position(Handle, LastPosition), {KeyLength, _ValueLength} = read_next_2_integers(Handle), - safe_read_next_key(Handle, KeyLength) + safe_read_next(Handle, KeyLength, key) end. @@ -1239,7 +1301,7 @@ extract_kvpair(_H, [], _K, _BinaryMode) -> extract_kvpair(Handle, [Position|Rest], Key, BinaryMode) -> {ok, _} = file:position(Handle, Position), {KeyLength, ValueLength} = read_next_2_integers(Handle), - case safe_read_next_keybin(Handle, KeyLength) of + case safe_read_next(Handle, KeyLength, keybin) of {Key, KeyBin} -> % If same key as passed in, then found! case checkread_next_value(Handle, ValueLength, KeyBin) of {false, _} -> @@ -1259,12 +1321,12 @@ extract_kvpair(Handle, [Position|Rest], Key, BinaryMode) -> extract_key(Handle, Position) -> {ok, _} = file:position(Handle, Position), {KeyLength, _ValueLength} = read_next_2_integers(Handle), - {safe_read_next_key(Handle, KeyLength)}. + {safe_read_next(Handle, KeyLength, key)}. extract_key_size(Handle, Position) -> {ok, _} = file:position(Handle, Position), {KeyLength, ValueLength} = read_next_2_integers(Handle), - K = safe_read_next_key(Handle, KeyLength), + K = safe_read_next(Handle, KeyLength, key), {K, ValueLength}. extract_key_value_check(Handle, Position, BinaryMode) -> @@ -1279,32 +1341,35 @@ extract_key_value_check(Handle, Position, BinaryMode) -> end. --spec startup_scan_over_file(file:io_device(), file_location()) - -> {file_location(), any()}. +-spec startup_scan_over_file( + file:io_device(), integer()) -> {integer(), {ets:tid(), term()}}. %% @doc %% Scan through the file until there is a failure to crc check an input, and %% at that point return the position and the key dictionary scanned so far startup_scan_over_file(Handle, Position) -> - HashTree = new_hashtree(), - {eof, Output} = scan_over_file(Handle, - Position, - fun startup_filter/5, - {HashTree, empty}, - empty), + Hashtree = new_hashtree(), + FilterFun = startup_filter(Hashtree), + {eof, LastKey} = scan_over_file(Handle, Position, FilterFun, empty, empty), {ok, FinalPos} = file:position(Handle, cur), - {FinalPos, Output}. - + {FinalPos, {Hashtree, LastKey}}. +-spec startup_filter(ets:tid()) -> filter_fun(). %% @doc %% Specific filter to be used at startup to build a hashtree for an incomplete %% cdb file, and returns at the end the hashtree and the final Key seen in the %% journal -startup_filter(Key, _ValueAsBin, Position, {Hashtree, _LastKey}, _ExtractFun) -> - {loop, {put_hashtree(Key, Position, Hashtree), Key}}. +startup_filter(Hashtree) -> + FilterFun = + fun(Key, _ValueAsBin, Position, _LastKey, _ExtractFun) -> + put_hashtree(Key, Position, Hashtree), + {loop, Key} + end, + FilterFun. --spec scan_over_file(file:io_device(), file_location(), - filter_fun(), any(), any()) -> {file_location(), any()}. +-spec scan_over_file + (file:io_device(), integer(), filter_fun(), term(), any()) -> + {file_location(), term()}. %% Scan for key changes - scan over file returning applying FilterFun %% The FilterFun should accept as input: %% - Key, ValueBin, Position, Accumulator, Fun (to extract values from Binary) @@ -1324,13 +1389,14 @@ scan_over_file(Handle, Position, FilterFun, Output, LastKey) -> {ok, Position} = file:position(Handle, {bof, Position}), {eof, Output}; {Key, ValueAsBin, KeyLength, ValueLength} -> - NewPosition = case Key of - LastKey -> - eof; - _ -> - Position + KeyLength + ValueLength - + ?DWORD_SIZE - end, + NewPosition = + case Key of + LastKey -> + eof; + _ -> + Position + KeyLength + ValueLength + + ?DWORD_SIZE + end, case FilterFun(Key, ValueAsBin, Position, @@ -1360,9 +1426,8 @@ check_last_key(empty) -> check_last_key(_LK) -> ok. - --spec saferead_keyvalue(file:io_device()) - -> false|{any(), any(), integer(), integer()}. +-spec saferead_keyvalue( + file:io_device()) -> false|{any(), binary(), integer(), integer()}. %% @doc %% Read the Key/Value at this point, returning {ok, Key, Value} %% catch expected exceptions associated with file corruption (or end) and @@ -1372,11 +1437,11 @@ saferead_keyvalue(Handle) -> eof -> false; {KeyL, ValueL} when is_integer(KeyL), is_integer(ValueL) -> - case safe_read_next_keybin(Handle, KeyL) of + case safe_read_next(Handle, KeyL, keybin) of false -> false; {Key, KeyBin} -> - case safe_read_next_value(Handle, ValueL, KeyBin) of + case safe_read_next(Handle, ValueL, {value, KeyBin}) of false -> false; TrueValue -> @@ -1388,66 +1453,37 @@ saferead_keyvalue(Handle) -> false end. - --spec safe_read_next_key(file:io_device(), integer()) -> false|term(). -%% @doc -%% Return the next key or have false returned if there is some sort of -%% potentially expected error (e.g. due to file truncation). Note that no -%% CRC check has been performed -safe_read_next_key(Handle, Length) -> - ReadFun = fun(Bin) -> binary_to_term(Bin) end, - safe_read_next(Handle, Length, ReadFun). - --spec safe_read_next_keybin(file:io_device(), integer()) - -> false|{term(), binary()}. -%% @doc -%% Return the next key or have false returned if there is some sort of -%% potentially expected error (e.g. due to file truncation). Note that no -%% CRC check has been performed -%% Returns both the Key and the Binary version, the binary version being -%% required for the CRC checking after the value fetch (see -%% safe_read_next_value/3) -safe_read_next_keybin(Handle, Length) -> - ReadFun = fun(Bin) -> {binary_to_term(Bin), Bin} end, - safe_read_next(Handle, Length, ReadFun). - --spec safe_read_next_value(file:io_device(), integer(), binary()) - -> binary()|false. -safe_read_next_value(Handle, Length, KeyBin) -> - ReadFun = fun(VBin) -> crccheck(VBin, KeyBin) end, - safe_read_next(Handle, Length, ReadFun). - --type read_output() :: {term(), binary()}|binary()|term()|false. --type read_fun() :: fun((binary()) -> read_output()). - --spec safe_read_next(file:io_device(), integer(), read_fun()) - -> read_output(). +-spec safe_read_next + (file:io_device(), integer(), key) -> false|term(); + (file:io_device(), integer(), keybin) -> false|{term(), binary()}; + (file:io_device(), integer(), {value, binary()}) -> false|binary(). %% @doc %% Read the next item of length Length %% Previously catching error:badarg was sufficient to capture errors of %% corruption, but on some OS versions may need to catch error:einval as well -safe_read_next(Handle, Length, ReadFun) -> +safe_read_next(Handle, Length, ReadType) -> + ReadFun = + case ReadType of + key -> + fun(Bin) -> binary_to_term(Bin) end; + keybin -> + fun(KBin) -> {binary_to_term(KBin), KBin} end; + {value, KeyBin} -> + fun(VBin) -> crccheck(VBin, KeyBin) end + end, try - loose_read(Handle, Length, ReadFun) + case file:read(Handle, Length) of + eof -> + false; + {ok, Result} -> + ReadFun(Result) + end catch error:ReadError -> leveled_log:log(cdb20, [ReadError, Length]), false end. --spec loose_read(file:io_device(), integer(), read_fun()) -> read_output(). -%% @doc -%% Read with minimal error handling (only eof) - to be wrapped in -%% safe_read_next/3 to catch exceptions. -loose_read(Handle, Length, ReadFun) -> - case file:read(Handle, Length) of - eof -> - false; - {ok, Result} -> - ReadFun(Result) - end. - - -spec crccheck(binary()|bitstring(), binary()) -> any(). %% @doc %% CRC chaeck the value which should be a binary, where the first four bytes @@ -1472,8 +1508,9 @@ crccheck(_V, _KB) -> calc_crc(KeyBin, Value) -> erlang:crc32(<>). --spec checkread_next_value(file:io_device(), integer(), binary()) - -> {boolean(), binary()|crc_wonky}. +-spec checkread_next_value + (file:io_device(), integer(), binary()) -> + {true, binary()}|{false, crc_wonky}. %% @doc %% Read next string where the string has a CRC prepended - stripping the crc %% and checking if requested @@ -1578,12 +1615,14 @@ search_hash_table(Handle, leveled_monitor:timing(), leveled_monitor:timing(), pos_integer()) -> ok. -maybelog_get_timing(_Monitor, no_timing, no_timing, _CC) -> - ok; -maybelog_get_timing({Pid, _StatsFreq}, IndexTime, ReadTime, CycleCount) -> +maybelog_get_timing( + {Pid, _StatsFreq}, IndexTime, ReadTime, CycleCount) + when is_pid(Pid), is_integer(IndexTime), is_integer(ReadTime) -> leveled_monitor:add_stat( - Pid, {cdb_get_update, CycleCount, IndexTime, ReadTime}). - + Pid, {cdb_get_update, CycleCount, IndexTime, ReadTime}); +maybelog_get_timing(_Monitor, _IndexTime, _ReadTime, _CC) -> + ok. + %% Write the actual hashtables at the bottom of the file. Each hash table %% entry is a doubleword in length. The first word is the hash value @@ -1916,7 +1955,7 @@ dump(FileName) -> {ok, _} = file:position(Handle, {bof, ?BASE_POSITION}), Fn1 = fun(_I, Acc) -> {KL, VL} = read_next_2_integers(Handle), - {Key, KB} = safe_read_next_keybin(Handle, KL), + {Key, KB} = safe_read_next(Handle, KL, keybin), Value = case checkread_next_value(Handle, VL, KB) of {true, V0} -> @@ -2632,7 +2671,7 @@ safe_read_test() -> {ok, HandleK} = file:open(TestFN, ?WRITE_OPS), ok = file:pwrite(HandleK, 0, BinToWrite), {ok, _} = file:position(HandleK, 8 + KeyL + ValueL), - ?assertMatch(false, safe_read_next_key(HandleK, KeyL)), + ?assertMatch(false, safe_read_next(HandleK, KeyL, key)), ok = file:close(HandleK), WrongKeyL = endian_flip(KeyL + ValueL), @@ -2749,11 +2788,15 @@ getpositions_sample_test() -> ok = cdb_close(P2), file:delete(F2). - nonsense_coverage_test() -> - ?assertMatch({ok, reader, #state{}}, code_change(nonsense, - reader, - #state{}, - nonsense)). + ?assertMatch( + {ok, reader, #state{}}, + code_change( + nonsense, + reader, + #state{max_count=1, max_size=100}, + nonsense + ) + ). -endif. diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index bb69d81..44e65ae 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -10,6 +10,12 @@ -include("leveled.hrl"). +-eqwalizer({nowarn_function, convert_to_ledgerv/5}). + +-ifdef(TEST). +-export([convert_to_ledgerv/5]). +-endif. + -export([ inker_reload_strategy/1, strip_to_seqonly/1, @@ -21,8 +27,10 @@ endkey_passed/2, key_dominates/2, maybe_reap_expiredkey/2, - to_ledgerkey/3, - to_ledgerkey/5, + to_objectkey/3, + to_objectkey/5, + to_querykey/3, + to_querykey/5, from_ledgerkey/1, from_ledgerkey/2, isvalid_ledgerkey/1, @@ -33,11 +41,11 @@ from_journalkey/1, revert_to_keydeltas/2, is_full_journalentry/1, - split_inkvalue/1, check_forinkertype/2, get_tagstrategy/2, maybe_compress/2, create_value_for_journal/3, + revert_value_from_journal/1, generate_ledgerkv/5, get_size/2, get_keyandobjhash/2, @@ -54,8 +62,9 @@ -type tag() :: leveled_head:object_tag()|?IDX_TAG|?HEAD_TAG|atom(). --type key() :: - binary()|string()|{binary(), binary()}. +-type single_key() :: binary(). +-type tuple_key() :: {single_key(), single_key()}. +-type key() :: single_key()|tuple_key(). % Keys SHOULD be binary() % string() support is a legacy of old tests -type sqn() :: @@ -75,8 +84,15 @@ -type ledger_status() :: tomb|{active, non_neg_integer()|infinity}. +-type primary_key() :: + {leveled_head:object_tag(), key(), single_key(), single_key()|null}. + % Primary key for an object +-type object_key() :: + {tag(), key(), key(), single_key()|null}. +-type query_key() :: + {tag(), key()|null, key()|null, single_key()|null}|all. -type ledger_key() :: - {tag(), any(), any(), any()}|all. + object_key()|query_key(). -type slimmed_key() :: {binary(), binary()|null}|binary()|null|all. -type ledger_value() :: @@ -86,7 +102,7 @@ -type ledger_value_v2() :: {sqn(), ledger_status(), segment_hash(), metadata(), last_moddate()}. -type ledger_kv() :: - {ledger_key(), ledger_value()}. + {object_key(), ledger_value()}. -type compaction_method() :: retain|recovr|recalc. -type compaction_strategy() :: @@ -94,14 +110,14 @@ -type journal_key_tag() :: ?INKT_STND|?INKT_TOMB|?INKT_MPUT|?INKT_KEYD. -type journal_key() :: - {sqn(), journal_key_tag(), ledger_key()}. + {sqn(), journal_key_tag(), primary_key()}. -type journal_ref() :: - {ledger_key(), sqn()}. + {object_key(), sqn()}. -type object_spec_v0() :: - {add|remove, key(), key(), key()|null, any()}. + {add|remove, key(), single_key(), single_key()|null, metadata()}. -type object_spec_v1() :: - {add|remove, v1, key(), key(), key()|null, - list(erlang:timestamp())|undefined, any()}. + {add|remove, v1, key(), single_key(), single_key()|null, + list(erlang:timestamp())|undefined, metadata()}. -type object_spec() :: object_spec_v0()|object_spec_v1(). -type compression_method() :: @@ -135,10 +151,14 @@ -export_type([tag/0, key/0, + single_key/0, sqn/0, object_spec/0, segment_hash/0, ledger_status/0, + primary_key/0, + object_key/0, + query_key/0, ledger_key/0, ledger_value/0, ledger_kv/0, @@ -186,30 +206,26 @@ segment_hash(KeyTuple) when is_tuple(KeyTuple) -> segment_hash(BinKey). -headkey_to_canonicalbinary({?HEAD_TAG, Bucket, Key, SubK}) - when is_binary(Bucket), is_binary(Key), is_binary(SubK) -> +headkey_to_canonicalbinary({ + ?HEAD_TAG, Bucket, Key, SubK}) + when is_binary(Bucket), is_binary(Key), is_binary(SubK) -> <>; -headkey_to_canonicalbinary({?HEAD_TAG, Bucket, Key, null}) - when is_binary(Bucket), is_binary(Key) -> +headkey_to_canonicalbinary( + {?HEAD_TAG, Bucket, Key, null}) + when is_binary(Bucket), is_binary(Key) -> <>; -headkey_to_canonicalbinary({?HEAD_TAG, {BucketType, Bucket}, Key, SubKey}) - when is_binary(BucketType), is_binary(Bucket) -> - headkey_to_canonicalbinary({?HEAD_TAG, - <>, - Key, - SubKey}); -headkey_to_canonicalbinary(Key) when element(1, Key) == ?HEAD_TAG -> - % In unit tests head specs can have non-binary keys, so handle - % this through hashing the whole key - leveled_util:t2b(Key). - +headkey_to_canonicalbinary( + {?HEAD_TAG, {BucketType, Bucket}, Key, SubKey}) + when is_binary(BucketType), is_binary(Bucket) -> + headkey_to_canonicalbinary( + {?HEAD_TAG, <>, Key, SubKey}). -spec to_lookup(ledger_key()) -> maybe_lookup(). %% @doc %% Should it be possible to lookup a key in the merge tree. This is not true %% For keys that should only be read through range queries. Direct lookup %% keys will have presence in bloom filters and other lookup accelerators. -to_lookup(Key) -> +to_lookup(Key) when is_tuple(Key) -> case element(1, Key) of ?IDX_TAG -> no_lookup; @@ -235,12 +251,12 @@ strip_to_keyseqonly({LK, V}) -> {LK, element(1, V)}. -spec strip_to_indexdetails(ledger_kv()) -> {integer(), segment_hash(), last_moddate()}. -strip_to_indexdetails({_, V}) when tuple_size(V) == 4 -> +strip_to_indexdetails({_, {SQN, _, SegmentHash, _}}) -> % A v1 value - {element(1, V), element(3, V), undefined}; -strip_to_indexdetails({_, V}) when tuple_size(V) > 4 -> + {SQN, SegmentHash, undefined}; +strip_to_indexdetails({_, {SQN, _, SegmentHash, _, LMD}}) -> % A v2 value should have a fith element - Last Modified Date - {element(1, V), element(3, V), element(5, V)}. + {SQN, SegmentHash, LMD}. -spec striphead_to_v1details(ledger_value()) -> ledger_value(). striphead_to_v1details(V) -> @@ -292,18 +308,25 @@ maybe_accumulate( maybe_accumulate(T, Acc, Count, Filter, AccFun). -spec accumulate_index( - {boolean(), undefined|leveled_runner:mp()}, leveled_runner:acc_fun()) - -> any(). + {boolean(), undefined|leveled_runner:mp()}, + leveled_runner:fold_keys_fun()) + -> leveled_penciller:pclacc_fun(). accumulate_index({false, undefined}, FoldKeysFun) -> - fun({?IDX_TAG, Bucket, _IndexInfo, ObjKey}, _Value, Acc) -> + fun( + {?IDX_TAG, Bucket, _IndexInfo, ObjKey}, _Value, Acc) + when ObjKey =/= null -> FoldKeysFun(Bucket, ObjKey, Acc) end; accumulate_index({true, undefined}, FoldKeysFun) -> - fun({?IDX_TAG, Bucket, {_IdxFld, IdxValue}, ObjKey}, _Value, Acc) -> + fun( + {?IDX_TAG, Bucket, {_IdxFld, IdxValue}, ObjKey}, _Value, Acc) + when IdxValue =/= null, ObjKey =/= null -> FoldKeysFun(Bucket, {IdxValue, ObjKey}, Acc) end; accumulate_index({AddTerm, TermRegex}, FoldKeysFun) -> - fun({?IDX_TAG, Bucket, {_IdxFld, IdxValue}, ObjKey}, _Value, Acc) -> + fun( + {?IDX_TAG, Bucket, {_IdxFld, IdxValue}, ObjKey}, _Value, Acc) + when IdxValue =/= null, ObjKey =/= null -> case re:run(IdxValue, TermRegex) of nomatch -> Acc; @@ -343,17 +366,17 @@ maybe_reap(_, _) -> false. -spec count_tombs( - list(ledger_kv()), non_neg_integer()|not_counted) -> - non_neg_integer()|not_counted. -count_tombs(_List, not_counted) -> - not_counted; + list(ledger_kv()), non_neg_integer()) -> + non_neg_integer(). count_tombs([], Count) -> Count; -count_tombs([{_K, V}|T], Count) when element(2, V) == tomb -> - count_tombs(T, Count + 1); -count_tombs([_KV|T], Count) -> - count_tombs(T, Count). - +count_tombs([{_K, V}|T], Count) when is_tuple(V) -> + case element(2, V) of + tomb -> + count_tombs(T, Count + 1); + _ -> + count_tombs(T, Count) + end. -spec from_ledgerkey(atom(), tuple()) -> false|tuple(). %% @doc @@ -375,18 +398,37 @@ from_ledgerkey({?HEAD_TAG, Bucket, Key, SubKey}) -> from_ledgerkey({_Tag, Bucket, Key, _SubKey}) -> {Bucket, Key}. --spec to_ledgerkey(any(), any(), tag(), any(), any()) -> ledger_key(). +-spec to_objectkey( + key(), single_key(), tag(), binary(), binary()) -> object_key(). %% @doc %% Convert something into a ledger key -to_ledgerkey(Bucket, Key, Tag, Field, Value) when Tag == ?IDX_TAG -> +to_objectkey(Bucket, Key, Tag, Field, Value) when Tag == ?IDX_TAG -> {?IDX_TAG, Bucket, {Field, Value}, Key}. --spec to_ledgerkey(any(), any(), tag()) -> ledger_key(). +-if(?OTP_RELEASE >= 26). +-spec to_objectkey + (key(), single_key(), leveled_head:object_tag()) -> primary_key(); + (key(), key(), tag()) -> object_key(). +-else. +-spec to_objectkey(key(), key()|single_key(), tag()) -> object_key(). +-endif. %% @doc %% Convert something into a ledger key -to_ledgerkey(Bucket, {Key, SubKey}, ?HEAD_TAG) -> +to_objectkey(Bucket, {Key, SubKey}, ?HEAD_TAG) -> {?HEAD_TAG, Bucket, Key, SubKey}; -to_ledgerkey(Bucket, Key, Tag) -> +to_objectkey(Bucket, Key, Tag) -> + {Tag, Bucket, Key, null}. + +-spec to_querykey( + key(), single_key()|null, tag(), binary(), binary()) + -> query_key(). +to_querykey(Bucket, Key, Tag, Field, Value) when Tag == ?IDX_TAG -> + {?IDX_TAG, Bucket, {Field, Value}, Key}. + +-spec to_querykey(key()|null, key()|null, tag()) -> query_key(). +%% @doc +%% Convert something into a ledger query key +to_querykey(Bucket, Key, Tag) -> {Tag, Bucket, Key, null}. %% No spec - due to tests @@ -399,8 +441,8 @@ isvalid_ledgerkey(_LK) -> false. -spec endkey_passed( - ledger_key()|slimmed_key(), - ledger_key()|slimmed_key()) -> boolean(). + query_key()|slimmed_key(), + object_key()|slimmed_key()) -> boolean(). %% @doc %% Compare a key against a query key, only comparing elements that are non-null %% in the Query key. @@ -480,14 +522,19 @@ get_tagstrategy(Tag, Strategy) -> %%% Manipulate Journal Key and Value %%%============================================================================ --spec to_inkerkey(ledger_key(), non_neg_integer()) -> journal_key(). +-spec to_inkerkey(primary_key(), non_neg_integer()) -> journal_key(). %% @doc %% convertion from ledger_key to journal_key to allow for the key to be fetched to_inkerkey(LedgerKey, SQN) -> {SQN, ?INKT_STND, LedgerKey}. --spec to_inkerkv(ledger_key(), non_neg_integer(), any(), journal_keychanges(), - compression_method(), boolean()) -> {journal_key(), any()}. +-spec to_inkerkv( + primary_key(), + non_neg_integer(), + any(), + journal_keychanges(), + compression_method(), boolean()) + -> {journal_key(), binary()}. %% @doc %% Convert to the correct format of a Journal key and value to_inkerkv(LedgerKey, SQN, Object, KeyChanges, PressMethod, Compress) -> @@ -496,7 +543,7 @@ to_inkerkv(LedgerKey, SQN, Object, KeyChanges, PressMethod, Compress) -> create_value_for_journal({Object, KeyChanges}, Compress, PressMethod), {{SQN, InkerType, LedgerKey}, Value}. --spec revert_to_keydeltas(journal_key(), any()) -> {journal_key(), any()}. +-spec revert_to_keydeltas(journal_key(), binary()) -> {journal_key(), any()}. %% @doc %% If we wish to retain key deltas when an object in the Journal has been %% replaced - then this converts a Journal Key and Value into one which has no @@ -575,7 +622,7 @@ serialise_object(Object, false, _Method) -> serialise_object(Object, true, _Method) -> term_to_binary(Object, [compressed]). --spec revert_value_from_journal(binary()) -> {any(), journal_keychanges()}. +-spec revert_value_from_journal(binary()) -> {dynamic(), journal_keychanges()}. %% @doc %% Revert the object back to its deserialised state, along with the list of %% key changes associated with the change @@ -661,10 +708,6 @@ decode_valuetype(TypeInt) -> from_journalkey({SQN, _Type, LedgerKey}) -> {SQN, LedgerKey}. - -split_inkvalue(VBin) when is_binary(VBin) -> - revert_value_from_journal(VBin). - check_forinkertype(_LedgerKey, delete) -> ?INKT_TOMB; check_forinkertype(_LedgerKey, head_only) -> @@ -709,7 +752,7 @@ idx_indexspecs(IndexSpecs, Bucket, Key, SQN, TTL) -> gen_indexspec(Bucket, Key, IdxOp, IdxField, IdxTerm, SQN, TTL) -> Status = set_status(IdxOp, TTL), - {to_ledgerkey(Bucket, Key, ?IDX_TAG, IdxField, IdxTerm), + {to_objectkey(Bucket, Key, ?IDX_TAG, IdxField, IdxTerm), {SQN, Status, no_lookup, null}}. -spec gen_headspec(object_spec(), integer(), integer()|infinity) -> ledger_kv(). @@ -717,22 +760,30 @@ gen_indexspec(Bucket, Key, IdxOp, IdxField, IdxTerm, SQN, TTL) -> %% Take an object_spec as passed in a book_mput, and convert it into to a %% valid ledger key and value. Supports different shaped tuples for different %% versions of the object_spec -gen_headspec({IdxOp, v1, Bucket, Key, SubKey, LMD, Value}, SQN, TTL) -> +gen_headspec( + {IdxOp, v1, Bucket, Key, SubKey, LMD, Value}, SQN, TTL) + when is_binary(Key) -> % v1 object spec Status = set_status(IdxOp, TTL), - K = to_ledgerkey(Bucket, {Key, SubKey}, ?HEAD_TAG), + K = + case SubKey of + null -> + to_objectkey(Bucket, Key, ?HEAD_TAG); + SKB when is_binary(SKB) -> + to_objectkey(Bucket, {Key, SKB}, ?HEAD_TAG) + end, {K, {SQN, Status, segment_hash(K), Value, get_last_lastmodification(LMD)}}; -gen_headspec({IdxOp, Bucket, Key, SubKey, Value}, SQN, TTL) -> - % v0 object spec - Status = set_status(IdxOp, TTL), - K = to_ledgerkey(Bucket, {Key, SubKey}, ?HEAD_TAG), - {K, {SQN, Status, segment_hash(K), Value, undefined}}. +gen_headspec( + {IdxOp, Bucket, Key, SubKey, Value}, SQN, TTL) + when is_binary(Key) -> + gen_headspec({IdxOp, v1, Bucket, Key, SubKey, undefined, Value}, SQN, TTL). --spec return_proxy(leveled_head:object_tag()|leveled_head:headonly_tag(), - leveled_head:object_metadata(), - pid(), journal_ref()) - -> proxy_objectbin()|leveled_head:object_metadata(). +-spec return_proxy + (leveled_head:headonly_tag(), leveled_head:object_metadata(), null, journal_ref()) + -> leveled_head:object_metadata(); + (leveled_head:object_tag(), leveled_head:object_metadata(), pid(), journal_ref()) + -> proxy_objectbin(). %% @doc %% If the object has a value, return the metadata and a proxy through which %% the applictaion or runner can access the value. If it is a ?HEAD_TAG @@ -751,6 +802,9 @@ return_proxy(Tag, ObjMetadata, InkerClone, JournalRef) -> InkerClone, JournalRef}}). +-spec set_status( + add|remove, non_neg_integer()|infinity) -> + tomb|{active, non_neg_integer()|infinity}. set_status(add, TTL) -> {active, TTL}; set_status(remove, _TTL) -> @@ -758,10 +812,18 @@ set_status(remove, _TTL) -> tomb. -spec generate_ledgerkv( - tuple(), integer(), any(), integer(), tuple()|infinity) -> - {any(), any(), any(), - {{integer(), integer()}|no_lookup, integer()}, - list()}. + primary_key(), + integer(), + dynamic(), + integer(), + non_neg_integer()|infinity) -> + { + key(), + single_key(), + ledger_value_v2(), + {segment_hash(), non_neg_integer()|null}, + list(erlang:timestamp()) + }. %% @doc %% Function to extract from an object the information necessary to populate %% the Penciller's ledger. @@ -776,24 +838,22 @@ set_status(remove, _TTL) -> %% siblings) generate_ledgerkv(PrimaryKey, SQN, Obj, Size, TS) -> {Tag, Bucket, Key, _} = PrimaryKey, - Status = case Obj of - delete -> - tomb; - _ -> - {active, TS} - end, + Status = case Obj of delete -> tomb; _ -> {active, TS} end, Hash = segment_hash(PrimaryKey), {MD, LastMods} = leveled_head:extract_metadata(Tag, Size, Obj), ObjHash = leveled_head:get_hash(Tag, MD), - Value = {SQN, - Status, - Hash, - MD, - get_last_lastmodification(LastMods)}, + Value = + { + SQN, + Status, + Hash, + MD, + get_last_lastmodification(LastMods) + }, {Bucket, Key, Value, {Hash, ObjHash}, LastMods}. --spec get_last_lastmodification(list(erlang:timestamp())|undefined) - -> pos_integer()|undefined. +-spec get_last_lastmodification( + list(erlang:timestamp())|undefined) -> pos_integer()|undefined. %% @doc %% Get the highest of the last modifications measured in seconds. This will be %% stored as 4 bytes (unsigned) so will last for another 80 + years @@ -830,10 +890,10 @@ get_keyandobjhash(LK, Value) -> %% Get the next key to iterate from a given point next_key(Key) when is_binary(Key) -> <>; -next_key(Key) when is_list(Key) -> - Key ++ [0]; next_key({Type, Bucket}) when is_binary(Type), is_binary(Bucket) -> - {Type, next_key(Bucket)}. + UpdBucket = next_key(Bucket), + true = is_binary(UpdBucket), + {Type, UpdBucket}. %%%============================================================================ @@ -844,6 +904,17 @@ next_key({Type, Bucket}) when is_binary(Type), is_binary(Bucket) -> -include_lib("eunit/include/eunit.hrl"). +-spec convert_to_ledgerv( + leveled_codec:ledger_key(), + integer(), + any(), + integer(), + non_neg_integer()|infinity) -> leveled_codec:ledger_value(). +convert_to_ledgerv(PK, SQN, Obj, Size, TS) -> + {_B, _K, MV, _H, _LMs} = + leveled_codec:generate_ledgerkv(PK, SQN, Obj, Size, TS), + MV. + valid_ledgerkey_test() -> UserDefTag = {user_defined, <<"B">>, <<"K">>, null}, ?assertMatch(true, isvalid_ledgerkey(UserDefTag)), @@ -870,8 +941,8 @@ indexspecs_test() -> endkey_passed_test() -> TestKey = {i, null, null, null}, - K1 = {i, 123, {"a", "b"}, <<>>}, - K2 = {o, 123, {"a", "b"}, <<>>}, + K1 = {i, <<"123">>, {<<"a">>, <<"b">>}, <<>>}, + K2 = {o, <<"123">>, {<<"a">>, <<"b">>}, <<>>}, ?assertMatch(false, endkey_passed(TestKey, K1)), ?assertMatch(true, endkey_passed(TestKey, K2)). @@ -881,7 +952,7 @@ endkey_passed_test() -> %% Maybe 5 microseconds per hash hashperf_test() -> - OL = lists:map(fun(_X) -> leveled_rand:rand_bytes(8192) end, lists:seq(1, 1000)), + OL = lists:map(fun(_X) -> crypto:strong_rand_bytes(8192) end, lists:seq(1, 1000)), SW = os:timestamp(), _HL = lists:map(fun(Obj) -> erlang:phash2(Obj) end, OL), io:format(user, "1000 object hashes in ~w microseconds~n", @@ -899,8 +970,8 @@ head_segment_compare_test() -> headspec_v0v1_test() -> % A v0 object spec generates the same outcome as a v1 object spec with the % last modified date undefined - V1 = {add, v1, <<"B">>, <<"K">>, <<"SK">>, undefined, <<"V">>}, - V0 = {add, <<"B">>, <<"K">>, <<"SK">>, <<"V">>}, + V1 = {add, v1, <<"B">>, <<"K">>, <<"SK">>, undefined, {<<"V">>}}, + V0 = {add, <<"B">>, <<"K">>, <<"SK">>, {<<"V">>}}, TTL = infinity, ?assertMatch(true, gen_headspec(V0, 1, TTL) == gen_headspec(V1, 1, TTL)). diff --git a/src/leveled_ebloom.erl b/src/leveled_ebloom.erl index b74a582..21a8f40 100644 --- a/src/leveled_ebloom.erl +++ b/src/leveled_ebloom.erl @@ -91,8 +91,8 @@ check_hash({_SegHash, Hash}, BloomBin) when is_binary(BloomBin)-> list(leveled_codec:segment_hash()), tuple(), slot_count()) -> tuple(). map_hashes([], HashListTuple, _SlotCount) -> HashListTuple; -map_hashes([Hash|Rest], HashListTuple, SlotCount) -> - {Slot, [H0, H1]} = split_hash(element(2, Hash), SlotCount), +map_hashes([{_SH, EH}|Rest], HashListTuple, SlotCount) -> + {Slot, [H0, H1]} = split_hash(EH, SlotCount), SlotHL = element(Slot + 1, HashListTuple), map_hashes( Rest, @@ -174,11 +174,15 @@ generate_orderedkeys(Seqn, Count, Acc, BucketLow, BucketHigh) -> BucketExt = io_lib:format("K~4..0B", [BucketLow + BNumber]), KeyExt = - io_lib:format("K~8..0B", [Seqn * 100 + leveled_rand:uniform(100)]), - LK = leveled_codec:to_ledgerkey("Bucket" ++ BucketExt, "Key" ++ KeyExt, o), - Chunk = leveled_rand:rand_bytes(16), - {_B, _K, MV, _H, _LMs} = - leveled_codec:generate_ledgerkv(LK, Seqn, Chunk, 64, infinity), + io_lib:format("K~8..0B", [Seqn * 100 + rand:uniform(100)]), + LK = + leveled_codec:to_objectkey( + list_to_binary("Bucket" ++ BucketExt), + list_to_binary("Key" ++ KeyExt), + o + ), + Chunk = crypto:strong_rand_bytes(16), + MV = leveled_codec:convert_to_ledgerv(LK, Seqn, Chunk, 64, infinity), generate_orderedkeys( Seqn + 1, Count - 1, [{LK, MV}|Acc], BucketLow, BucketHigh). @@ -236,7 +240,7 @@ test_bloom(N, Runs) -> fun(HashList) -> HitOrMissFun = fun (Entry, {HitL, MissL}) -> - case leveled_rand:uniform() < 0.5 of + case rand:uniform() < 0.5 of true -> {[Entry|HitL], MissL}; false -> diff --git a/src/leveled_head.erl b/src/leveled_head.erl index b4bd6a3..64c0e19 100644 --- a/src/leveled_head.erl +++ b/src/leveled_head.erl @@ -82,7 +82,7 @@ -type appdefinable_headfun() :: fun((object_tag(), object_metadata()) -> head()). -type appdefinable_metadatafun() :: - fun(({leveled_codec:tag(), non_neg_integer(), any()}) -> + fun((leveled_codec:tag(), non_neg_integer(), binary()|delete) -> {object_metadata(), list(erlang:timestamp())}). -type appdefinable_indexspecsfun() :: fun((object_tag(), object_metadata(), object_metadata()|not_present) -> @@ -117,11 +117,13 @@ %% @doc %% Convert a key to a binary in a consistent way for the tag. The binary will %% then be used to create the hash -key_to_canonicalbinary({?RIAK_TAG, Bucket, Key, null}) - when is_binary(Bucket), is_binary(Key) -> +key_to_canonicalbinary( + {?RIAK_TAG, Bucket, Key, null}) + when is_binary(Bucket), is_binary(Key) -> <>; -key_to_canonicalbinary({?RIAK_TAG, {BucketType, Bucket}, Key, SubKey}) - when is_binary(BucketType), is_binary(Bucket) -> +key_to_canonicalbinary( + {?RIAK_TAG, {BucketType, Bucket}, Key, SubKey}) + when is_binary(BucketType), is_binary(Bucket) -> key_to_canonicalbinary({?RIAK_TAG, <>, Key, @@ -130,9 +132,11 @@ key_to_canonicalbinary(Key) when element(1, Key) == ?STD_TAG -> default_key_to_canonicalbinary(Key); key_to_canonicalbinary(Key) -> OverrideFun = - get_appdefined_function(key_to_canonicalbinary, - fun default_key_to_canonicalbinary/1, - 1), + get_appdefined_function( + key_to_canonicalbinary, + fun default_key_to_canonicalbinary/1, + 1 + ), OverrideFun(Key). default_key_to_canonicalbinary(Key) -> @@ -162,7 +166,7 @@ default_build_head(_Tag, Metadata) -> Metadata. --spec extract_metadata(object_tag(), non_neg_integer(), any()) +-spec extract_metadata(object_tag(), non_neg_integer(), binary()) -> {object_metadata(), list(erlang:timestamp())}. %% @doc %% Take the inbound object and extract from it the metadata to be stored within @@ -239,9 +243,8 @@ defined_objecttags() -> [?STD_TAG, ?RIAK_TAG]. --spec default_reload_strategy(object_tag()) - -> {object_tag(), - leveled_codec:compaction_method()}. +-spec default_reload_strategy( + object_tag()) -> {object_tag(), leveled_codec:compaction_method()}. %% @doc %% State the compaction_method to be used when reloading the Ledger from the %% journal for each object tag. Note, no compaction strategy required for @@ -249,25 +252,24 @@ defined_objecttags() -> default_reload_strategy(Tag) -> {Tag, retain}. - --spec get_size(object_tag()|headonly_tag(), object_metadata()) - -> non_neg_integer(). +-spec get_size( + object_tag()|headonly_tag(), object_metadata()) -> non_neg_integer(). %% @doc %% Fetch the size from the metadata -get_size(?RIAK_TAG, RiakObjectMetadata) -> - element(4, RiakObjectMetadata); -get_size(_Tag, ObjectMetadata) -> - element(2, ObjectMetadata). +get_size(?RIAK_TAG, {_, _, _, Size}) -> + Size; +get_size(_Tag, {_, Size, _}) -> + Size. --spec get_hash(object_tag()|headonly_tag(), object_metadata()) - -> non_neg_integer(). +-spec get_hash( + object_tag()|headonly_tag(), object_metadata()) -> non_neg_integer()|null. %% @doc %% Fetch the hash from the metadata -get_hash(?RIAK_TAG, RiakObjectMetadata) -> - element(3, RiakObjectMetadata); -get_hash(_Tag, ObjectMetadata) -> - element(1, ObjectMetadata). +get_hash(?RIAK_TAG, {_, _, Hash, _}) -> + Hash; +get_hash(_Tag, {Hash, _, _}) -> + Hash. -spec standard_hash(any()) -> non_neg_integer(). %% @doc @@ -280,9 +282,15 @@ standard_hash(Obj) -> %%% Handling Override Functions %%%============================================================================ --spec get_appdefined_function( - appdefinable_function(), appdefinable_function_fun(), non_neg_integer()) -> - appdefinable_function_fun(). +-spec get_appdefined_function + (key_to_canonicalbinary, appdefinable_keyfun(), 1) -> + appdefinable_keyfun(); + (build_head, appdefinable_headfun(), 2) -> + appdefinable_headfun(); + (extract_metadata, appdefinable_metadatafun(), 3) -> + appdefinable_metadatafun(); + (diff_indexspecs, appdefinable_indexspecsfun(), 3) -> + appdefinable_indexspecsfun(). %% @doc %% If a keylist of [{function_name, fun()}] has been set as an environment %% variable for a tag, then this FunctionName can be used instead of the @@ -300,8 +308,8 @@ get_appdefined_function(FunctionName, DefaultFun, RequiredArity) -> %%%============================================================================ --spec riak_extract_metadata(binary()|delete, non_neg_integer()) -> - {riak_metadata(), list()}. +-spec riak_extract_metadata( + binary()|delete, non_neg_integer()) -> {riak_metadata(), list()}. %% @doc %% Riak extract metadata should extract a metadata object which is a %% five-tuple of: diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index e241348..a2f9d32 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -114,10 +114,10 @@ scoring_state :: scoring_state()|undefined, score_onein = 1 :: pos_integer()}). --record(candidate, {low_sqn :: integer() | undefined, - filename :: string() | undefined, - journal :: pid() | undefined, - compaction_perc :: float() | undefined}). +-record(candidate, {low_sqn :: integer(), + filename :: string(), + journal :: pid(), + compaction_perc :: float()}). -record(scoring_state, {filter_fun :: leveled_inker:filterfun(), filter_server :: leveled_inker:filterserver(), @@ -158,7 +158,13 @@ %% @doc %% Generate a new clerk clerk_new(InkerClerkOpts) -> - gen_server:start_link(?MODULE, [leveled_log:get_opts(), InkerClerkOpts], []). + {ok, Clerk} = + gen_server:start_link( + ?MODULE, + [leveled_log:get_opts(), InkerClerkOpts], + [] + ), + {ok, Clerk}. -spec clerk_compact(pid(), pid(), @@ -310,60 +316,71 @@ handle_cast({compact, Checker, InitiateFun, CloseFun, FilterFun, Manifest0}, end, ok = clerk_scorefilelist(self(), lists:filter(NotRollingFun, Manifest)), ScoringState = - #scoring_state{filter_fun = FilterFun, - filter_server = FilterServer, - max_sqn = MaxSQN, - close_fun = CloseFun, - start_time = SW}, + #scoring_state{ + filter_fun = FilterFun, + filter_server = FilterServer, + max_sqn = MaxSQN, + close_fun = CloseFun, + start_time = SW + }, {noreply, State#state{scored_files = [], scoring_state = ScoringState}}; -handle_cast({score_filelist, [Entry|Tail]}, State) -> +handle_cast( + {score_filelist, [Entry|Tail]}, + State = #state{scoring_state = ScoringState}) + when ?IS_DEF(ScoringState) -> Candidates = State#state.scored_files, {LowSQN, FN, JournalP, _LK} = Entry, - ScoringState = State#state.scoring_state, CpctPerc = case {leveled_cdb:cdb_getcachedscore(JournalP, os:timestamp()), - leveled_rand:uniform(State#state.score_onein) == 1, + rand:uniform(State#state.score_onein) == 1, State#state.score_onein} of {CachedScore, _UseNewScore, ScoreOneIn} when CachedScore == undefined; ScoreOneIn == 1 -> % If caches are not used, always use the current score - check_single_file(JournalP, - ScoringState#scoring_state.filter_fun, - ScoringState#scoring_state.filter_server, - ScoringState#scoring_state.max_sqn, - ?SAMPLE_SIZE, - ?BATCH_SIZE, - State#state.reload_strategy); + check_single_file( + JournalP, + ScoringState#scoring_state.filter_fun, + ScoringState#scoring_state.filter_server, + ScoringState#scoring_state.max_sqn, + ?SAMPLE_SIZE, + ?BATCH_SIZE, + State#state.reload_strategy + ); {CachedScore, true, _ScoreOneIn} -> % If caches are used roll the score towards the current score % Expectation is that this will reduce instances of individual % files being compacted when a run is missed due to cached % scores being used in surrounding journals NewScore = - check_single_file(JournalP, - ScoringState#scoring_state.filter_fun, - ScoringState#scoring_state.filter_server, - ScoringState#scoring_state.max_sqn, - ?SAMPLE_SIZE, - ?BATCH_SIZE, - State#state.reload_strategy), + check_single_file( + JournalP, + ScoringState#scoring_state.filter_fun, + ScoringState#scoring_state.filter_server, + ScoringState#scoring_state.max_sqn, + ?SAMPLE_SIZE, + ?BATCH_SIZE, + State#state.reload_strategy + ), (NewScore + CachedScore) / 2; {CachedScore, false, _ScoreOneIn} -> CachedScore end, ok = leveled_cdb:cdb_putcachedscore(JournalP, CpctPerc), Candidate = - #candidate{low_sqn = LowSQN, - filename = FN, - journal = JournalP, - compaction_perc = CpctPerc}, + #candidate{ + low_sqn = LowSQN, + filename = FN, + journal = JournalP, + compaction_perc = CpctPerc + }, ok = clerk_scorefilelist(self(), Tail), {noreply, State#state{scored_files = [Candidate|Candidates]}}; -handle_cast(scoring_complete, State) -> +handle_cast( + scoring_complete, State = #state{scoring_state = ScoringState}) + when ?IS_DEF(ScoringState) -> MaxRunLength = State#state.max_run_length, CDBopts = State#state.cdb_options, Candidates = lists:reverse(State#state.scored_files), - ScoringState = State#state.scoring_state, FilterFun = ScoringState#scoring_state.filter_fun, FilterServer = ScoringState#scoring_state.filter_server, MaxSQN = ScoringState#scoring_state.max_sqn, @@ -379,35 +396,45 @@ handle_cast(scoring_complete, State) -> true -> BestRun1 = sort_run(BestRun0), print_compaction_run(BestRun1, ScoreParams), - ManifestSlice = compact_files(BestRun1, - CDBopts, - FilterFun, - FilterServer, - MaxSQN, - State#state.reload_strategy, - State#state.compression_method), - FilesToDelete = lists:map(fun(C) -> - {C#candidate.low_sqn, - C#candidate.filename, - C#candidate.journal, - undefined} - end, - BestRun1), + ManifestSlice = + compact_files( + BestRun1, + CDBopts, + FilterFun, + FilterServer, + MaxSQN, + State#state.reload_strategy, + State#state.compression_method + ), + FilesToDelete = + lists:map( + fun(C) -> + { + C#candidate.low_sqn, + C#candidate.filename, + C#candidate.journal, + undefined + } + end, + BestRun1 + ), leveled_log:log(ic002, [length(FilesToDelete)]), ok = CloseFun(FilterServer), - ok = leveled_inker:ink_clerkcomplete(State#state.inker, - ManifestSlice, - FilesToDelete); + ok = + leveled_inker:ink_clerkcomplete( + State#state.inker, ManifestSlice, FilesToDelete); false -> ok = CloseFun(FilterServer), ok = leveled_inker:ink_clerkcomplete(State#state.inker, [], []) end, {noreply, State#state{scoring_state = undefined}, hibernate}; -handle_cast({trim, PersistedSQN, ManifestAsList}, State) -> +handle_cast( + {trim, PersistedSQN, ManifestAsList}, State = #state{inker = Ink}) + when ?IS_DEF(Ink) -> FilesToDelete = leveled_imanifest:find_persistedentries(PersistedSQN, ManifestAsList), leveled_log:log(ic007, []), - ok = leveled_inker:ink_clerkcomplete(State#state.inker, [], FilesToDelete), + ok = leveled_inker:ink_clerkcomplete(Ink, [], FilesToDelete), {noreply, State}; handle_cast({hashtable_calc, HashTree, StartPos, CDBpid}, State) -> {IndexList, HashTreeBin} = leveled_cdb:hashtable_calc(HashTree, StartPos), @@ -445,9 +472,9 @@ code_change(_OldVsn, State, _Extra) -> %%% External functions %%%============================================================================ --spec schedule_compaction(list(integer()), - integer(), - {integer(), integer(), integer()}) -> integer(). +-spec schedule_compaction( + list(integer()), integer(), {integer(), integer(), integer()}) -> + integer(). %% @doc %% Schedule the next compaction event for this store. Chooses a random %% interval, and then a random start time within the first third @@ -483,11 +510,11 @@ schedule_compaction(CompactionHours, RunsPerDay, CurrentTS) -> % today. RandSelect = fun(_X) -> - {lists:nth(leveled_rand:uniform(TotalHours), CompactionHours), - leveled_rand:uniform(?INTERVALS_PER_HOUR)} + {lists:nth(rand:uniform(TotalHours), CompactionHours), + rand:uniform(?INTERVALS_PER_HOUR)} end, - RandIntervals = lists:sort(lists:map(RandSelect, - lists:seq(1, RunsPerDay))), + RandIntervals = + lists:sort(lists:map(RandSelect, lists:seq(1, RunsPerDay))), % Pick the next interval from the list. The intervals before current time % are considered as intervals tomorrow, so will only be next if there are @@ -508,11 +535,11 @@ schedule_compaction(CompactionHours, RunsPerDay, CurrentTS) -> % Calculate the offset in seconds to this next interval NextS0 = NextI * (IntervalLength * 60) - - leveled_rand:uniform(IntervalLength * 60), + - rand:uniform(IntervalLength * 60), NextM = NextS0 div 60, NextS = NextS0 rem 60, - TimeDiff = calendar:time_difference(LocalTime, - {NextDate, {NextH, NextM, NextS}}), + TimeDiff = + calendar:time_difference(LocalTime, {NextDate, {NextH, NextM, NextS}}), {Days, {Hours, Mins, Secs}} = TimeDiff, Days * 86400 + Hours * 3600 + Mins * 60 + Secs. @@ -521,13 +548,14 @@ schedule_compaction(CompactionHours, RunsPerDay, CurrentTS) -> %%% Internal functions %%%============================================================================ --spec check_single_file(pid(), - leveled_inker:filterfun(), - leveled_inker:filterserver(), - leveled_codec:sqn(), - non_neg_integer(), non_neg_integer(), - leveled_codec:compaction_strategy()) -> - float(). +-spec check_single_file( + pid(), + leveled_inker:filterfun(), + leveled_inker:filterserver(), + leveled_codec:sqn(), + non_neg_integer(), non_neg_integer(), + leveled_codec:compaction_strategy()) -> + float(). %% @doc %% Get a score for a single CDB file in the journal. This will pull out a bunch %% of keys and sizes at random in an efficient way (by scanning the hashtable @@ -624,8 +652,8 @@ fetch_inbatches(PositionList, BatchSize, CDB, CheckedList) -> fetch_inbatches(Tail, BatchSize, CDB, CheckedList ++ KL_List). --spec assess_candidates(list(candidate()), score_parameters()) - -> {list(candidate()), float()}. +-spec assess_candidates( + list(candidate()), score_parameters()) -> {list(candidate()), float()}. %% @doc %% For each run length we need to assess all the possible runs of candidates, %% to determine which is the best score - to be put forward as the best @@ -704,10 +732,12 @@ score_run(Run, {MaxRunLength, MR_CT, SF_CT}) -> (MR_CT - SF_CT) / (MaxRunSize - 1) end, Target = SF_CT + TargetIncr * (length(Run) - 1), - RunTotal = lists:foldl(fun(Cand, Acc) -> - Acc + Cand#candidate.compaction_perc end, - 0.0, - Run), + RunTotal = + lists:foldl( + fun(Cand, Acc) -> Acc + Cand#candidate.compaction_perc end, + 0.0, + Run + ), Target - RunTotal / length(Run). @@ -750,26 +780,29 @@ compact_files([Batch|T], CDBopts, ActiveJournal0, FilterFun, FilterServer, MaxSQN, RStrategy, PressMethod, ManSlice0) -> {SrcJournal, PositionList} = Batch, - KVCs0 = leveled_cdb:cdb_directfetch(SrcJournal, - PositionList, - key_value_check), - KVCs1 = filter_output(KVCs0, - FilterFun, - FilterServer, - MaxSQN, - RStrategy), - {ActiveJournal1, ManSlice1} = write_values(KVCs1, - CDBopts, - ActiveJournal0, - ManSlice0, - PressMethod), + KVCs0 = + leveled_cdb:cdb_directfetch(SrcJournal, PositionList, key_value_check), + KVCs1 = + filter_output(KVCs0, FilterFun, FilterServer, MaxSQN, RStrategy), + {ActiveJournal1, ManSlice1} = + write_values( + KVCs1, CDBopts, ActiveJournal0, ManSlice0, PressMethod), % The inker's clerk will no longer need these (potentially large) binaries, % so force garbage collection at this point. This will mean when we roll % each CDB file there will be no remaining references to the binaries that % have been transferred and the memory can immediately be cleared garbage_collect(), - compact_files(T, CDBopts, ActiveJournal1, FilterFun, FilterServer, MaxSQN, - RStrategy, PressMethod, ManSlice1). + compact_files( + T, + CDBopts, + ActiveJournal1, + FilterFun, + FilterServer, + MaxSQN, + RStrategy, + PressMethod, + ManSlice1 + ). get_all_positions([], PositionBatches) -> PositionBatches; @@ -777,23 +810,25 @@ get_all_positions([HeadRef|RestOfBest], PositionBatches) -> SrcJournal = HeadRef#candidate.journal, Positions = leveled_cdb:cdb_getpositions(SrcJournal, all), leveled_log:log(ic008, [HeadRef#candidate.filename, length(Positions)]), - Batches = split_positions_into_batches(lists:sort(Positions), - SrcJournal, - []), + Batches = + split_positions_into_batches( + lists:sort(Positions), SrcJournal, [] + ), get_all_positions(RestOfBest, PositionBatches ++ Batches). split_positions_into_batches([], _Journal, Batches) -> Batches; split_positions_into_batches(Positions, Journal, Batches) -> - {ThisBatch, Tail} = if - length(Positions) > ?BATCH_SIZE -> - lists:split(?BATCH_SIZE, Positions); - true -> - {Positions, []} - end, - split_positions_into_batches(Tail, - Journal, - Batches ++ [{Journal, ThisBatch}]). + {ThisBatch, Tail} = + if + length(Positions) > ?BATCH_SIZE -> + lists:split(?BATCH_SIZE, Positions); + true -> + {Positions, []} + end, + split_positions_into_batches( + Tail, Journal, Batches ++ [{Journal, ThisBatch}] + ). %% @doc @@ -918,7 +953,7 @@ clear_waste(State) -> N = calendar:datetime_to_gregorian_seconds(calendar:local_time()), DeleteJournalFun = fun(DelJ) -> - LMD = filelib:last_modified(WP ++ DelJ), + LMD = {_,_} = filelib:last_modified(WP ++ DelJ), case N - calendar:datetime_to_gregorian_seconds(LMD) of LMD_Delta when LMD_Delta >= WRP -> ok = file:delete(WP ++ DelJ), @@ -931,12 +966,10 @@ clear_waste(State) -> lists:foreach(DeleteJournalFun, ClearedJournals) end. - %%%============================================================================ %%% Test %%%============================================================================ - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -966,24 +999,36 @@ local_time_to_now(DateTime) -> {Seconds div 1000000, Seconds rem 1000000, 0}. simple_score_test() -> - Run1 = [#candidate{compaction_perc = 75.0}, - #candidate{compaction_perc = 75.0}, - #candidate{compaction_perc = 76.0}, - #candidate{compaction_perc = 70.0}], + DummyC = + #candidate{ + low_sqn = 1, filename="dummy", journal=self(), compaction_perc = 0 + }, + Run1 = [DummyC#candidate{compaction_perc = 75.0}, + DummyC#candidate{compaction_perc = 75.0}, + DummyC#candidate{compaction_perc = 76.0}, + DummyC#candidate{compaction_perc = 70.0}], ?assertMatch(-4.0, score_run(Run1, {4, 70.0, 40.0})), - Run2 = [#candidate{compaction_perc = 75.0}], + Run2 = [DummyC#candidate{compaction_perc = 75.0}], ?assertMatch(-35.0, score_run(Run2, {4, 70.0, 40.0})), ?assertEqual(0.0, score_run([], {4, 40.0, 70.0})), - Run3 = [#candidate{compaction_perc = 100.0}], + Run3 = [DummyC#candidate{compaction_perc = 100.0}], ?assertMatch(-60.0, score_run(Run3, {4, 70.0, 40.0})). file_gc_test() -> - State = #state{waste_path="test/test_area/waste/", - waste_retention_period=1}, + State = + #state{ + waste_path="test/test_area/waste/", waste_retention_period = 1 + }, ok = filelib:ensure_dir(State#state.waste_path), - file:write_file(State#state.waste_path ++ "1.cdb", term_to_binary("Hello")), + file:write_file( + filename:join(State#state.waste_path, "1.cdb"), + term_to_binary("Hello") + ), timer:sleep(1100), - file:write_file(State#state.waste_path ++ "2.cdb", term_to_binary("Hello")), + file:write_file( + filename:join(State#state.waste_path, "2.cdb"), + term_to_binary("Hello") + ), clear_waste(State), {ok, ClearedJournals} = file:list_dir(State#state.waste_path), ?assertMatch(["2.cdb"], ClearedJournals), @@ -1004,27 +1049,47 @@ find_bestrun_test() -> %% -define(MAXRUNLENGTH_COMPACTION_TARGET, 60.0). %% Tested first with blocks significant as no back-tracking Params = {4, 60.0, 40.0}, - Block1 = [#candidate{compaction_perc = 55.0, filename = "a"}, - #candidate{compaction_perc = 65.0, filename = "b"}, - #candidate{compaction_perc = 42.0, filename = "c"}, - #candidate{compaction_perc = 50.0, filename = "d"}], - Block2 = [#candidate{compaction_perc = 38.0, filename = "e"}, - #candidate{compaction_perc = 75.0, filename = "f"}, - #candidate{compaction_perc = 75.0, filename = "g"}, - #candidate{compaction_perc = 45.0, filename = "h"}], - Block3 = [#candidate{compaction_perc = 70.0, filename = "i"}, - #candidate{compaction_perc = 100.0, filename = "j"}, - #candidate{compaction_perc = 100.0, filename = "k"}, - #candidate{compaction_perc = 100.0, filename = "l"}], - Block4 = [#candidate{compaction_perc = 55.0, filename = "m"}, - #candidate{compaction_perc = 56.0, filename = "n"}, - #candidate{compaction_perc = 57.0, filename = "o"}, - #candidate{compaction_perc = 40.0, filename = "p"}], - Block5 = [#candidate{compaction_perc = 60.0, filename = "q"}, - #candidate{compaction_perc = 60.0, filename = "r"}], + DummyC = + #candidate{ + low_sqn = 1, filename="dummy", journal=self(), compaction_perc = 0 + }, + Block1 = + [ + DummyC#candidate{compaction_perc = 55.0, filename = "a"}, + DummyC#candidate{compaction_perc = 65.0, filename = "b"}, + DummyC#candidate{compaction_perc = 42.0, filename = "c"}, + DummyC#candidate{compaction_perc = 50.0, filename = "d"} + ], + Block2 = + [ + DummyC#candidate{compaction_perc = 38.0, filename = "e"}, + DummyC#candidate{compaction_perc = 75.0, filename = "f"}, + DummyC#candidate{compaction_perc = 75.0, filename = "g"}, + DummyC#candidate{compaction_perc = 45.0, filename = "h"} + ], + Block3 = + [ + DummyC#candidate{compaction_perc = 70.0, filename = "i"}, + DummyC#candidate{compaction_perc = 100.0, filename = "j"}, + DummyC#candidate{compaction_perc = 100.0, filename = "k"}, + DummyC#candidate{compaction_perc = 100.0, filename = "l"} + ], + Block4 = + [ + DummyC#candidate{compaction_perc = 55.0, filename = "m"}, + DummyC#candidate{compaction_perc = 56.0, filename = "n"}, + DummyC#candidate{compaction_perc = 57.0, filename = "o"}, + DummyC#candidate{compaction_perc = 40.0, filename = "p"} + ], + Block5 = + [ + DummyC#candidate{compaction_perc = 60.0, filename = "q"}, + DummyC#candidate{compaction_perc = 60.0, filename = "r"} + ], CList0 = Block1 ++ Block2 ++ Block3 ++ Block4 ++ Block5, ?assertMatch(["b", "c", "d", "e"], check_bestrun(CList0, Params)), - CList1 = CList0 ++ [#candidate{compaction_perc = 20.0, filename="s"}], + CList1 = + CList0 ++ [DummyC#candidate{compaction_perc = 20.0, filename="s"}], ?assertMatch(["s"], check_bestrun(CList1, Params)), CList2 = Block4 ++ Block3 ++ Block2 ++ Block1 ++ Block5, ?assertMatch(["h", "a", "b", "c"], check_bestrun(CList2, Params)), @@ -1219,12 +1284,18 @@ compact_empty_file_test() -> ok = leveled_cdb:cdb_destroy(CDB2). compare_candidate_test() -> - Candidate1 = #candidate{low_sqn=1}, - Candidate2 = #candidate{low_sqn=2}, - Candidate3 = #candidate{low_sqn=3}, - Candidate4 = #candidate{low_sqn=4}, - ?assertMatch([Candidate1, Candidate2, Candidate3, Candidate4], - sort_run([Candidate3, Candidate2, Candidate4, Candidate1])). + DummyC = + #candidate{ + low_sqn = 1, filename="dummy", journal=self(), compaction_perc = 0 + }, + Candidate1 = DummyC#candidate{low_sqn=1}, + Candidate2 = DummyC#candidate{low_sqn=2}, + Candidate3 = DummyC#candidate{low_sqn=3}, + Candidate4 = DummyC#candidate{low_sqn=4}, + ?assertMatch( + [Candidate1, Candidate2, Candidate3, Candidate4], + sort_run([Candidate3, Candidate2, Candidate4, Candidate1]) + ). compact_singlefile_totwosmallfiles_test_() -> {timeout, 60, fun compact_singlefile_totwosmallfiles_testto/0}. @@ -1236,24 +1307,31 @@ compact_singlefile_totwosmallfiles_testto() -> FN1 = leveled_inker:filepath(RP, 1, new_journal), CDBoptsLarge = #cdb_options{binary_mode=true, max_size=30000000}, {ok, CDB1} = leveled_cdb:cdb_open_writer(FN1, CDBoptsLarge), - lists:foreach(fun(X) -> - LK = test_ledgerkey("Key" ++ integer_to_list(X)), - Value = leveled_rand:rand_bytes(1024), - {IK, IV} = - leveled_codec:to_inkerkv(LK, X, Value, - {[], infinity}, - native, true), - ok = leveled_cdb:cdb_put(CDB1, IK, IV) - end, - lists:seq(1, 1000)), + lists:foreach( + fun(X) -> + LK = test_ledgerkey("Key" ++ integer_to_list(X)), + Value = crypto:strong_rand_bytes(1024), + {IK, IV} = + leveled_codec:to_inkerkv(LK, X, Value, + {[], infinity}, + native, true), + ok = leveled_cdb:cdb_put(CDB1, IK, IV) + end, + lists:seq(1, 1000) + ), {ok, NewName} = leveled_cdb:cdb_complete(CDB1), {ok, CDBr} = leveled_cdb:cdb_open_reader(NewName), CDBoptsSmall = #cdb_options{binary_mode=true, max_size=400000, file_path=CP}, - BestRun1 = [#candidate{low_sqn=1, - filename=leveled_cdb:cdb_filename(CDBr), - journal=CDBr, - compaction_perc=50.0}], + BestRun1 = + [ + #candidate{ + low_sqn=1, + filename=leveled_cdb:cdb_filename(CDBr), + journal=CDBr, + compaction_perc=50.0 + } + ], FakeFilterFun = fun(_FS, _LK, SQN) -> case SQN rem 2 of @@ -1262,19 +1340,24 @@ compact_singlefile_totwosmallfiles_testto() -> end end, - ManifestSlice = compact_files(BestRun1, - CDBoptsSmall, - FakeFilterFun, - null, - 900, - [{?STD_TAG, recovr}], - native), + ManifestSlice = + compact_files( + BestRun1, + CDBoptsSmall, + FakeFilterFun, + null, + 900, + [{?STD_TAG, recovr}], + native + ), ?assertMatch(2, length(ManifestSlice)), - lists:foreach(fun({_SQN, _FN, CDB, _LK}) -> - ok = leveled_cdb:cdb_deletepending(CDB), - ok = leveled_cdb:cdb_destroy(CDB) - end, - ManifestSlice), + lists:foreach( + fun({_SQN, _FN, CDB, _LK}) -> + ok = leveled_cdb:cdb_deletepending(CDB), + ok = leveled_cdb:cdb_destroy(CDB) + end, + ManifestSlice + ), ok = leveled_cdb:cdb_deletepending(CDBr), ok = leveled_cdb:cdb_destroy(CDBr). @@ -1304,11 +1387,13 @@ size_score_test() -> end end, Score = - size_comparison_score(KeySizeList, - FilterFun, - CurrentList, - MaxSQN, - leveled_codec:inker_reload_strategy([])), + size_comparison_score( + KeySizeList, + FilterFun, + CurrentList, + MaxSQN, + leveled_codec:inker_reload_strategy([]) + ), io:format("Score ~w", [Score]), ?assertMatch(true, Score > 69.0), ?assertMatch(true, Score < 70.0). diff --git a/src/leveled_imanifest.erl b/src/leveled_imanifest.erl index 480c197..919c240 100644 --- a/src/leveled_imanifest.erl +++ b/src/leveled_imanifest.erl @@ -28,9 +28,7 @@ -type manifest() :: list({integer(), list()}). %% The manifest is divided into blocks by sequence number, with each block %% being a list of manifest entries for that SQN range. --type manifest_entry() :: {integer(), string(), pid()|string(), any()}. -%% The Entry should have a pid() as the third element, but a string() may be -%% used in unit tests +-type manifest_entry() :: {integer(), string(), pid(), any()}. -export_type([manifest/0, manifest_entry/0]). @@ -75,8 +73,8 @@ add_entry(Manifest, Entry, ToEnd) -> from_list(Man1) end. --spec append_lastkey(manifest(), pid(), leveled_codec:journal_key()) - -> manifest(). +-spec append_lastkey( + manifest(), pid(), leveled_codec:journal_key()) -> manifest(). %% @doc %% On discovery of the last key in the last journal entry, the manifest can %% be updated through this function to have the last key @@ -100,7 +98,7 @@ remove_entry(Manifest, Entry) -> Man0 = lists:keydelete(SQN, 1, to_list(Manifest)), from_list(Man0). --spec find_entry(integer(), manifest()) -> pid()|string(). +-spec find_entry(integer(), manifest()) -> pid(). %% @doc %% Given a SQN find the relevant manifest_entry, returning just the pid() of %% the journal file (which may be a string() in unit tests) @@ -257,18 +255,31 @@ build_testmanifest_aslist() -> ManifestMapFun = fun(N) -> NStr = integer_to_list(N), - {max(1, N * 1000), "FN" ++ NStr, "pid" ++ NStr, "LK" ++ NStr} + { + max(1, N * 1000), + "FN" ++ NStr, + set_pid(N), + "LK" ++ NStr + } end, lists:map(ManifestMapFun, lists:reverse(lists:seq(0, 50))). +set_pid(N) -> + lists:flatten(io_lib:format("<0.1~2..0w.0>", [N])). + test_testmanifest(Man0) -> - ?assertMatch("pid0", find_entry(1, Man0)), - ?assertMatch("pid0", find_entry(2, Man0)), - ?assertMatch("pid1", find_entry(1001, Man0)), - ?assertMatch("pid20", find_entry(20000, Man0)), - ?assertMatch("pid20", find_entry(20001, Man0)), - ?assertMatch("pid20", find_entry(20999, Man0)), - ?assertMatch("pid50", find_entry(99999, Man0)). + P0 = set_pid(0), + P1 = set_pid(1), + P20 = set_pid(20), + P50 = set_pid(50), + + ?assertMatch(P0, find_entry(1, Man0)), + ?assertMatch(P0, find_entry(2, Man0)), + ?assertMatch(P1, find_entry(1001, Man0)), + ?assertMatch(P20, find_entry(20000, Man0)), + ?assertMatch(P20, find_entry(20001, Man0)), + ?assertMatch(P20, find_entry(20999, Man0)), + ?assertMatch(P50, find_entry(99999, Man0)). buildfromlist_test() -> ManL = build_testmanifest_aslist(), @@ -303,7 +314,7 @@ buildrandomfashion_test() -> ManL0 = build_testmanifest_aslist(), RandMapFun = fun(X) -> - {leveled_rand:uniform(), X} + {rand:uniform(), X} end, ManL1 = lists:map(RandMapFun, ManL0), ManL2 = lists:sort(ManL1), @@ -317,7 +328,7 @@ buildrandomfashion_test() -> test_testmanifest(Man0), ?assertMatch(ManL0, to_list(Man0)), - RandomEntry = lists:nth(leveled_rand:uniform(50), ManL0), + RandomEntry = lists:nth(rand:uniform(50), ManL0), Man1 = remove_entry(Man0, RandomEntry), Man2 = add_entry(Man1, RandomEntry, false), diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 6ad5ced..9721267 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -147,7 +147,7 @@ -record(state, {manifest = [] :: list(), manifest_sqn = 0 :: integer(), - journal_sqn = 0 :: integer(), + journal_sqn = 0 :: non_neg_integer(), active_journaldb :: pid() | undefined, pending_removals = [] :: list(), registered_snapshots = [] :: list(registered_snapshot()), @@ -159,7 +159,9 @@ is_snapshot = false :: boolean(), compression_method = native :: lz4|native|none, compress_on_receipt = false :: boolean(), - snap_timeout :: pos_integer() | undefined, % in seconds + snap_timeout = 0 :: non_neg_integer(), + % in seconds, 0 for snapshots + % (only relevant for primary Inker) source_inker :: pid() | undefined, shutdown_loops = ?SHUTDOWN_LOOPS :: non_neg_integer()}). @@ -196,20 +198,26 @@ %% The inker will need to know what the reload strategy is, to inform the %% clerk about the rules to enforce during compaction. ink_start(InkerOpts) -> - gen_server:start_link(?MODULE, [leveled_log:get_opts(), InkerOpts], []). + {ok, Inker} = + gen_server:start_link( + ?MODULE, [leveled_log:get_opts(), InkerOpts], []), + {ok, Inker}. -spec ink_snapstart(inker_options()) -> {ok, pid()}. %% @doc %% Don't link on startup as snapshot ink_snapstart(InkerOpts) -> - gen_server:start(?MODULE, [leveled_log:get_opts(), InkerOpts], []). + {ok, Inker} = + gen_server:start( + ?MODULE, [leveled_log:get_opts(), InkerOpts], []), + {ok, Inker}. -spec ink_put(pid(), leveled_codec:ledger_key(), any(), leveled_codec:journal_keychanges(), boolean()) -> - {ok, integer(), integer()}. + {ok, non_neg_integer(), pos_integer()}. %% @doc %% PUT an object into the journal, returning the sequence number for the PUT %% as well as the size of the object (information required by the ledger). @@ -504,14 +512,13 @@ ink_getclerkpid(Pid) -> init([LogOpts, InkerOpts]) -> leveled_log:save(LogOpts), - leveled_rand:seed(), case {InkerOpts#inker_options.root_path, - InkerOpts#inker_options.start_snapshot} of - {undefined, true} -> + InkerOpts#inker_options.start_snapshot, + InkerOpts#inker_options.source_inker} of + {undefined, true, SrcInker} when ?IS_DEF(SrcInker) -> %% monitor the bookie, and close the snapshot when bookie %% exits BookieMonitor = erlang:monitor(process, InkerOpts#inker_options.bookies_pid), - SrcInker = InkerOpts#inker_options.source_inker, {Manifest, ActiveJournalDB, JournalSQN} = ink_registersnapshot(SrcInker, self()), @@ -522,7 +529,7 @@ init([LogOpts, InkerOpts]) -> bookie_monref = BookieMonitor, is_snapshot = true}}; %% Need to do something about timeout - {_RootPath, false} -> + {_RootPath, false, _SrcInker} -> start_from_file(InkerOpts) end. @@ -557,10 +564,12 @@ handle_call({fold, Manifest = lists:reverse(leveled_imanifest:to_list(State#state.manifest)), Folder = fun() -> - fold_from_sequence(StartSQN, - {FilterFun, InitAccFun, FoldFun}, - Acc, - Manifest) + fold_from_sequence( + StartSQN, + {FilterFun, InitAccFun, FoldFun}, + Acc, + Manifest + ) end, case By of as_ink -> @@ -583,25 +592,21 @@ handle_call(get_manifest, _From, State) -> handle_call(print_manifest, _From, State) -> leveled_imanifest:printer(State#state.manifest), {reply, ok, State}; -handle_call({compact, - Checker, - InitiateFun, - CloseFun, - FilterFun}, - _From, State=#state{is_snapshot=Snap}) when Snap == false -> +handle_call( + {compact, Checker, InitiateFun, CloseFun, FilterFun}, + _From, + State=#state{is_snapshot=Snap}) + when Snap == false -> Clerk = State#state.clerk, Manifest = leveled_imanifest:to_list(State#state.manifest), - leveled_iclerk:clerk_compact(State#state.clerk, - Checker, - InitiateFun, - CloseFun, - FilterFun, - Manifest), + leveled_iclerk:clerk_compact( + Clerk, Checker, InitiateFun, CloseFun, FilterFun, Manifest), {reply, {ok, Clerk}, State#state{compaction_pending=true}}; handle_call(compaction_pending, _From, State) -> {reply, State#state.compaction_pending, State}; -handle_call({trim, PersistedSQN}, _From, State=#state{is_snapshot=Snap}) - when Snap == false -> +handle_call( + {trim, PersistedSQN}, _From, State=#state{is_snapshot=Snap}) + when Snap == false -> Manifest = leveled_imanifest:to_list(State#state.manifest), ok = leveled_iclerk:clerk_trim(State#state.clerk, PersistedSQN, Manifest), {reply, ok, State}; @@ -625,8 +630,9 @@ handle_call(roll, _From, State=#state{is_snapshot=Snap}) when Snap == false -> manifest_sqn = NewManSQN, active_journaldb = NewJournalP}} end; -handle_call({backup, BackupPath}, _from, State) - when State#state.is_snapshot == true -> +handle_call( + {backup, BackupPath}, _from, State) + when State#state.is_snapshot == true -> SW = os:timestamp(), BackupJFP = filepath(filename:join(BackupPath, ?JOURNAL_FP), journal_dir), ok = filelib:ensure_dir(BackupJFP), @@ -665,7 +671,7 @@ handle_call({backup, BackupPath}, _from, State) leveled_log:log(i0022, [RFN]), RemoveFile = filename:join(BackupJFP, RFN), case filelib:is_file(RemoveFile) - and not filelib:is_dir(RemoveFile) of + andalso not filelib:is_dir(RemoveFile) of true -> ok = file:delete(RemoveFile); false -> @@ -699,12 +705,13 @@ handle_call(get_clerkpid, _From, State) -> handle_call(close, _From, State=#state{is_snapshot=Snap}) when Snap == true -> ok = ink_releasesnapshot(State#state.source_inker, self()), {stop, normal, ok, State}; -handle_call(ShutdownType, From, State) - when ShutdownType == close; ShutdownType == doom -> +handle_call( + ShutdownType, From, State = #state{clerk = Clerk}) + when ?IS_DEF(Clerk) -> case ShutdownType of doom -> leveled_log:log(i0018, []); - _ -> + close -> ok end, leveled_log:log(i0005, [ShutdownType]), @@ -714,9 +721,10 @@ handle_call(ShutdownType, From, State) gen_server:cast(self(), {maybe_defer_shutdown, ShutdownType, From}), {noreply, State}. - -handle_cast({clerk_complete, ManifestSnippet, FilesToDelete}, State) -> - CDBOpts = State#state.cdb_options, +handle_cast( + {clerk_complete, ManifestSnippet, FilesToDelete}, + State = #state{cdb_options = CDBOpts}) + when ?IS_DEF(CDBOpts) -> DropFun = fun(E, Acc) -> leveled_imanifest:remove_entry(Acc, E) @@ -854,8 +862,10 @@ handle_cast({complete_shutdown, ShutdownType, From}, State) -> {stop, normal, State}. %% handle the bookie stopping and stop this snapshot -handle_info({'DOWN', BookieMonRef, process, _BookiePid, _Info}, - State=#state{bookie_monref = BookieMonRef}) -> +handle_info( + {'DOWN', BookieMonRef, process, _BookiePid, _Info}, + State=#state{bookie_monref = BookieMonRef, source_inker = SrcInker}) + when ?IS_DEF(SrcInker) -> %% Monitor only registered on snapshots ok = ink_releasesnapshot(State#state.source_inker, self()), {stop, normal, State}; @@ -879,7 +889,10 @@ code_change(_OldVsn, State, _Extra) -> -spec start_from_file(inker_options()) -> {ok, ink_state()}. %% @doc %% Start an Inker from the state on disk (i.e. not a snapshot). -start_from_file(InkOpts) -> +start_from_file( + InkOpts = + #inker_options{root_path = RootPath, snaptimeout_long = SnapTimeout}) + when ?IS_DEF(RootPath), ?IS_DEF(SnapTimeout) -> % Setting the correct CDB options is important when starting the inker, in % particular for waste retention which is determined by the CDB options % with which the file was last opened @@ -926,9 +939,8 @@ start_from_file(InkOpts) -> {Manifest, ManifestSQN, JournalSQN, - ActiveJournal} = build_manifest(ManifestFilenames, - RootPath, - CDBopts), + ActiveJournal} = + build_manifest(ManifestFilenames, RootPath, CDBopts), {ok, #state{manifest = Manifest, manifest_sqn = ManifestSQN, journal_sqn = JournalSQN, @@ -971,61 +983,74 @@ get_cdbopts(InkOpts)-> CDBopts#cdb_options{waste_path = WasteFP}. --spec put_object(leveled_codec:ledger_key(), - any(), - leveled_codec:journal_keychanges(), - boolean(), - ink_state()) - -> {ok|rolling, ink_state(), integer()}. +-spec put_object( + leveled_codec:primary_key(), + any(), + leveled_codec:journal_keychanges(), + boolean(), + ink_state()) + -> {ok|rolling, ink_state(), integer()}. %% @doc %% Add the object to the current journal if it fits. If it doesn't fit, a new %% journal must be started, and the old journal is set to "roll" into a read %% only Journal. %% The reply contains the byte_size of the object, using the size calculated %% to store the object. -put_object(LedgerKey, Object, KeyChanges, Sync, State) -> +put_object( + LedgerKey, + Object, + KeyChanges, + Sync, + State = + #state{ + active_journaldb = ActiveJournal, + cdb_options = CDBOpts, + root_path = RP + }) + when ?IS_DEF(ActiveJournal), ?IS_DEF(CDBOpts), ?IS_DEF(RP) -> NewSQN = State#state.journal_sqn + 1, - ActiveJournal = State#state.active_journaldb, {JournalKey, JournalBin} = - leveled_codec:to_inkerkv(LedgerKey, - NewSQN, - Object, - KeyChanges, - State#state.compression_method, - State#state.compress_on_receipt), - case leveled_cdb:cdb_put(ActiveJournal, - JournalKey, - JournalBin, - Sync) of + leveled_codec:to_inkerkv( + LedgerKey, + NewSQN, + Object, + KeyChanges, + State#state.compression_method, + State#state.compress_on_receipt + ), + PutR = leveled_cdb:cdb_put(ActiveJournal, JournalKey, JournalBin, Sync), + case PutR of ok -> - {ok, - State#state{journal_sqn=NewSQN}, - byte_size(JournalBin)}; + {ok, State#state{journal_sqn=NewSQN}, byte_size(JournalBin)}; roll -> SWroll = os:timestamp(), {NewJournalP, Manifest1, NewManSQN} = - roll_active(ActiveJournal, - State#state.manifest, - NewSQN, - State#state.cdb_options, - State#state.root_path, - State#state.manifest_sqn), + roll_active( + ActiveJournal, + State#state.manifest, + NewSQN, + State#state.cdb_options, + State#state.root_path, + State#state.manifest_sqn + ), leveled_log:log_timer(i0008, [], SWroll), - ok = leveled_cdb:cdb_put(NewJournalP, - JournalKey, - JournalBin), + ok = + leveled_cdb:cdb_put( + NewJournalP, JournalKey, JournalBin), {rolling, - State#state{journal_sqn=NewSQN, - manifest=Manifest1, - manifest_sqn = NewManSQN, - active_journaldb=NewJournalP}, + State#state{ + journal_sqn=NewSQN, + manifest=Manifest1, + manifest_sqn = NewManSQN, + active_journaldb=NewJournalP}, byte_size(JournalBin)} end. --spec get_object(leveled_codec:ledger_key(), - integer(), - leveled_imanifest:manifest()) -> any(). +-spec get_object( + leveled_codec:ledger_key(), + integer(), + leveled_imanifest:manifest()) -> any(). %% @doc %% Find the SQN in the manifest and then fetch the object from the Journal, %% in the manifest. If the fetch is in response to a user GET request then @@ -1041,28 +1066,36 @@ get_object(LedgerKey, SQN, Manifest, ToIgnoreKeyChanges) -> leveled_codec:from_inkerkv(Obj, ToIgnoreKeyChanges). --spec roll_active(pid(), leveled_imanifest:manifest(), - integer(), #cdb_options{}, string(), integer()) -> - {pid(), leveled_imanifest:manifest(), integer()}. +-spec roll_active( + pid(), + leveled_imanifest:manifest(), + integer(), + #cdb_options{}, + string(), + integer()) -> {pid(), leveled_imanifest:manifest(), integer()}. %% @doc %% Roll the active journal, and start a new active journal, updating the %% manifest roll_active(ActiveJournal, Manifest, NewSQN, CDBopts, RootPath, ManifestSQN) -> - LastKey = leveled_cdb:cdb_lastkey(ActiveJournal), - ok = leveled_cdb:cdb_roll(ActiveJournal), - Manifest0 = - leveled_imanifest:append_lastkey(Manifest, ActiveJournal, LastKey), - ManEntry = - start_new_activejournal(NewSQN, RootPath, CDBopts), - {_, _, NewJournalP, _} = ManEntry, - Manifest1 = leveled_imanifest:add_entry(Manifest0, ManEntry, true), - ok = leveled_imanifest:writer(Manifest1, ManifestSQN + 1, RootPath), - - {NewJournalP, Manifest1, ManifestSQN + 1}. + case leveled_cdb:cdb_lastkey(ActiveJournal) of + LastKey when LastKey =/= empty -> + ok = leveled_cdb:cdb_roll(ActiveJournal), + Manifest0 = + leveled_imanifest:append_lastkey(Manifest, ActiveJournal, LastKey), + ManEntry = + start_new_activejournal(NewSQN, RootPath, CDBopts), + {_, _, NewJournalP, _} = ManEntry, + Manifest1 = leveled_imanifest:add_entry(Manifest0, ManEntry, true), + ok = + leveled_imanifest:writer(Manifest1, ManifestSQN + 1, RootPath), + + {NewJournalP, Manifest1, ManifestSQN + 1} + end. --spec key_check(leveled_codec:ledger_key(), - integer(), - leveled_imanifest:manifest()) -> missing|probably. +-spec key_check( + leveled_codec:primary_key(), + integer(), + leveled_imanifest:manifest()) -> missing|probably. %% @doc %% Checks for the presence of the key at that SQN withing the journal, %% avoiding the cost of actually reading the object from disk. @@ -1081,40 +1114,36 @@ key_check(LedgerKey, SQN, Manifest) -> %% Selects the correct manifest to open, and then starts a process for each %% file in the manifest, storing the PID for that process within the manifest. %% Opens an active journal if one is not present. -build_manifest(ManifestFilenames, - RootPath, - CDBopts) -> +build_manifest(ManifestFilenames, RootPath, CDBopts) -> % Find the manifest with a highest Manifest sequence number % Open it and read it to get the current Confirmed Manifest ManifestRegex = "(?[0-9]+)\\." ++ leveled_imanifest:complete_filex(), - ValidManSQNs = sequencenumbers_fromfilenames(ManifestFilenames, - ManifestRegex, - 'MSQN'), - {Manifest, - ManifestSQN} = case length(ValidManSQNs) of - 0 -> - {[], 1}; - _ -> - PersistedManSQN = lists:max(ValidManSQNs), - M1 = leveled_imanifest:reader(PersistedManSQN, - RootPath), - {M1, PersistedManSQN} - end, + ValidManSQNs = + sequencenumbers_fromfilenames( + ManifestFilenames, ManifestRegex, 'MSQN'), + {Manifest, ManifestSQN} = + case length(ValidManSQNs) of + 0 -> + {[], 1}; + _ -> + PersistedManSQN = lists:max(ValidManSQNs), + M1 = leveled_imanifest:reader(PersistedManSQN, RootPath), + {M1, PersistedManSQN} + end, % Open the manifest files, completing if necessary and ensure there is % a valid active journal at the head of the manifest OpenManifest = open_all_manifest(Manifest, RootPath, CDBopts), - {ActiveLowSQN, - _FN, - ActiveJournal, - _LK} = leveled_imanifest:head_entry(OpenManifest), - JournalSQN = case leveled_cdb:cdb_lastkey(ActiveJournal) of - empty -> - ActiveLowSQN; - {JSQN, _Type, _LastKey} -> - JSQN - end, + {ActiveLowSQN, _FN, ActiveJournal, _LK} = + leveled_imanifest:head_entry(OpenManifest), + JournalSQN = + case leveled_cdb:cdb_lastkey(ActiveJournal) of + empty -> + ActiveLowSQN; + {JSQN, _Type, _LastKey} -> + JSQN + end, % Update the manifest if it has been changed by the process of loading % the manifest (must also increment the manifest SQN). @@ -1146,8 +1175,9 @@ close_allmanifest([H|ManifestT]) -> close_allmanifest(ManifestT). --spec open_all_manifest(leveled_imanifest:manifest(), list(), #cdb_options{}) - -> leveled_imanifest:manifest(). +-spec open_all_manifest( + leveled_imanifest:manifest(), list(), #cdb_options{}) + -> leveled_imanifest:manifest(). %% @doc %% Open all the files in the manifets, and updating the manifest with the PIDs %% of the opened files @@ -1185,24 +1215,21 @@ open_all_manifest(Man0, RootPath, CDBOpts) -> true -> leveled_log:log(i0012, [HeadFN]), {ok, HeadR} = leveled_cdb:cdb_open_reader(CompleteHeadFN), - LastKey = leveled_cdb:cdb_lastkey(HeadR), - LastSQN = element(1, LastKey), - ManToHead = leveled_imanifest:add_entry(OpenedTail, - {HeadSQN, - HeadFN, - HeadR, - LastKey}, - true), - NewManEntry = start_new_activejournal(LastSQN + 1, - RootPath, - CDBOpts), + LastKey = {LastSQN, _, _} = leveled_cdb:cdb_lastkey(HeadR), + ManToHead = + leveled_imanifest:add_entry( + OpenedTail, + {HeadSQN, HeadFN, HeadR, LastKey}, + true + ), + NewManEntry = + start_new_activejournal(LastSQN + 1, RootPath, CDBOpts), leveled_imanifest:add_entry(ManToHead, NewManEntry, true); false -> - {ok, HeadW} = leveled_cdb:cdb_open_writer(PendingHeadFN, - CDBOpts), - leveled_imanifest:add_entry(OpenedTail, - {HeadSQN, HeadFN, HeadW, HeadLK}, - true) + {ok, HeadW} = + leveled_cdb:cdb_open_writer(PendingHeadFN, CDBOpts), + leveled_imanifest:add_entry( + OpenedTail, {HeadSQN, HeadFN, HeadW, HeadLK}, true) end. @@ -1288,17 +1315,19 @@ foldfile_between_sequence(MinSQN, MaxSQN, FoldFuns, sequencenumbers_fromfilenames(Filenames, Regex, IntName) -> - lists:foldl(fun(FN, Acc) -> - case re:run(FN, - Regex, - [{capture, [IntName], list}]) of - nomatch -> - Acc; - {match, [Int]} when is_list(Int) -> - Acc ++ [list_to_integer(Int)] - end end, - [], - Filenames). + lists:foldl( + fun(FN, Acc) -> + case re:run(FN, + Regex, + [{capture, [IntName], list}]) of + nomatch -> + Acc; + {match, [Int]} when is_list(Int) -> + Acc ++ [list_to_integer(Int)] + end + end, + [], + Filenames). filepath(RootPath, journal_dir) -> RootPath ++ "/" ++ ?FILES_FP ++ "/"; @@ -1525,7 +1554,7 @@ compact_journal_testto(WRP, ExpectedFiles) -> PK = "KeyZ" ++ integer_to_list(X), {ok, SQN, _} = ink_put(Ink1, test_ledgerkey(PK), - leveled_rand:rand_bytes(10000), + crypto:strong_rand_bytes(10000), {[], infinity}, false), {SQN, test_ledgerkey(PK)} diff --git a/src/leveled_log.erl b/src/leveled_log.erl index f698201..d35a655 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -377,7 +377,7 @@ get_opts() -> } end. --spec return_settings() -> {log_level(), list(string())}. +-spec return_settings() -> {log_level(), list(atom())}. %% @doc %% Return the settings outside of the record return_settings() -> @@ -454,7 +454,7 @@ log_timer(LogRef, Subs, StartTime, SupportedLevels) -> -spec log_randomtimer(atom(), list(), erlang:timestamp(), float()) -> ok. log_randomtimer(LogReference, Subs, StartTime, RandomProb) -> - R = leveled_rand:uniform(), + R = rand:uniform(), case R < RandomProb of true -> log_timer(LogReference, Subs, StartTime); diff --git a/src/leveled_monitor.erl b/src/leveled_monitor.erl index 78b10db..4e776f7 100644 --- a/src/leveled_monitor.erl +++ b/src/leveled_monitor.erl @@ -136,13 +136,13 @@ {leveled_pmanifest:lsm_level(), #sst_fetch_timings{}}. -type log_type() :: bookie_head|bookie_get|bookie_put|bookie_snap|pcl_fetch|sst_fetch|cdb_get. --type pcl_level() :: mem|leveled_pmanifest:lsm_level(). +-type pcl_level() :: memory|leveled_pmanifest:lsm_level(). -type sst_fetch_type() :: fetch_cache|slot_cachedblock|slot_noncachedblock|not_found. -type microsecs() :: pos_integer(). -type byte_size() :: pos_integer(). -type monitor() :: {no_monitor, 0}|{pid(), 0..100}. --type timing() :: no_timing|pos_integer(). +-type timing() :: no_timing|microsecs(). -type bookie_get_update() :: @@ -173,8 +173,10 @@ -spec monitor_start(pos_integer(), list(log_type())) -> {ok, pid()}. monitor_start(LogFreq, LogOrder) -> - gen_server:start_link( - ?MODULE, [leveled_log:get_opts(), LogFreq, LogOrder], []). + {ok, Monitor} = + gen_server:start_link( + ?MODULE, [leveled_log:get_opts(), LogFreq, LogOrder], []), + {ok, Monitor}. -spec add_stat(pid(), statistic()) -> ok. add_stat(Watcher, Statistic) -> @@ -204,7 +206,7 @@ log_remove(Pid, ForcedLogs) -> -spec maybe_time(monitor()) -> erlang:timestamp()|no_timing. maybe_time({_Pid, TimingProbability}) -> - case leveled_rand:uniform(100) of + case rand:uniform(100) of N when N =< TimingProbability -> os:timestamp(); _ -> @@ -230,16 +232,15 @@ get_defaults() -> init([LogOpts, LogFrequency, LogOrder]) -> leveled_log:save(LogOpts), - leveled_rand:seed(), RandomLogOrder = lists:map( fun({_R, SL}) -> SL end, lists:keysort( 1, lists:map( - fun(L) -> {leveled_rand:uniform(), L} end, + fun(L) -> {rand:uniform(), L} end, LogOrder))), - InitialJitter = leveled_rand:uniform(2 * 1000 * LogFrequency), + InitialJitter = rand:uniform(2 * 1000 * LogFrequency), erlang:send_after(InitialJitter, self(), report_next_stats), {ok, #state{log_frequency = LogFrequency, log_order = RandomLogOrder}}. diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index dcc3812..107bcf1 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -48,8 +48,8 @@ -define(MIN_TIMEOUT, 200). -define(GROOMING_PERC, 50). --record(state, {owner :: pid() | undefined, - root_path :: string() | undefined, +-record(state, {owner :: pid()|undefined, + root_path :: string()|undefined, pending_deletions = dict:new() :: dict:dict(), sst_options :: sst_options() }). @@ -123,18 +123,20 @@ handle_call(close, _From, State) -> handle_cast(prompt, State) -> handle_info(timeout, State); -handle_cast({push_work, Work}, State) -> +handle_cast( + {push_work, Work}, State = #state{root_path = RP, owner = PCL}) + when ?IS_DEF(RP), is_pid(PCL) -> {ManifestSQN, Deletions} = - handle_work( - Work, - State#state.root_path, State#state.sst_options, State#state.owner), + handle_work(Work, RP, State#state.sst_options, PCL), PDs = dict:store(ManifestSQN, Deletions, State#state.pending_deletions), leveled_log:log(pc022, [ManifestSQN]), {noreply, State#state{pending_deletions = PDs}, ?MIN_TIMEOUT}; -handle_cast({prompt_deletions, ManifestSQN}, State) -> - {Deletions, UpdD} = return_deletions(ManifestSQN, - State#state.pending_deletions), - ok = notify_deletions(Deletions, State#state.owner), +handle_cast( + {prompt_deletions, ManifestSQN}, State = #state{owner = PCL}) + when is_pid(PCL) -> + {Deletions, UpdD} = + return_deletions(ManifestSQN, State#state.pending_deletions), + ok = notify_deletions(Deletions, PCL), {noreply, State#state{pending_deletions = UpdD}, ?MIN_TIMEOUT}; handle_cast({log_level, LogLevel}, State) -> ok = leveled_log:set_loglevel(LogLevel), @@ -152,8 +154,8 @@ handle_cast({remove_logs, ForcedLogs}, State) -> SSTopts0 = SSTopts#sst_options{log_options = leveled_log:get_opts()}, {noreply, State#state{sst_options = SSTopts0}}. -handle_info(timeout, State) -> - ok = leveled_penciller:pcl_workforclerk(State#state.owner), +handle_info(timeout, State = #state{owner = PCL}) when is_pid(PCL) -> + ok = leveled_penciller:pcl_workforclerk(PCL), % When handling work, the clerk can collect a large number of binary % references, so proactively GC this process before receiving any future % work. In under pressure clusters, clerks with large binary memory @@ -207,7 +209,7 @@ merge(SrcLevel, Manifest, RootPath, OptsSST) -> [SrcLevel + 1, FCnt, MnHBS, MnHS, MnLHS, MnBVHS]) end, SelectMethod = - case leveled_rand:uniform(100) of + case rand:uniform(100) of R when R =< ?GROOMING_PERC -> {grooming, fun grooming_scorer/1}; _ -> @@ -220,16 +222,22 @@ merge(SrcLevel, Manifest, RootPath, OptsSST) -> leveled_pmanifest:merge_lookup( Manifest, SrcLevel + 1, - Src#manifest_entry.start_key, - Src#manifest_entry.end_key + leveled_pmanifest:entry_startkey(Src), + leveled_pmanifest:entry_endkey(Src) ), Candidates = length(SinkList), leveled_log:log(pc008, [SrcLevel, Candidates]), case Candidates of 0 -> NewLevel = SrcLevel + 1, - leveled_log:log(pc009, [Src#manifest_entry.filename, NewLevel]), - leveled_sst:sst_switchlevels(Src#manifest_entry.owner, NewLevel), + leveled_log:log( + pc009, + [leveled_pmanifest:entry_filename(Src), NewLevel] + ), + leveled_sst:sst_switchlevels( + leveled_pmanifest:entry_owner(Src), + NewLevel + ), Man0 = leveled_pmanifest:switch_manifest_entry( Manifest, @@ -249,7 +257,11 @@ merge(SrcLevel, Manifest, RootPath, OptsSST) -> notify_deletions([], _Penciller) -> ok; notify_deletions([Head|Tail], Penciller) -> - ok = leveled_sst:sst_setfordelete(Head#manifest_entry.owner, Penciller), + ok = + leveled_sst:sst_setfordelete( + leveled_pmanifest:entry_owner(Head), + Penciller + ), notify_deletions(Tail, Penciller). @@ -259,9 +271,12 @@ notify_deletions([Head|Tail], Penciller) -> %% SrcLevel is the level of the src sst file, the sink should be srcLevel + 1 perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN, OptsSST) -> - leveled_log:log(pc010, [Src#manifest_entry.filename, NewSQN]), + leveled_log:log(pc010, [leveled_pmanifest:entry_filename(Src), NewSQN]), SrcList = [{next, Src, all}], - MaxSQN = leveled_sst:sst_getmaxsequencenumber(Src#manifest_entry.owner), + MaxSQN = + leveled_sst:sst_getmaxsequencenumber( + leveled_pmanifest:entry_owner(Src) + ), SinkLevel = SrcLevel + 1, SinkBasement = leveled_pmanifest:is_basement(Manifest, SinkLevel), Additions = @@ -319,13 +334,8 @@ do_merge(KL1, KL2, SinkLevel, SinkB, RP, NewSQN, MaxSQN, OptsSST, Additions) -> {ok, Pid, Reply, Bloom} -> {{KL1Rem, KL2Rem}, SmallestKey, HighestKey} = Reply, Entry = - #manifest_entry{ - start_key=SmallestKey, - end_key=HighestKey, - owner=Pid, - filename=FileName, - bloom=Bloom - }, + leveled_pmanifest:new_entry( + SmallestKey, HighestKey, Pid, FileName, Bloom), leveled_log:log_timer(pc015, [], TS1), do_merge( KL1Rem, KL2Rem, @@ -340,7 +350,8 @@ do_merge(KL1, KL2, SinkLevel, SinkB, RP, NewSQN, MaxSQN, OptsSST, Additions) -> list(leveled_pmanifest:manifest_entry())) -> leveled_pmanifest:manifest_entry(). grooming_scorer([ME | MEs]) -> - InitTombCount = leveled_sst:sst_gettombcount(ME#manifest_entry.owner), + InitTombCount = + leveled_sst:sst_gettombcount(leveled_pmanifest:entry_owner(ME)), {HighestTC, BestME} = grooming_scorer(InitTombCount, ME, MEs), leveled_log:log(pc024, [HighestTC]), BestME. @@ -348,7 +359,8 @@ grooming_scorer([ME | MEs]) -> grooming_scorer(HighestTC, BestME, []) -> {HighestTC, BestME}; grooming_scorer(HighestTC, BestME, [ME | MEs]) -> - TombCount = leveled_sst:sst_gettombcount(ME#manifest_entry.owner), + TombCount = + leveled_sst:sst_gettombcount(leveled_pmanifest:entry_owner(ME)), case TombCount > HighestTC of true -> grooming_scorer(TombCount, ME, MEs); @@ -385,11 +397,17 @@ generate_randomkeys(Count, Acc, BucketLow, BRange) -> BNumber = lists:flatten( io_lib:format("~4..0B", - [BucketLow + leveled_rand:uniform(BRange)])), + [BucketLow + rand:uniform(BRange)])), KNumber = lists:flatten( - io_lib:format("~4..0B", [leveled_rand:uniform(1000)])), - K = {o, "Bucket" ++ BNumber, "Key" ++ KNumber, null}, + io_lib:format("~4..0B", [rand:uniform(1000)])), + K = + { + o, + list_to_binary("Bucket" ++ BNumber), + list_to_binary("Key" ++ KNumber), + null + }, RandKey = {K, {Count + 1, {active, infinity}, leveled_codec:segment_hash(K), @@ -415,7 +433,6 @@ grooming_score_test() -> 3, 999999, #sst_options{}, - true, true), {ok, PidL3_1B, _, _} = leveled_sst:sst_newmerge("test/test_area/ledger_files/", @@ -427,7 +444,6 @@ grooming_score_test() -> 3, 999999, #sst_options{}, - true, true), {ok, PidL3_2, _, _} = @@ -439,102 +455,116 @@ grooming_score_test() -> 3, 999999, #sst_options{}, - true, true), - {ok, PidL3_2NC, _, _} = - leveled_sst:sst_newmerge("test/test_area/ledger_files/", - "2NC_L3.sst", - KL3_L3, - KL4_L3, - false, - 3, - 999999, - #sst_options{}, - true, - false), - - ME1 = #manifest_entry{owner=PidL3_1}, - ME1B = #manifest_entry{owner=PidL3_1B}, - ME2 = #manifest_entry{owner=PidL3_2}, - ME2NC = #manifest_entry{owner=PidL3_2NC}, + DSK = {o, <<"B">>, <<"SK">>, null}, + DEK = {o, <<"E">>, <<"EK">>, null}, + ME1 = leveled_pmanifest:new_entry(DSK, DEK, PidL3_1, "dummyL3_1", none), + ME1B = leveled_pmanifest:new_entry(DSK, DEK, PidL3_1B, "dummyL3_1B", none), + ME2 = leveled_pmanifest:new_entry(DSK, DEK, PidL3_2, "dummyL3_2", none), ?assertMatch(ME1, grooming_scorer([ME1, ME2])), ?assertMatch(ME1, grooming_scorer([ME2, ME1])), % prefer the file with the tombstone - ?assertMatch(ME2NC, grooming_scorer([ME1, ME2NC])), - ?assertMatch(ME2NC, grooming_scorer([ME2NC, ME1])), - % not_counted > 1 - we will merge files in unexpected (i.e. legacy) - % format first ?assertMatch(ME1B, grooming_scorer([ME1B, ME2])), ?assertMatch(ME2, grooming_scorer([ME2, ME1B])), % If the file with the tombstone is in the basement, it will have % no tombstone so the first file will be chosen lists:foreach(fun(P) -> leveled_sst:sst_clear(P) end, - [PidL3_1, PidL3_1B, PidL3_2, PidL3_2NC]). + [PidL3_1, PidL3_1B, PidL3_2]). merge_file_test() -> ok = filelib:ensure_dir("test/test_area/ledger_files/"), KL1_L1 = lists:sort(generate_randomkeys(8000, 0, 1000)), {ok, PidL1_1, _, _} = - leveled_sst:sst_new("test/test_area/ledger_files/", - "KL1_L1.sst", - 1, - KL1_L1, - 999999, - #sst_options{}), + leveled_sst:sst_new( + "test/test_area/ledger_files/", + "KL1_L1.sst", + 1, + KL1_L1, + 999999, + #sst_options{} + ), KL1_L2 = lists:sort(generate_randomkeys(8000, 0, 250)), {ok, PidL2_1, _, _} = - leveled_sst:sst_new("test/test_area/ledger_files/", - "KL1_L2.sst", - 2, - KL1_L2, - 999999, - #sst_options{}), + leveled_sst:sst_new( + "test/test_area/ledger_files/", + "KL1_L2.sst", + 2, + KL1_L2, + 999999, + #sst_options{} + ), KL2_L2 = lists:sort(generate_randomkeys(8000, 250, 250)), {ok, PidL2_2, _, _} = - leveled_sst:sst_new("test/test_area/ledger_files/", - "KL2_L2.sst", - 2, - KL2_L2, - 999999, - #sst_options{press_method = lz4}), + leveled_sst:sst_new( + "test/test_area/ledger_files/", + "KL2_L2.sst", + 2, + KL2_L2, + 999999, + #sst_options{press_method = lz4} + ), KL3_L2 = lists:sort(generate_randomkeys(8000, 500, 250)), {ok, PidL2_3, _, _} = - leveled_sst:sst_new("test/test_area/ledger_files/", - "KL3_L2.sst", - 2, - KL3_L2, - 999999, - #sst_options{press_method = lz4}), + leveled_sst:sst_new( + "test/test_area/ledger_files/", + "KL3_L2.sst", + 2, + KL3_L2, + 999999, + #sst_options{press_method = lz4} + ), KL4_L2 = lists:sort(generate_randomkeys(8000, 750, 250)), {ok, PidL2_4, _, _} = - leveled_sst:sst_new("test/test_area/ledger_files/", - "KL4_L2.sst", - 2, - KL4_L2, - 999999, - #sst_options{press_method = lz4}), - E1 = #manifest_entry{owner = PidL1_1, - filename = "./KL1_L1.sst", - end_key = lists:last(KL1_L1), - start_key = lists:nth(1, KL1_L1)}, - E2 = #manifest_entry{owner = PidL2_1, - filename = "./KL1_L2.sst", - end_key = lists:last(KL1_L2), - start_key = lists:nth(1, KL1_L2)}, - E3 = #manifest_entry{owner = PidL2_2, - filename = "./KL2_L2.sst", - end_key = lists:last(KL2_L2), - start_key = lists:nth(1, KL2_L2)}, - E4 = #manifest_entry{owner = PidL2_3, - filename = "./KL3_L2.sst", - end_key = lists:last(KL3_L2), - start_key = lists:nth(1, KL3_L2)}, - E5 = #manifest_entry{owner = PidL2_4, - filename = "./KL4_L2.sst", - end_key = lists:last(KL4_L2), - start_key = lists:nth(1, KL4_L2)}, + leveled_sst:sst_new( + "test/test_area/ledger_files/", + "KL4_L2.sst", + 2, + KL4_L2, + 999999, + #sst_options{press_method = lz4} + ), + E1 = + leveled_pmanifest:new_entry( + lists:nth(1, KL1_L1), + lists:last(KL1_L1), + PidL1_1, + "./KL1_L1.sst", + none + ), + E2 = + leveled_pmanifest:new_entry( + lists:nth(1, KL1_L2), + lists:last(KL1_L2), + PidL2_1, + "./KL1_L2.sst", + none + ), + E3 = + leveled_pmanifest:new_entry( + lists:nth(1, KL2_L2), + lists:last(KL2_L2), + PidL2_2, + "./KL2_L2.sst", + none + ), + E4 = + leveled_pmanifest:new_entry( + lists:nth(1, KL3_L2), + lists:last(KL3_L2), + PidL2_3, + "./KL3_L2.sst", + none + ), + E5 = + leveled_pmanifest:new_entry( + lists:nth(1, KL4_L2), + lists:last(KL4_L2), + PidL2_4, + "./KL4_L2.sst", + none + ), Man0 = leveled_pmanifest:new_manifest(), Man1 = leveled_pmanifest:insert_manifest_entry(Man0, 1, 2, E2), diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 64c325a..0a432ae 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -202,6 +202,10 @@ -export([pcl_getsstpids/1, pcl_getclerkpid/1]). +% Test functions to ignore for equalizer +-eqwalizer({nowarn_function, fetch_status_test/0}). +-eqwalizer({nowarn_function, maybe_pause_push/2}). + -ifdef(TEST). -export([clean_testdir/1]). -endif. @@ -226,6 +230,8 @@ -record(state, {manifest :: leveled_pmanifest:manifest() | undefined | redacted, + % Can be undefined in some snapshots, but must always be + % defined when using a primary penciller query_manifest :: {list(), leveled_codec:ledger_key(), @@ -233,6 +239,7 @@ % Slimmed down version of the manifest containing part % related to specific query, and the StartKey/EndKey % used to extract this part + % Only found in snapshots persisted_sqn = 0 :: integer(), % The highest SQN persisted ledger_sqn = 0 :: integer(), % The highest SQN added to L0 @@ -241,7 +248,8 @@ levelzero_constructor :: pid() | undefined, levelzero_cache = [] :: levelzero_cache() | redacted, levelzero_size = 0 :: integer(), - levelzero_maxcachesize :: integer() | undefined, + levelzero_maxcachesize = 0 :: non_neg_integer(), + % Will default to 0 in snapshots (when not required) levelzero_cointoss = false :: boolean(), levelzero_index :: leveled_pmem:index_array() | undefined | redacted, @@ -249,6 +257,7 @@ root_path = "test" :: string(), clerk :: pid() | undefined, + % Can only be undefined in a snapshot is_snapshot = false :: boolean(), snapshot_fully_loaded = false :: boolean(), @@ -274,14 +283,16 @@ -type penciller_options() :: #penciller_options{}. --type bookies_memory() :: {tuple()|empty_cache, - array:array()|empty_array, - integer()|infinity, - integer()}. +-type bookies_memory() :: + { + ets:table()|tuple()|empty_cache, + array:array()|empty_array, + integer()|infinity, + integer() + }. -type pcl_state() :: #state{}. -type levelzero_cacheentry() :: {pos_integer(), leveled_tree:leveled_tree()}. -type levelzero_cache() :: list(levelzero_cacheentry()). --type bad_ledgerkey() :: list(). -type sqn_check() :: current|replaced|missing. -type sst_fetchfun() :: fun((pid(), @@ -291,9 +302,9 @@ leveled_codec:ledger_kv()|not_present). -type levelzero_returnfun() :: fun((levelzero_cacheentry()) -> ok). -type pclacc_fun() :: - fun((leveled_codec:ledger_key(), + fun((leveled_codec:object_key(), leveled_codec:ledger_value(), - term()) -> term()). + dynamic()) -> dynamic()). -type sst_options() :: #sst_options{}. -export_type( @@ -318,13 +329,17 @@ %% query is run against the level zero space and just the query results are %% copied into the clone. pcl_start(PCLopts) -> - gen_server:start_link(?MODULE, [leveled_log:get_opts(), PCLopts], []). + {ok, Pcl} = + gen_server:start_link(?MODULE, [leveled_log:get_opts(), PCLopts], []), + {ok, Pcl}. -spec pcl_snapstart(penciller_options()) -> {ok, pid()}. %% @doc %% Don't link to the bookie - this is a snpashot pcl_snapstart(PCLopts) -> - gen_server:start(?MODULE, [leveled_log:get_opts(), PCLopts], []). + {ok, PclSnap} = + gen_server:start(?MODULE, [leveled_log:get_opts(), PCLopts], []), + {ok, PclSnap}. -spec pcl_pushmem(pid(), bookies_memory()) -> ok|returned. %% @doc @@ -371,16 +386,20 @@ pcl_fetchlevelzero(Pid, Slot, ReturnFun) -> boolean()) -> leveled_codec:ledger_kv()|not_present. %% @doc %% Fetch a key, return the first (highest SQN) occurrence of that Key along -%% with the value. +%% with the value. %% %% Hash should be result of leveled_codec:segment_hash(Key) +%% The L0Index cannot be used when in head_only mode - as although such keys +%% are fetchable no index entries are created whne added to the ledger cache pcl_fetch(Pid, Key, Hash, UseL0Index) -> gen_server:call(Pid, {fetch, Key, Hash, UseL0Index}, infinity). --spec pcl_fetchkeys(pid(), - leveled_codec:ledger_key(), - leveled_codec:ledger_key(), - pclacc_fun(), any(), as_pcl|by_runner) -> any(). +-spec pcl_fetchkeys( + pid(), + leveled_codec:query_key(), + leveled_codec:query_key(), + pclacc_fun(), + dynamic()) -> dynamic(). %% @doc %% Run a range query between StartKey and EndKey (inclusive). This will cover %% all keys in the range - so must only be run against snapshots of the @@ -393,6 +412,19 @@ pcl_fetch(Pid, Key, Hash, UseL0Index) -> pcl_fetchkeys(Pid, StartKey, EndKey, AccFun, InitAcc) -> pcl_fetchkeys(Pid, StartKey, EndKey, AccFun, InitAcc, as_pcl). +-spec pcl_fetchkeys + (pid(), + leveled_codec:query_key(), + leveled_codec:query_key(), + pclacc_fun(), + dynamic(), + as_pcl) -> dynamic(); + (pid(), + leveled_codec:query_key(), + leveled_codec:query_key(), + pclacc_fun(), + dynamic(), + by_runner) -> fun(() -> dynamic()). pcl_fetchkeys(Pid, StartKey, EndKey, AccFun, InitAcc, By) -> gen_server:call(Pid, {fetch_keys, @@ -403,13 +435,14 @@ pcl_fetchkeys(Pid, StartKey, EndKey, AccFun, InitAcc, By) -> infinity). --spec pcl_fetchkeysbysegment(pid(), - leveled_codec:ledger_key(), - leveled_codec:ledger_key(), - pclacc_fun(), any(), - leveled_codec:segment_list(), - false | leveled_codec:lastmod_range(), - boolean()) -> any(). +-spec pcl_fetchkeysbysegment( + pid(), + leveled_codec:ledger_key(), + leveled_codec:ledger_key(), + pclacc_fun(), any(), + leveled_codec:segment_list(), + false | leveled_codec:lastmod_range(), + boolean()) -> any(). %% @doc %% Run a range query between StartKey and EndKey (inclusive). This will cover %% all keys in the range - so must only be run against snapshots of the @@ -440,26 +473,26 @@ pcl_fetchkeysbysegment(Pid, StartKey, EndKey, AccFun, InitAcc, by_runner}, infinity). --spec pcl_fetchnextkey(pid(), - leveled_codec:ledger_key(), - leveled_codec:ledger_key(), - pclacc_fun(), any()) -> any(). +-spec pcl_fetchnextkey( + pid(), + leveled_codec:ledger_key(), + leveled_codec:ledger_key(), + pclacc_fun(), any()) -> any(). %% @doc %% Run a range query between StartKey and EndKey (inclusive). This has the %% same constraints as pcl_fetchkeys/5, but will only return the first key %% found in erlang term order. pcl_fetchnextkey(Pid, StartKey, EndKey, AccFun, InitAcc) -> - gen_server:call(Pid, - {fetch_keys, - StartKey, EndKey, - AccFun, InitAcc, - false, false, 1, - as_pcl}, - infinity). + gen_server:call( + Pid, + {fetch_keys, + StartKey, EndKey, AccFun, InitAcc, false, false, 1, as_pcl + }, + infinity + ). --spec pcl_checksequencenumber(pid(), - leveled_codec:ledger_key()|bad_ledgerkey(), - integer()) -> sqn_check(). +-spec pcl_checksequencenumber( + pid(), leveled_codec:ledger_key(), integer()) -> sqn_check(). %% @doc %% Check if the sequence number of the passed key is not replaced by a change %% after the passed sequence number. Will return: @@ -598,13 +631,13 @@ pcl_checkforwork(Pid) -> pcl_loglevel(Pid, LogLevel) -> gen_server:cast(Pid, {log_level, LogLevel}). --spec pcl_addlogs(pid(), list(string())) -> ok. +-spec pcl_addlogs(pid(), list(atom())) -> ok. %% @doc %% Add to the list of forced logs, a list of more forced logs pcl_addlogs(Pid, ForcedLogs) -> gen_server:cast(Pid, {add_logs, ForcedLogs}). --spec pcl_removelogs(pid(), list(string())) -> ok. +-spec pcl_removelogs(pid(), list(atom())) -> ok. %% @doc %% Remove from the list of forced logs, a list of forced logs pcl_removelogs(Pid, ForcedLogs) -> @@ -628,13 +661,14 @@ pcl_getclerkpid(Pid) -> init([LogOpts, PCLopts]) -> leveled_log:save(LogOpts), - leveled_rand:seed(), case {PCLopts#penciller_options.root_path, PCLopts#penciller_options.start_snapshot, PCLopts#penciller_options.snapshot_query, - PCLopts#penciller_options.bookies_mem} of - {undefined, _Snapshot=true, Query, BookiesMem} -> - SrcPenciller = PCLopts#penciller_options.source_penciller, + PCLopts#penciller_options.bookies_mem, + PCLopts#penciller_options.source_penciller + } of + {undefined, _Snapshot=true, Query, BookiesMem, SrcPenciller} + when ?IS_DEF(BookiesMem), ?IS_DEF(SrcPenciller) -> LongRunning = PCLopts#penciller_options.snapshot_longrunning, %% monitor the bookie, and close the snapshot when bookie %% exits @@ -650,7 +684,7 @@ init([LogOpts, PCLopts]) -> clerk = undefined, bookie_monref = BookieMonitor, source_penciller = SrcPenciller}}; - {_RootPath, _Snapshot=false, _Q, _BM} -> + {_RootPath, _Snapshot=false, _Q, _BM, _SP} -> start_from_file(PCLopts) end. @@ -722,7 +756,9 @@ handle_call({push_mem, {LedgerTable, PushedIdx, MinSQN, MaxSQN}}, ledger_sqn = UpdMaxSQN}} end end; -handle_call({fetch, Key, Hash, UseL0Index}, _From, State) -> +handle_call( + {fetch, Key, Hash, UseL0Index}, _From, State = #state{manifest = M}) + when ?IS_DEF(M) -> L0Idx = case UseL0Index of true -> @@ -732,20 +768,25 @@ handle_call({fetch, Key, Hash, UseL0Index}, _From, State) -> end, R = timed_fetch_mem( - Key, Hash, State#state.manifest, - State#state.levelzero_cache, L0Idx, - State#state.monitor), + Key, + Hash, + M, + State#state.levelzero_cache, + L0Idx, + State#state.monitor + ), {reply, R, State}; -handle_call({check_sqn, Key, Hash, SQN}, _From, State) -> +handle_call( + {check_sqn, Key, Hash, SQN}, + _From, + State = #state{manifest = M, levelzero_cache = L0C, levelzero_index = L0I}) + % This is either a primary penciller, or snapshot taken without a query + % so that it contains the full level zero. + % Not to be used in head_only mode (where levelzero_index may not be + % complete) + when ?IS_DEF(M), ?IS_DEF(L0C), ?IS_DEF(L0I) -> {reply, - compare_to_sqn( - fetch_sqn( - Key, - Hash, - State#state.manifest, - State#state.levelzero_cache, - State#state.levelzero_index), - SQN), + compare_to_sqn(fetch_sqn(Key, Hash, M, L0C, L0I), SQN), State}; handle_call({fetch_keys, StartKey, EndKey, @@ -786,7 +827,10 @@ handle_call({fetch_keys, leveled_sst:extract_hash( leveled_codec:strip_to_segmentonly(LKV)), case CheckSeg of - CheckSeg when CheckSeg >= Min, CheckSeg =< Max -> + CheckSeg + when is_integer(CheckSeg), + CheckSeg >= Min, + CheckSeg =< Max -> CheckFun(CheckSeg); _ -> false @@ -827,8 +871,11 @@ handle_call({fetch_keys, end; handle_call(get_startup_sqn, _From, State) -> {reply, State#state.persisted_sqn, State}; -handle_call({register_snapshot, Snapshot, Query, BookiesMem, LongRunning}, - _From, State) -> +handle_call( + {register_snapshot, Snapshot, Query, BookiesMem, LongRunning}, + _From, + State = #state{manifest = Manifest}) + when ?IS_DEF(Manifest) -> % Register and load a snapshot % % For setup of the snapshot to be efficient should pass a query @@ -843,8 +890,7 @@ handle_call({register_snapshot, Snapshot, Query, BookiesMem, LongRunning}, false -> State#state.snaptimeout_short end, - Manifest0 = - leveled_pmanifest:add_snapshot(State#state.manifest, Snapshot, TimeO), + Manifest0 = leveled_pmanifest:add_snapshot(Manifest, Snapshot, TimeO), {BookieIncrTree, BookieIdx, MinSQN, MaxSQN} = BookiesMem, LM1Cache = @@ -869,7 +915,7 @@ handle_call({register_snapshot, Snapshot, Query, BookiesMem, LongRunning}, ledger_sqn = UpdMaxSQN, levelzero_size = UpdSize, persisted_sqn = State#state.persisted_sqn}, - leveled_pmanifest:copy_manifest(State#state.manifest), + leveled_pmanifest:copy_manifest(Manifest), undefined}; {StartKey, EndKey} -> SW = os:timestamp(), @@ -885,7 +931,7 @@ handle_call({register_snapshot, Snapshot, Query, BookiesMem, LongRunning}, persisted_sqn = State#state.persisted_sqn}, undefined, {leveled_pmanifest:query_manifest( - State#state.manifest, StartKey, EndKey), + Manifest, StartKey, EndKey), StartKey, EndKey}}; undefined -> @@ -911,20 +957,29 @@ handle_call({register_snapshot, Snapshot, Query, BookiesMem, LongRunning}, levelzero_size = UpdSize, ledger_sqn = UpdMaxSQN, persisted_sqn = State#state.persisted_sqn}, - leveled_pmanifest:copy_manifest(State#state.manifest), + leveled_pmanifest:copy_manifest(Manifest), undefined} end, {reply, {ok, - CloneState#state{snapshot_fully_loaded = true, - snapshot_time = leveled_util:integer_now(), - manifest = ManifestClone, - query_manifest = QueryManifest}}, + CloneState#state{ + snapshot_fully_loaded = true, + snapshot_time = leveled_util:integer_now(), + manifest = ManifestClone, + query_manifest = QueryManifest + } + }, State#state{manifest = Manifest0}}; handle_call(close, _From, State=#state{is_snapshot=Snap}) when Snap == true -> ok = pcl_releasesnapshot(State#state.source_penciller, self()), {stop, normal, ok, State}; -handle_call(close, From, State) -> +handle_call( + close, + From, + State = #state{manifest = Manifest, clerk = Clerk, levelzero_cache = L0C}) + % By definition not a snapshot (as snapshot covered by clause above), + % so manifest, clerk and cache must all be present + when ?IS_DEF(Manifest), ?IS_DEF(Clerk), ?IS_DEF(L0C) -> % Level 0 files lie outside of the manifest, and so if there is no L0 % file present it is safe to write the current contents of memory. If % there is a L0 file present - then the memory can be dropped (it is @@ -934,19 +989,18 @@ handle_call(close, From, State) -> % % The penciller should close each file in the manifest, and call a close % on the clerk. - ok = leveled_pclerk:clerk_close(State#state.clerk), + ok = leveled_pclerk:clerk_close(Clerk), leveled_log:log(p0008, [close]), L0Left = State#state.levelzero_size > 0, case (not State#state.levelzero_pending and L0Left) of true -> - Man0 = State#state.manifest, {Constructor, _} = roll_memory( - leveled_pmanifest:get_manifest_sqn(Man0) + 1, + leveled_pmanifest:get_manifest_sqn(Manifest) + 1, State#state.ledger_sqn, State#state.root_path, - State#state.levelzero_cache, - length(State#state.levelzero_cache), + L0C, + length(L0C), State#state.sst_options, true), ok = leveled_sst:sst_close(Constructor); @@ -955,49 +1009,59 @@ handle_call(close, From, State) -> end, gen_server:cast(self(), {maybe_defer_shutdown, close, From}), {noreply, State}; -handle_call(doom, From, State) -> +handle_call( + doom, From, State = #state{clerk = Clerk}) + when ?IS_DEF(Clerk) -> leveled_log:log(p0030, []), - ok = leveled_pclerk:clerk_close(State#state.clerk), + ok = leveled_pclerk:clerk_close(Clerk), gen_server:cast(self(), {maybe_defer_shutdown, doom, From}), {noreply, State}; -handle_call({checkbloom_fortest, Key, Hash}, _From, State) -> - Manifest = State#state.manifest, +handle_call( + {checkbloom_fortest, Key, Hash}, _From, State = #state{manifest = Man}) + when ?IS_DEF(Man) -> FoldFun = fun(Level, Acc) -> case Acc of true -> true; false -> - case leveled_pmanifest:key_lookup(Manifest, Level, Key) of + case leveled_pmanifest:key_lookup(Man, Level, Key) of false -> false; FP -> - leveled_pmanifest:check_bloom(Manifest, FP, Hash) + leveled_pmanifest:check_bloom(Man, FP, Hash) end end end, {reply, lists:foldl(FoldFun, false, lists:seq(0, ?MAX_LEVELS)), State}; -handle_call(check_for_work, _From, State) -> - {_WL, WC} = leveled_pmanifest:check_for_work(State#state.manifest), +handle_call( + check_for_work, _From, State = #state{manifest = Manifest}) + when ?IS_DEF(Manifest) -> + {_WL, WC} = leveled_pmanifest:check_for_work(Manifest), {reply, WC > 0, State}; handle_call(persisted_sqn, _From, State) -> {reply, State#state.persisted_sqn, State}; -handle_call(get_sstpids, _From, State) -> - {reply, leveled_pmanifest:get_sstpids(State#state.manifest), State}; +handle_call( + get_sstpids, _From, State = #state{manifest = Manifest}) + when ?IS_DEF(Manifest) -> + {reply, leveled_pmanifest:get_sstpids(Manifest), State}; handle_call(get_clerkpid, _From, State) -> {reply, State#state.clerk, State}. -handle_cast({manifest_change, Manifest}, State) -> +handle_cast( + {manifest_change, Manifest}, + State = #state{manifest = OldManifest, clerk = Clerk}) + when ?IS_DEF(OldManifest), ?IS_DEF(Clerk) -> NewManSQN = leveled_pmanifest:get_manifest_sqn(Manifest), - OldManSQN = leveled_pmanifest:get_manifest_sqn(State#state.manifest), + OldManSQN = leveled_pmanifest:get_manifest_sqn(OldManifest), leveled_log:log(p0041, [OldManSQN, NewManSQN]), % Only safe to update the manifest if the SQN increments if NewManSQN > OldManSQN -> ok = - leveled_pclerk:clerk_promptdeletions(State#state.clerk, NewManSQN), + leveled_pclerk:clerk_promptdeletions(Clerk, NewManSQN), % This is accepted as the new manifest, files may be deleted UpdManifest0 = - leveled_pmanifest:merge_snapshot(State#state.manifest, Manifest), + leveled_pmanifest:merge_snapshot(OldManifest, Manifest), % Need to preserve the penciller's view of snapshots stored in % the manifest UpdManifest1 = @@ -1012,9 +1076,11 @@ handle_cast({manifest_change, Manifest}, State) -> maybe_release = false, work_ongoing=false}} end; -handle_cast({release_snapshot, Snapshot}, State) -> +handle_cast( + {release_snapshot, Snapshot}, State = #state{manifest = Manifest}) + when ?IS_DEF(Manifest) -> Manifest0 = - leveled_pmanifest:release_snapshot(State#state.manifest, Snapshot), + leveled_pmanifest:release_snapshot(Manifest, Snapshot), leveled_log:log(p0003, [Snapshot]), {noreply, State#state{manifest=Manifest0}}; handle_cast({confirm_delete, PDFN, FilePid}, State=#state{is_snapshot=Snap}) @@ -1066,19 +1132,17 @@ handle_cast({confirm_delete, PDFN, FilePid}, State=#state{is_snapshot=Snap}) State#state{manifest = UpdManifest}} end end; -handle_cast({levelzero_complete, FN, StartKey, EndKey, Bloom}, State) -> +handle_cast( + {levelzero_complete, FN, StartKey, EndKey, Bloom}, + State = #state{manifest = Man, levelzero_constructor = L0C, clerk = Clerk}) + when ?IS_DEF(Man), ?IS_DEF(L0C), ?IS_DEF(Clerk) -> leveled_log:log(p0029, []), - ManEntry = #manifest_entry{start_key=StartKey, - end_key=EndKey, - owner=State#state.levelzero_constructor, - filename=FN, - bloom=Bloom}, - ManifestSQN = leveled_pmanifest:get_manifest_sqn(State#state.manifest) + 1, + ManEntry = leveled_pmanifest:new_entry(StartKey, EndKey, L0C, FN, Bloom), + ManifestSQN = leveled_pmanifest:get_manifest_sqn(Man) + 1, UpdMan = - leveled_pmanifest:insert_manifest_entry( - State#state.manifest, ManifestSQN, 0, ManEntry), + leveled_pmanifest:insert_manifest_entry(Man, ManifestSQN, 0, ManEntry), % Prompt clerk to ask about work - do this for every L0 roll - ok = leveled_pclerk:clerk_prompt(State#state.clerk), + ok = leveled_pclerk:clerk_prompt(Clerk), {noreply, State#state{levelzero_cache=[], levelzero_index=[], levelzero_pending=false, @@ -1086,17 +1150,19 @@ handle_cast({levelzero_complete, FN, StartKey, EndKey, Bloom}, State) -> levelzero_size=0, manifest=UpdMan, persisted_sqn=State#state.ledger_sqn}}; -handle_cast(work_for_clerk, State) -> +handle_cast( + work_for_clerk, + State = #state{manifest = Man, levelzero_cache = L0Cache, clerk = Clerk}) + when ?IS_DEF(Man), ?IS_DEF(L0Cache), ?IS_DEF(Clerk) -> case {(State#state.levelzero_pending or State#state.work_ongoing), - leveled_pmanifest:levelzero_present(State#state.manifest)} of + leveled_pmanifest:levelzero_present(Man)} of {true, _L0Present} -> % Work is blocked by ongoing activity {noreply, State}; {false, true} -> % If L0 present, and no work ongoing - dropping L0 to L1 is the % priority - ok = leveled_pclerk:clerk_push( - State#state.clerk, {0, State#state.manifest}), + ok = leveled_pclerk:clerk_push(Clerk, {0, Man}), {noreply, State#state{work_ongoing=true}}; {false, false} -> % No impediment to work - see what other work may be required @@ -1106,10 +1172,9 @@ handle_cast(work_for_clerk, State) -> State#state.levelzero_size, State#state.levelzero_maxcachesize, State#state.levelzero_cointoss), - CacheAlreadyFull = - leveled_pmem:cache_full(State#state.levelzero_cache), + CacheAlreadyFull = leveled_pmem:cache_full(L0Cache), % Check for a backlog of work - {WL, WC} = leveled_pmanifest:check_for_work(State#state.manifest), + {WL, WC} = leveled_pmanifest:check_for_work(Man), case {WC, (CacheAlreadyFull or CacheOverSize)} of {0, false} -> % No work required @@ -1119,15 +1184,14 @@ handle_cast(work_for_clerk, State) -> % Must not do this if there is a work backlog beyond the % tolerance, as then the backlog may never be addressed. NextSQN = - leveled_pmanifest:get_manifest_sqn( - State#state.manifest) + 1, + leveled_pmanifest:get_manifest_sqn(Man) + 1, {Constructor, none} = roll_memory( NextSQN, State#state.ledger_sqn, State#state.root_path, none, - length(State#state.levelzero_cache), + length(L0Cache), State#state.sst_options, false), {noreply, @@ -1142,15 +1206,16 @@ handle_cast(work_for_clerk, State) -> Backlog = WC >= ?WORKQUEUE_BACKLOG_TOLERANCE, leveled_log:log(p0024, [WC, Backlog, L0Full]), [TL|_Tail] = WL, - ok = - leveled_pclerk:clerk_push( - State#state.clerk, {TL, State#state.manifest}), + ok = leveled_pclerk:clerk_push(Clerk, {TL, Man}), {noreply, State#state{ work_backlog = Backlog, work_ongoing = true}} end end; -handle_cast({fetch_levelzero, Slot, ReturnFun}, State) -> +handle_cast( + {fetch_levelzero, Slot, ReturnFun}, + State = #state{levelzero_cache = L0Cache}) + when ?IS_DEF(L0Cache) -> ReturnFun(lists:nth(Slot, State#state.levelzero_cache)), {noreply, State}; handle_cast({log_level, LogLevel}, State) -> @@ -1173,8 +1238,11 @@ handle_cast({remove_logs, ForcedLogs}, State) -> SSTopts = State#state.sst_options, SSTopts0 = SSTopts#sst_options{log_options = leveled_log:get_opts()}, {noreply, State#state{sst_options = SSTopts0}}; -handle_cast({maybe_defer_shutdown, ShutdownType, From}, State) -> - case length(leveled_pmanifest:snapshot_pids(State#state.manifest)) of +handle_cast( + {maybe_defer_shutdown, ShutdownType, From}, + State = #state{manifest = Manifest}) + when ?IS_DEF(Manifest) -> + case length(leveled_pmanifest:snapshot_pids(Manifest)) of 0 -> gen_server:cast(self(), {complete_shutdown, ShutdownType, From}), {noreply, State}; @@ -1195,11 +1263,14 @@ handle_cast({maybe_defer_shutdown, ShutdownType, From}, State) -> {noreply, State} end end; -handle_cast({complete_shutdown, ShutdownType, From}, State) -> +handle_cast( + {complete_shutdown, ShutdownType, From}, + State = #state{manifest = Manifest}) + when ?IS_DEF(Manifest) -> lists:foreach( fun(Snap) -> ok = pcl_snapclose(Snap) end, - leveled_pmanifest:snapshot_pids(State#state.manifest)), - shutdown_manifest(State#state.manifest, State#state.levelzero_constructor), + leveled_pmanifest:snapshot_pids(Manifest)), + shutdown_manifest(Manifest, State#state.levelzero_constructor), case ShutdownType of doom -> ManifestFP = State#state.root_path ++ "/" ++ ?MANIFEST_FP ++ "/", @@ -1211,8 +1282,10 @@ handle_cast({complete_shutdown, ShutdownType, From}, State) -> {stop, normal, State}. %% handle the bookie stopping and stop this snapshot -handle_info({'DOWN', BookieMonRef, process, _BookiePid, _Info}, - State=#state{bookie_monref = BookieMonRef}) -> +handle_info( + {'DOWN', BookieMonRef, process, _BookiePid, _Info}, + State=#state{bookie_monref = BookieMonRef, source_penciller = SrcPCL}) + when ?IS_DEF(SrcPCL) -> ok = pcl_releasesnapshot(State#state.source_penciller, self()), {stop, normal, State}; handle_info(_Info, State) -> @@ -1262,7 +1335,12 @@ sst_filename(ManSQN, Level, Count) -> %%% Internal functions %%%============================================================================ --spec update_clerk(pid()|undefined, fun((pid(), term()) -> ok), term()) -> ok. +-type update_forcedlogs_fun() :: fun((pid(), list(atom())) -> ok). +-type update_loglevel_fun() :: fun((pid(), atom()) -> ok). + +-spec update_clerk + (pid()|undefined, update_loglevel_fun(), atom()) -> ok; + (pid()|undefined, update_forcedlogs_fun(), list(atom())) -> ok. update_clerk(undefined, _F, _T) -> ok; update_clerk(Clerk, F, T) when is_pid(Clerk) -> @@ -1272,7 +1350,12 @@ update_clerk(Clerk, F, T) when is_pid(Clerk) -> %% @doc %% Normal start of a penciller (i.e. not a snapshot), needs to read the %% filesystem and reconstruct the ledger from the files that it finds -start_from_file(PCLopts) -> +start_from_file( + PCLopts = + #penciller_options{ + root_path = RootPath, max_inmemory_tablesize = MaxTableSize} + ) + when ?IS_DEF(RootPath), ?IS_DEF(MaxTableSize) -> RootPath = PCLopts#penciller_options.root_path, MaxTableSize = PCLopts#penciller_options.max_inmemory_tablesize, OptsSST = PCLopts#penciller_options.sst_options, @@ -1288,25 +1371,13 @@ start_from_file(PCLopts) -> % vnode syncronisation issues (e.g. stop them all by default merging to % level zero concurrently) - InitState = - #state{ - clerk = MergeClerk, - root_path = RootPath, - levelzero_maxcachesize = MaxTableSize, - levelzero_cointoss = CoinToss, - levelzero_index = [], - snaptimeout_short = SnapTimeoutShort, - snaptimeout_long = SnapTimeoutLong, - sst_options = OptsSST, - monitor = Monitor}, - %% Open manifest Manifest0 = leveled_pmanifest:open_manifest(RootPath), OpenFun = fun(FN, Level) -> {ok, Pid, {_FK, _LK}, Bloom} = - leveled_sst:sst_open(sst_rootpath(RootPath), - FN, OptsSST, Level), + leveled_sst:sst_open( + sst_rootpath(RootPath), FN, OptsSST, Level), {Pid, Bloom} end, SQNFun = fun leveled_sst:sst_getmaxsequencenumber/1, @@ -1317,7 +1388,7 @@ start_from_file(PCLopts) -> leveled_log:log(p0035, [ManSQN]), %% Find any L0 files L0FN = sst_filename(ManSQN + 1, 0, 0), - {State0, FileList0} = + {{InitManifest, InitLedgerSQN, InitPersistSQN}, FileList0} = case filelib:is_file(filename:join(sst_rootpath(RootPath), L0FN)) of true -> leveled_log:log(p0015, [L0FN]), @@ -1327,32 +1398,39 @@ start_from_file(PCLopts) -> {ok, L0Pid, {L0StartKey, L0EndKey}, Bloom} = L0Open, L0SQN = leveled_sst:sst_getmaxsequencenumber(L0Pid), L0Entry = - #manifest_entry{ - start_key = L0StartKey, - end_key = L0EndKey, - filename = L0FN, - owner = L0Pid, - bloom = Bloom}, + leveled_pmanifest:new_entry( + L0StartKey, L0EndKey, L0Pid, L0FN, Bloom), Manifest2 = leveled_pmanifest:insert_manifest_entry( Manifest1, ManSQN + 1, 0, L0Entry), leveled_log:log(p0016, [L0SQN]), LedgerSQN = max(MaxSQN, L0SQN), - {InitState#state{ - manifest = Manifest2, - ledger_sqn = LedgerSQN, - persisted_sqn = LedgerSQN}, - [L0FN|FileList]}; + { + {Manifest2, LedgerSQN, LedgerSQN}, + [L0FN|FileList] + }; false -> leveled_log:log(p0017, []), - {InitState#state{ - manifest = Manifest1, - ledger_sqn = MaxSQN, - persisted_sqn = MaxSQN}, - FileList} + {{Manifest1, MaxSQN, MaxSQN}, FileList} end, ok = archive_files(RootPath, FileList0), - {ok, State0}. + { + ok, + #state{ + clerk = MergeClerk, + root_path = RootPath, + levelzero_maxcachesize = MaxTableSize, + levelzero_cointoss = CoinToss, + levelzero_index = [], + snaptimeout_short = SnapTimeoutShort, + snaptimeout_long = SnapTimeoutLong, + sst_options = OptsSST, + monitor = Monitor, + manifest = InitManifest, + ledger_sqn = InitLedgerSQN, + persisted_sqn = InitPersistSQN + } + }. -spec shutdown_manifest(leveled_pmanifest:manifest(), pid()|undefined) -> ok. @@ -1362,13 +1440,13 @@ shutdown_manifest(Manifest, L0Constructor) -> EntryCloseFun = fun(ME) -> Owner = - case is_record(ME, manifest_entry) of + case leveled_pmanifest:is_entry(ME) of true -> - ME#manifest_entry.owner; + leveled_pmanifest:entry_owner(ME); false -> case ME of {_SK, ME0} -> - ME0#manifest_entry.owner; + leveled_pmanifest:entry_owner(ME0); ME -> ME end @@ -1442,7 +1520,7 @@ maybe_cache_too_big(NewL0Size, L0MaxSize, CoinToss) -> RandomFactor = case CoinToss of true -> - case leveled_rand:uniform(?COIN_SIDECOUNT) of + case rand:uniform(?COIN_SIDECOUNT) of 1 -> true; _ -> @@ -1482,7 +1560,9 @@ roll_memory(NextManSQN, LedgerSQN, RootPath, none, CL, SSTOpts, false) -> leveled_sst:sst_newlevelzero( L0Path, L0FN, CL, FetchFun, PCL, LedgerSQN, SSTOpts), {Constructor, none}; -roll_memory(NextManSQN, LedgerSQN, RootPath, L0Cache, CL, SSTOpts, true) -> +roll_memory( + NextManSQN, LedgerSQN, RootPath, L0Cache, CL, SSTOpts, true) + when is_list(L0Cache) -> L0Path = sst_rootpath(RootPath), L0FN = sst_filename(NextManSQN, 0, 0), FetchFun = fun(Slot) -> lists:nth(Slot, L0Cache) end, @@ -1495,7 +1575,8 @@ roll_memory(NextManSQN, LedgerSQN, RootPath, L0Cache, CL, SSTOpts, true) -> -spec timed_fetch_mem( tuple(), {integer(), integer()}, - leveled_pmanifest:manifest(), list(), + leveled_pmanifest:manifest(), + list(), leveled_pmem:index_array(), leveled_monitor:monitor()) -> leveled_codec:ledger_kv()|not_found. %% @doc @@ -1622,9 +1703,11 @@ compare_to_sqn(Obj, SQN) -> boolean()) -> ok. maybelog_fetch_timing(_Monitor, _Level, no_timing, _NF) -> ok; -maybelog_fetch_timing({Pid, _StatsFreq}, _Level, FetchTime, true) -> +maybelog_fetch_timing( + {Pid, _StatsFreq}, _Level, FetchTime, true) when is_pid(Pid) -> leveled_monitor:add_stat(Pid, {pcl_fetch_update, not_found, FetchTime}); -maybelog_fetch_timing({Pid, _StatsFreq}, Level, FetchTime, _NF) -> +maybelog_fetch_timing( + {Pid, _StatsFreq}, Level, FetchTime, _NF) when is_pid(Pid) -> leveled_monitor:add_stat(Pid, {pcl_fetch_update, Level, FetchTime}). %%%============================================================================ @@ -1758,12 +1841,13 @@ find_nextkeys( {no_more_keys, FoundKVs}; find_nextkeys( Iter, {[], {BKL, BestKV}}, FoundKVs, _Ls, {W, _SW}, _SearchInfo) - when length(FoundKVs) == W - 1 -> + when length(FoundKVs) == W - 1, BestKV =/= null -> % All levels scanned, and there are now W keys (W - 1 previously found plus % the latest best key) {maps:update_with(BKL, fun tl/1, Iter), [BestKV|FoundKVs]}; find_nextkeys( - Iter, {[], {BKL, BestKV}}, FoundKVs, Ls, BatchInfo, SearchInfo) -> + Iter, {[], {BKL, BestKV}}, FoundKVs, Ls, BatchInfo, SearchInfo) + when BestKV =/= null -> % All levels scanned so this is the best key ... now loop to find more find_nextkeys( maps:update_with(BKL, fun tl/1, Iter), @@ -1828,7 +1912,7 @@ find_nextkeys( {OtherLevels, PrevBest}, FoundKVs, Ls, BI, SI); - [{Key, Val}|_RestOfKeys] -> + [{Key, Val}|_RestOfKeys] when BKV =/= null -> case leveled_codec:key_dominates({Key, Val}, BKV) of true -> find_nextkeys( @@ -1897,15 +1981,18 @@ generate_randomkeys({Count, StartSQN}) -> generate_randomkeys(0, _SQN, Acc) -> lists:reverse(Acc); generate_randomkeys(Count, SQN, Acc) -> - K = {o, - lists:concat(["Bucket", leveled_rand:uniform(1024)]), - lists:concat(["Key", leveled_rand:uniform(1024)]), - null}, - RandKey = {K, - {SQN, - {active, infinity}, - leveled_codec:segment_hash(K), - null}}, + K = + { + o, + list_to_binary(lists:concat(["Bucket", rand:uniform(1024)])), + list_to_binary(lists:concat(["Key", rand:uniform(1024)])), + null + }, + RandKey = + { + K, + {SQN, {active, infinity}, leveled_codec:segment_hash(K), null} + }, generate_randomkeys(Count - 1, SQN + 1, [RandKey|Acc]). clean_testdir(RootPath) -> @@ -1916,12 +2003,13 @@ clean_subdir(DirPath) -> case filelib:is_dir(DirPath) of true -> {ok, Files} = file:list_dir(DirPath), - lists:foreach(fun(FN) -> - File = filename:join(DirPath, FN), - ok = file:delete(File), - io:format("Success deleting ~s~n", [File]) - end, - Files); + lists:foreach( + fun(FN) -> + File = filename:join(DirPath, FN), + ok = file:delete(File), + io:format("Success deleting ~s~n", [File]) + end, + Files); false -> ok end. @@ -1929,15 +2017,18 @@ clean_subdir(DirPath) -> maybe_pause_push(PCL, KL) -> T0 = [], I0 = leveled_pmem:new_index(), - T1 = lists:foldl(fun({K, V}, {AccSL, AccIdx, MinSQN, MaxSQN}) -> - UpdSL = [{K, V}|AccSL], - SQN = leveled_codec:strip_to_seqonly({K, V}), - H = leveled_codec:segment_hash(K), - UpdIdx = leveled_pmem:prepare_for_index(AccIdx, H), - {UpdSL, UpdIdx, min(SQN, MinSQN), max(SQN, MaxSQN)} - end, - {T0, I0, infinity, 0}, - KL), + T1 = + lists:foldl( + fun({K, V}, {AccSL, AccIdx, MinSQN, MaxSQN}) -> + UpdSL = [{K, V}|AccSL], + SQN = leveled_codec:strip_to_seqonly({K, V}), + H = leveled_codec:segment_hash(K), + UpdIdx = leveled_pmem:prepare_for_index(AccIdx, H), + {UpdSL, UpdIdx, min(SQN, MinSQN), max(SQN, MaxSQN)} + end, + {T0, I0, infinity, 0}, + KL + ), SL = element(1, T1), Tree = leveled_tree:from_orderedlist(lists:ukeysort(1, SL), ?CACHE_TYPE), T2 = setelement(1, T1, Tree), @@ -1985,7 +2076,7 @@ shutdown_when_compact(Pid) -> io:format("No outstanding compaction work for ~w~n", [Pid]), pcl_close(Pid). -format_status_test() -> +fetch_status_test() -> RootPath = "test/test_area/ledger", clean_testdir(RootPath), {ok, PCL} = @@ -2038,70 +2129,134 @@ simple_server_test() -> RootPath = "test/test_area/ledger", clean_testdir(RootPath), {ok, PCL} = - pcl_start(#penciller_options{root_path=RootPath, - max_inmemory_tablesize=1000, - sst_options=#sst_options{}}), - Key1_Pre = {{o,"Bucket0001", "Key0001", null}, - {1, {active, infinity}, null}}, + pcl_start( + #penciller_options{ + root_path=RootPath, + max_inmemory_tablesize=1000, + sst_options=#sst_options{} + } + ), + Key1_Pre = + { + {o, <<"Bucket0001">>, <<"Key0001">>, null}, + {1, {active, infinity}, null} + }, Key1 = add_missing_hash(Key1_Pre), KL1 = generate_randomkeys({1000, 2}), - Key2_Pre = {{o,"Bucket0002", "Key0002", null}, - {1002, {active, infinity}, null}}, + Key2_Pre = + { + {o, <<"Bucket0002">>, <<"Key0002">>, null}, + {1002, {active, infinity}, null} + }, Key2 = add_missing_hash(Key2_Pre), KL2 = generate_randomkeys({900, 1003}), % Keep below the max table size by having 900 not 1000 - Key3_Pre = {{o,"Bucket0003", "Key0003", null}, - {2003, {active, infinity}, null}}, + Key3_Pre = + { + {o, <<"Bucket0003">>, <<"Key0003">>, null}, + {2003, {active, infinity}, null} + }, Key3 = add_missing_hash(Key3_Pre), KL3 = generate_randomkeys({1000, 2004}), - Key4_Pre = {{o,"Bucket0004", "Key0004", null}, - {3004, {active, infinity}, null}}, + Key4_Pre = + { + {o, <<"Bucket0004">>, <<"Key0004">>, null}, + {3004, {active, infinity}, null} + }, Key4 = add_missing_hash(Key4_Pre), KL4 = generate_randomkeys({1000, 3005}), ok = maybe_pause_push(PCL, [Key1]), - ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), + ?assertMatch( + Key1, + pcl_fetch(PCL, {o, <<"Bucket0001">>, <<"Key0001">>, null}) + ), ok = maybe_pause_push(PCL, KL1), - ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), + ?assertMatch( + Key1, + pcl_fetch(PCL, {o, <<"Bucket0001">>, <<"Key0001">>, null}) + ), ok = maybe_pause_push(PCL, [Key2]), - ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), - ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), + ?assertMatch( + Key1, + pcl_fetch(PCL, {o, <<"Bucket0001">>, <<"Key0001">>, null}) + ), + ?assertMatch( + Key2, + pcl_fetch(PCL, {o, <<"Bucket0002">>, <<"Key0002">>, null}) + ), ok = maybe_pause_push(PCL, KL2), - ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), + ?assertMatch( + Key2, + pcl_fetch(PCL, {o, <<"Bucket0002">>, <<"Key0002">>, null}) + ), ok = maybe_pause_push(PCL, [Key3]), - ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), - ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), - ?assertMatch(Key3, pcl_fetch(PCL, {o,"Bucket0003", "Key0003", null})), + ?assertMatch( + Key1, + pcl_fetch(PCL, {o, <<"Bucket0001">>, <<"Key0001">>, null}) + ), + ?assertMatch( + Key2, + pcl_fetch(PCL, {o, <<"Bucket0002">>, <<"Key0002">>, null}) + ), + ?assertMatch( + Key3, + pcl_fetch(PCL, {o, <<"Bucket0003">>, <<"Key0003">>, null})), - true = pcl_checkbloomtest(PCL, {o,"Bucket0001", "Key0001", null}), - true = pcl_checkbloomtest(PCL, {o,"Bucket0002", "Key0002", null}), - true = pcl_checkbloomtest(PCL, {o,"Bucket0003", "Key0003", null}), - false = pcl_checkbloomtest(PCL, {o,"Bucket9999", "Key9999", null}), + true = pcl_checkbloomtest(PCL, {o, <<"Bucket0001">>, <<"Key0001">>, null}), + true = pcl_checkbloomtest(PCL, {o, <<"Bucket0002">>, <<"Key0002">>, null}), + true = pcl_checkbloomtest(PCL, {o, <<"Bucket0003">>, <<"Key0003">>, null}), + false = + pcl_checkbloomtest(PCL, {o, <<"Bucket9999">>, <<"Key9999">>, null}), ok = shutdown_when_compact(PCL), {ok, PCLr} = - pcl_start(#penciller_options{root_path=RootPath, - max_inmemory_tablesize=1000, - sst_options=#sst_options{}}), + pcl_start( + #penciller_options{ + root_path=RootPath, + max_inmemory_tablesize=1000, + sst_options=#sst_options{} + }), ?assertMatch(2003, pcl_getstartupsequencenumber(PCLr)), - % ok = maybe_pause_push(PCLr, [Key2] ++ KL2 ++ [Key3]), - true = pcl_checkbloomtest(PCLr, {o,"Bucket0001", "Key0001", null}), - true = pcl_checkbloomtest(PCLr, {o,"Bucket0002", "Key0002", null}), - true = pcl_checkbloomtest(PCLr, {o,"Bucket0003", "Key0003", null}), - false = pcl_checkbloomtest(PCLr, {o,"Bucket9999", "Key9999", null}), + true = + pcl_checkbloomtest(PCLr, {o, <<"Bucket0001">>, <<"Key0001">>, null}), + true = + pcl_checkbloomtest(PCLr, {o, <<"Bucket0002">>, <<"Key0002">>, null}), + true = + pcl_checkbloomtest(PCLr, {o, <<"Bucket0003">>, <<"Key0003">>, null}), + false = + pcl_checkbloomtest(PCLr, {o, <<"Bucket9999">>, <<"Key9999">>, null}), - ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001", null})), - ?assertMatch(Key2, pcl_fetch(PCLr, {o,"Bucket0002", "Key0002", null})), - ?assertMatch(Key3, pcl_fetch(PCLr, {o,"Bucket0003", "Key0003", null})), + ?assertMatch( + Key1, + pcl_fetch(PCLr, {o, <<"Bucket0001">>, <<"Key0001">>, null}) + ), + ?assertMatch( + Key2, + pcl_fetch(PCLr, {o, <<"Bucket0002">>, <<"Key0002">>, null})), + ?assertMatch( + Key3, + pcl_fetch(PCLr, {o, <<"Bucket0003">>, <<"Key0003">>, null})), ok = maybe_pause_push(PCLr, KL3), ok = maybe_pause_push(PCLr, [Key4]), ok = maybe_pause_push(PCLr, KL4), - ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001", null})), - ?assertMatch(Key2, pcl_fetch(PCLr, {o,"Bucket0002", "Key0002", null})), - ?assertMatch(Key3, pcl_fetch(PCLr, {o,"Bucket0003", "Key0003", null})), - ?assertMatch(Key4, pcl_fetch(PCLr, {o,"Bucket0004", "Key0004", null})), + + ?assertMatch( + Key1, + pcl_fetch(PCLr, {o, <<"Bucket0001">>, <<"Key0001">>, null}) + ), + ?assertMatch( + Key2, + pcl_fetch(PCLr, {o, <<"Bucket0002">>, <<"Key0002">>, null})), + ?assertMatch( + Key3, + pcl_fetch(PCLr, {o, <<"Bucket0003">>, <<"Key0003">>, null})), + ?assertMatch( + Key4, + pcl_fetch(PCLr, {o, <<"Bucket0004">>, <<"Key0004">>, null}) + ), {ok, PclSnap, null} = leveled_bookie:snapshot_store( @@ -2113,33 +2268,44 @@ simple_server_test() -> undefined, false), - ?assertMatch(Key1, pcl_fetch(PclSnap, {o,"Bucket0001", "Key0001", null})), - ?assertMatch(Key2, pcl_fetch(PclSnap, {o,"Bucket0002", "Key0002", null})), - ?assertMatch(Key3, pcl_fetch(PclSnap, {o,"Bucket0003", "Key0003", null})), - ?assertMatch(Key4, pcl_fetch(PclSnap, {o,"Bucket0004", "Key0004", null})), + ?assertMatch( + Key1, + pcl_fetch(PclSnap, {o, <<"Bucket0001">>, <<"Key0001">>, null})), + ?assertMatch( + Key2, + pcl_fetch(PclSnap, {o, <<"Bucket0002">>, <<"Key0002">>, null})), + ?assertMatch( + Key3, + pcl_fetch(PclSnap, {o, <<"Bucket0003">>, <<"Key0003">>, null})), + ?assertMatch( + Key4, + pcl_fetch(PclSnap, {o, <<"Bucket0004">>, <<"Key0004">>, null})), ?assertMatch( current, pcl_checksequencenumber( - PclSnap, {o, "Bucket0001", "Key0001", null}, 1)), + PclSnap, {o, <<"Bucket0001">>, <<"Key0001">>, null}, 1)), ?assertMatch( current, pcl_checksequencenumber( - PclSnap, {o, "Bucket0002", "Key0002", null}, 1002)), + PclSnap, {o, <<"Bucket0002">>, <<"Key0002">>, null}, 1002)), ?assertMatch( current, pcl_checksequencenumber( - PclSnap, {o, "Bucket0003", "Key0003", null}, 2003)), + PclSnap, {o, <<"Bucket0003">>, <<"Key0003">>, null}, 2003)), ?assertMatch( current, pcl_checksequencenumber( - PclSnap, {o, "Bucket0004", "Key0004", null}, 3004)), + PclSnap, {o, <<"Bucket0004">>, <<"Key0004">>, null}, 3004)), % Add some more keys and confirm that check sequence number still % sees the old version in the previous snapshot, but will see the new % version in a new snapshot - Key1A_Pre = {{o,"Bucket0001", "Key0001", null}, - {4005, {active, infinity}, null}}, + Key1A_Pre = + { + {o, <<"Bucket0001">>, <<"Key0001">>, null}, + {4005, {active, infinity}, null} + }, Key1A = add_missing_hash(Key1A_Pre), KL1A = generate_randomkeys({2000, 4006}), ok = maybe_pause_push(PCLr, [Key1A]), @@ -2147,7 +2313,7 @@ simple_server_test() -> ?assertMatch( current, pcl_checksequencenumber( - PclSnap, {o, "Bucket0001", "Key0001", null}, 1)), + PclSnap, {o, <<"Bucket0001">>, <<"Key0001">>, null}, 1)), ok = pcl_close(PclSnap), {ok, PclSnap2, null} = @@ -2163,177 +2329,393 @@ simple_server_test() -> ?assertMatch( replaced, pcl_checksequencenumber( - PclSnap2, {o, "Bucket0001", "Key0001", null}, 1)), + PclSnap2, {o, <<"Bucket0001">>, <<"Key0001">>, null}, 1)), ?assertMatch( current, pcl_checksequencenumber( - PclSnap2, {o, "Bucket0001", "Key0001", null}, 4005)), + PclSnap2, {o, <<"Bucket0001">>, <<"Key0001">>, null}, 4005)), ?assertMatch( current, pcl_checksequencenumber( - PclSnap2, {o, "Bucket0002", "Key0002", null}, 1002)), + PclSnap2, {o, <<"Bucket0002">>, <<"Key0002">>, null}, 1002)), ok = pcl_close(PclSnap2), ok = pcl_close(PCLr), clean_testdir(RootPath). simple_findnextkey_test() -> - QueryArrayAsList = [ - {2, [{{o, "Bucket1", "Key1", null}, {5, {active, infinity}, {0, 0}, null}}, - {{o, "Bucket1", "Key5", null}, {4, {active, infinity}, {0, 0}, null}}]}, - {3, [{{o, "Bucket1", "Key3", null}, {3, {active, infinity}, {0, 0}, null}}]}, - {5, [{{o, "Bucket1", "Key2", null}, {2, {active, infinity}, {0, 0}, null}}]} - ], + QueryArrayAsList = + [ + {2, + [ + { + {o, <<"Bucket1">>, <<"Key1">>, null}, + {5, {active, infinity}, {0, 0}, null} + }, + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {4, {active, infinity}, {0, 0}, null} + } + ] + }, + {3, + [ + { + {o, <<"Bucket1">>, <<"Key3">>, null}, + {3, {active, infinity}, {0, 0}, null} + } + ] + }, + {5, + [ + { + {o, <<"Bucket1">>, <<"Key2">>, null}, + {2, {active, infinity}, {0, 0}, null} + } + ] + } + ], QueryArray = convert_qmanifest_tomap(QueryArrayAsList), - {Array2, KV1} = find_nextkey(QueryArray, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key1", null}, - {5, {active, infinity}, {0, 0}, null}}, - KV1), - {Array3, KV2} = find_nextkey(Array2, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key2", null}, - {2, {active, infinity}, {0, 0}, null}}, - KV2), - {Array4, KV3} = find_nextkey(Array3, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key3", null}, - {3, {active, infinity}, {0, 0}, null}}, - KV3), - {Array5, KV4} = find_nextkey(Array4, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key5", null}, - {4, {active, infinity}, {0, 0}, null}}, - KV4), - ER = find_nextkey(Array5, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), + {Array2, KV1} = + find_nextkey( + QueryArray, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key1">>, null}, + {5, {active, infinity}, {0, 0}, null} + }, + KV1), + {Array3, KV2} = + find_nextkey( + Array2, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key2">>, null}, + {2, {active, infinity}, {0, 0}, null} + }, + KV2), + {Array4, KV3} = + find_nextkey( + Array3, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key3">>, null}, + {3, {active, infinity}, {0, 0}, null} + }, + KV3), + {Array5, KV4} = + find_nextkey( + Array4, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {4, {active, infinity}, {0, 0}, null} + }, + KV4), + ER = + find_nextkey( + Array5, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), ?assertMatch(no_more_keys, ER). sqnoverlap_findnextkey_test() -> - QueryArrayAsList = [ - {2, [{{o, "Bucket1", "Key1", null}, {5, {active, infinity}, {0, 0}, null}}, - {{o, "Bucket1", "Key5", null}, {4, {active, infinity}, {0, 0}, null}}]}, - {3, [{{o, "Bucket1", "Key3", null}, {3, {active, infinity}, {0, 0}, null}}]}, - {5, [{{o, "Bucket1", "Key5", null}, {2, {active, infinity}, {0, 0}, null}}]} - ], + QueryArrayAsList = + [ + {2, + [ + { + {o, <<"Bucket1">>, <<"Key1">>, null}, + {5, {active, infinity}, {0, 0}, null} + }, + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {4, {active, infinity}, {0, 0}, null} + } + ] + }, + {3, + [ + { + {o, <<"Bucket1">>, <<"Key3">>, null}, + {3, {active, infinity}, {0, 0}, null} + } + ] + }, + {5, + [ + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {2, {active, infinity}, {0, 0}, null} + } + ] + } + ], QueryArray = convert_qmanifest_tomap(QueryArrayAsList), - {Array2, KV1} = find_nextkey(QueryArray, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key1", null}, - {5, {active, infinity}, {0, 0}, null}}, - KV1), - {Array3, KV2} = find_nextkey(Array2, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key3", null}, - {3, {active, infinity}, {0, 0}, null}}, - KV2), - {Array4, KV3} = find_nextkey(Array3, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key5", null}, - {4, {active, infinity}, {0, 0}, null}}, - KV3), - ER = find_nextkey(Array4, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), + {Array2, KV1} = + find_nextkey( + QueryArray, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key1">>, null}, + {5, {active, infinity}, {0, 0}, null} + }, + KV1), + {Array3, KV2} = + find_nextkey( + Array2, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key3">>, null}, + {3, {active, infinity}, {0, 0}, null} + }, + KV2), + {Array4, KV3} = + find_nextkey( + Array3, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {4, {active, infinity}, {0, 0}, null} + }, + KV3), + ER = + find_nextkey( + Array4, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), ?assertMatch(no_more_keys, ER). sqnoverlap_otherway_findnextkey_test() -> - QueryArrayAsList = [ - {2, [{{o, "Bucket1", "Key1", null}, {5, {active, infinity}, {0, 0}, null}}, - {{o, "Bucket1", "Key5", null}, {1, {active, infinity}, {0, 0}, null}}]}, - {3, [{{o, "Bucket1", "Key3", null}, {3, {active, infinity}, {0, 0}, null}}]}, - {5, [{{o, "Bucket1", "Key5", null}, {2, {active, infinity}, {0, 0}, null}}]} - ], + QueryArrayAsList = + [ + {2, + [ + { + {o, <<"Bucket1">>, <<"Key1">>, null}, + {5, {active, infinity}, {0, 0}, null} + }, + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {1, {active, infinity}, {0, 0}, null} + } + ] + }, + {3, + [ + { + {o, <<"Bucket1">>, <<"Key3">>, null}, + {3, {active, infinity}, {0, 0}, null} + } + ] + }, + {5, + [ + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {2, {active, infinity}, {0, 0}, null} + } + ] + } + ], QueryArray = convert_qmanifest_tomap(QueryArrayAsList), - {Array2, KV1} = find_nextkey(QueryArray, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key1", null}, - {5, {active, infinity}, {0, 0}, null}}, - KV1), - {Array3, KV2} = find_nextkey(Array2, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key3", null}, - {3, {active, infinity}, {0, 0}, null}}, - KV2), - {Array4, KV3} = find_nextkey(Array3, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key5", null}, - {2, {active, infinity}, {0, 0}, null}}, - KV3), - ER = find_nextkey(Array4, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), + {Array2, KV1} = + find_nextkey( + QueryArray, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key1">>, null}, + {5, {active, infinity}, {0, 0}, null} + }, + KV1), + {Array3, KV2} = + find_nextkey( + Array2, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key3">>, null}, + {3, {active, infinity}, {0, 0}, null} + }, + KV2), + {Array4, KV3} = + find_nextkey( + Array3, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {2, {active, infinity}, {0, 0}, null} + }, + KV3), + ER = find_nextkey( + Array4, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), ?assertMatch(no_more_keys, ER). foldwithimm_simple_test() -> Now = leveled_util:integer_now(), - QueryArrayAsList = [ - {2, [{{o, "Bucket1", "Key1", null}, - {5, {active, infinity}, 0, null}}, - {{o, "Bucket1", "Key5", null}, - {1, {active, infinity}, 0, null}}]}, - {3, [{{o, "Bucket1", "Key3", null}, - {3, {active, infinity}, 0, null}}]}, - {5, [{{o, "Bucket1", "Key5", null}, - {2, {active, infinity}, 0, null}}]} - ], + QueryArrayAsList = + [ + {2, + [ + { + {o, <<"Bucket1">>, <<"Key1">>, null}, + {5, {active, infinity}, {0, 0}, null} + }, + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {1, {active, infinity}, {0, 0}, null} + } + ] + }, + {3, + [ + { + {o, <<"Bucket1">>, <<"Key3">>, null}, + {3, {active, infinity}, {0, 0}, null} + } + ] + }, + {5, + [ + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {2, {active, infinity}, {0, 0}, null} + } + ] + } + ], QueryArray = convert_qmanifest_tomap(QueryArrayAsList), - KL1A = [{{o, "Bucket1", "Key6", null}, {7, {active, infinity}, 0, null}}, - {{o, "Bucket1", "Key1", null}, {8, {active, infinity}, 0, null}}, - {{o, "Bucket1", "Key8", null}, {9, {active, infinity}, 0, null}}], + KL1A = + [ + { + {o, <<"Bucket1">>, <<"Key6">>, null}, + {7, {active, infinity}, 0, null} + }, + { + {o, <<"Bucket1">>, <<"Key1">>, null}, + {8, {active, infinity}, 0, null} + }, + { + {o, <<"Bucket1">>, <<"Key8">>, null}, + {9, {active, infinity}, 0, null} + } + ], IMM2 = leveled_tree:from_orderedlist(lists:ukeysort(1, KL1A), ?CACHE_TYPE), - IMMiter = leveled_tree:match_range({o, "Bucket1", "Key1", null}, - {o, null, null, null}, - IMM2), - AccFun = fun(K, V, Acc) -> SQN = leveled_codec:strip_to_seqonly({K, V}), - Acc ++ [{K, SQN}] end, - Acc = keyfolder_test(IMMiter, - QueryArray, - {o, "Bucket1", "Key1", null}, {o, "Bucket1", "Key6", null}, - {AccFun, [], Now}), - ?assertMatch([{{o, "Bucket1", "Key1", null}, 8}, - {{o, "Bucket1", "Key3", null}, 3}, - {{o, "Bucket1", "Key5", null}, 2}, - {{o, "Bucket1", "Key6", null}, 7}], Acc), + IMMiter = + leveled_tree:match_range( + {o, <<"Bucket1">>, <<"Key1">>, null}, + {o, null, null, null}, + IMM2), + AccFun = + fun(K, V, Acc) -> + SQN = leveled_codec:strip_to_seqonly({K, V}), + Acc ++ [{K, SQN}] + end, + Acc = + keyfolder_test( + IMMiter, + QueryArray, + {o, <<"Bucket1">>, <<"Key1">>, null}, + {o, <<"Bucket1">>, <<"Key6">>, null}, + {AccFun, [], Now} + ), + ?assertMatch( + [ + {{o, <<"Bucket1">>, <<"Key1">>, null}, 8}, + {{o, <<"Bucket1">>, <<"Key3">>, null}, 3}, + {{o, <<"Bucket1">>, <<"Key5">>, null}, 2}, + {{o, <<"Bucket1">>, <<"Key6">>, null}, 7} + ], + Acc), - IMMiterA = [{{o, "Bucket1", "Key1", null}, - {8, {active, infinity}, 0, null}}], - AccA = keyfolder_test(IMMiterA, - QueryArray, - {o, "Bucket1", "Key1", null}, - {o, "Bucket1", "Key6", null}, - {AccFun, [], Now}), - ?assertMatch([{{o, "Bucket1", "Key1", null}, 8}, - {{o, "Bucket1", "Key3", null}, 3}, - {{o, "Bucket1", "Key5", null}, 2}], AccA), + IMMiterA = + [ + { + {o, <<"Bucket1">>, <<"Key1">>, null}, + {8, {active, infinity}, 0, null} + } + ], + AccA = + keyfolder_test( + IMMiterA, + QueryArray, + {o, <<"Bucket1">>, <<"Key1">>, null}, + {o, <<"Bucket1">>, <<"Key6">>, null}, + {AccFun, [], Now} + ), + ?assertMatch( + [ + {{o, <<"Bucket1">>, <<"Key1">>, null}, 8}, + {{o, <<"Bucket1">>, <<"Key3">>, null}, 3}, + {{o, <<"Bucket1">>, <<"Key5">>, null}, 2} + ], + AccA), - AddKV = {{o, "Bucket1", "Key4", null}, {10, {active, infinity}, 0, null}}, + AddKV = + { + {o, <<"Bucket1">>, <<"Key4">>, null}, + {10, {active, infinity}, 0, null} + }, KL1B = [AddKV|KL1A], IMM3 = leveled_tree:from_orderedlist(lists:ukeysort(1, KL1B), ?CACHE_TYPE), - IMMiterB = leveled_tree:match_range({o, "Bucket1", "Key1", null}, - {o, null, null, null}, - IMM3), + IMMiterB = + leveled_tree:match_range( + {o, <<"Bucket1">>, <<"Key1">>, null}, + {o, null, null, null}, + IMM3 + ), io:format("Compare IMM3 with QueryArrary~n"), - AccB = keyfolder_test(IMMiterB, - QueryArray, - {o, "Bucket1", "Key1", null}, {o, "Bucket1", "Key6", null}, - {AccFun, [], Now}), - ?assertMatch([{{o, "Bucket1", "Key1", null}, 8}, - {{o, "Bucket1", "Key3", null}, 3}, - {{o, "Bucket1", "Key4", null}, 10}, - {{o, "Bucket1", "Key5", null}, 2}, - {{o, "Bucket1", "Key6", null}, 7}], AccB). + AccB = + keyfolder_test( + IMMiterB, + QueryArray, + {o, <<"Bucket1">>, <<"Key1">>, null}, + {o, <<"Bucket1">>, <<"Key6">>, null}, + {AccFun, [], Now} + ), + ?assertMatch( + [ + {{o, <<"Bucket1">>, <<"Key1">>, null}, 8}, + {{o, <<"Bucket1">>, <<"Key3">>, null}, 3}, + {{o, <<"Bucket1">>, <<"Key4">>, null}, 10}, + {{o, <<"Bucket1">>, <<"Key5">>, null}, 2}, + {{o, <<"Bucket1">>, <<"Key6">>, null}, 7} + ], + AccB). create_file_test() -> {RP, Filename} = {"test/test_area/", "new_file.sst"}, diff --git a/src/leveled_pmanifest.erl b/src/leveled_pmanifest.erl index 35f437a..63d5038 100644 --- a/src/leveled_pmanifest.erl +++ b/src/leveled_pmanifest.erl @@ -7,7 +7,7 @@ %% each level. This is fine for short-lived volume tests, but as the deeper %% levels are used there will be an exponential penalty. %% -%% The originial intention was to swap out this implementation for a +%% The original intention was to swap out this implementation for a %% multi-version ETS table - but that became complex. So one of two changes %% are pending: %% - Use a single version ES cache for lower levels (and not allow snapshots to @@ -16,6 +16,10 @@ -module(leveled_pmanifest). +% Test uses extracted/edited array related to specific issue, which is not in +% an expected format +-eqwalizer({nowarn_function, potential_issue_test/0}). + -include("leveled.hrl"). -export([ @@ -49,6 +53,17 @@ get_sstpids/1 ]). +-export( + [ + new_entry/5, + entry_startkey/1, + entry_endkey/1, + entry_filename/1, + entry_owner/1, + is_entry/1 + ] +). + -export([ filepath/2 ]). @@ -85,28 +100,41 @@ -define(MANIFESTS_TO_RETAIN, 5). -define(GROOM_SAMPLE, 16). --record(manifest, {levels, - % an array of lists or trees representing the manifest - manifest_sqn = 0 :: non_neg_integer(), - % The current manifest SQN - snapshots = [] - :: list(snapshot()), - % A list of snaphots (i.e. clones) - min_snapshot_sqn = 0 :: integer(), - % The smallest snapshot manifest SQN in the snapshot - % list - pending_deletes = dict:new() :: dict:dict(), - basement :: non_neg_integer(), - % Currently the lowest level (the largest number) - blooms :: dict:dict() - }). +-record(manifest, + { + levels :: array:array(dynamic()), + % an array of lists or trees representing the manifest, where the + % list is created using the to_list function on leveled_treee + manifest_sqn = 0 :: non_neg_integer(), + % The current manifest SQN + snapshots = [] :: list(snapshot()), + % A list of snaphots (i.e. clones) + min_snapshot_sqn = 0 :: integer(), + % The smallest snapshot manifest SQN in the snapshot list + pending_deletes = new_pending_deletions() :: pending_deletions(), + basement :: non_neg_integer(), + % Currently the lowest level (the largest number) + blooms = new_blooms() :: blooms() + }). + +-record(manifest_entry, + { + start_key :: leveled_codec:object_key(), + end_key :: leveled_codec:object_key(), + owner :: pid(), + filename :: string(), + bloom = none :: leveled_ebloom:bloom() | none + } +). -type snapshot() :: {pid(), non_neg_integer(), pos_integer(), pos_integer()}. -type manifest() :: #manifest{}. -type manifest_entry() :: #manifest_entry{}. --type manifest_owner() :: pid()|list(). +-type manifest_owner() :: pid(). -type lsm_level() :: 0..7. +-type pending_deletions() :: dict:dict(). +-type blooms() :: dict:dict(). -type selector_strategy() :: random|{grooming, fun((list(manifest_entry())) -> manifest_entry())}. @@ -128,16 +156,15 @@ new_manifest() -> fun(IDX, Acc) -> array:set(IDX, leveled_tree:empty(?TREE_TYPE), Acc) end, - LevelArray1 = lists:foldl(SetLowerLevelFun, - LevelArray0, - lists:seq(2, ?MAX_LEVELS)), + LevelArray1 = + lists:foldl( + SetLowerLevelFun, LevelArray0, lists:seq(2, ?MAX_LEVELS) + ), #manifest{ levels = LevelArray1, manifest_sqn = 0, snapshots = [], - pending_deletes = dict:new(), - basement = 0, - blooms = dict:new() + basement = 0 }. -spec open_manifest(string()) -> manifest(). @@ -159,9 +186,8 @@ open_manifest(RootPath) -> Acc ++ [list_to_integer(Int)] end end, - ValidManSQNs = lists:reverse(lists:sort(lists:foldl(ExtractSQNFun, - [], - Filenames))), + ValidManSQNs = + lists:reverse(lists:sort(lists:foldl(ExtractSQNFun, [], Filenames))), open_manifestfile(RootPath, ValidManSQNs). -spec copy_manifest(manifest()) -> manifest(). @@ -171,7 +197,9 @@ open_manifest(RootPath) -> copy_manifest(Manifest) -> % Copy the manifest ensuring anything only the master process should care % about is switched to be empty - Manifest#manifest{snapshots = [], pending_deletes = dict:new()}. + Manifest#manifest{ + snapshots = [], pending_deletes = new_pending_deletions() + }. -spec load_manifest( manifest(), @@ -236,10 +264,15 @@ close_manifest(Manifest, CloseEntryFun) -> save_manifest(Manifest, RootPath) -> TFP = filepath(RootPath, Manifest#manifest.manifest_sqn, pending_manifest), AFP = filepath(RootPath, Manifest#manifest.manifest_sqn, current_manifest), - ManBin = term_to_binary(Manifest#manifest{snapshots = [], - pending_deletes = dict:new(), - min_snapshot_sqn = 0, - blooms = dict:new()}), + ManBin = + term_to_binary( + Manifest#manifest{ + snapshots = [], + pending_deletes = new_pending_deletions(), + min_snapshot_sqn = 0, + blooms = new_blooms() + } + ), CRC = erlang:crc32(ManBin), ToPersist = <>, ok = leveled_util:safe_rename(TFP, AFP, ToPersist, true), @@ -324,10 +357,12 @@ report_manifest_level(Manifest, LevelIdx) -> TotalBVBS div LevelSize} end. - --spec replace_manifest_entry(manifest(), integer(), integer(), - list()|manifest_entry(), - list()|manifest_entry()) -> manifest(). +-spec replace_manifest_entry( + manifest(), + integer(), + integer(), + list()|manifest_entry(), + list()|manifest_entry()) -> manifest(). %% @doc %% Replace a list of manifest entries in the manifest with a new set of entries %% Pass in the new manifest SQN to be used for this manifest. The list of @@ -342,28 +377,31 @@ replace_manifest_entry(Manifest, ManSQN, LevelIdx, Removals, Additions) -> UpdLevel = replace_entry(LevelIdx, Level, Removals, StrippedAdditions), leveled_log:log(pc019, ["insert", LevelIdx, UpdLevel]), PendingDeletes = - update_pendingdeletes(ManSQN, - Removals, - Manifest#manifest.pending_deletes), + update_pendingdeletes( + ManSQN, Removals, Manifest#manifest.pending_deletes), UpdLevels = array:set(LevelIdx, UpdLevel, Levels), case is_empty(LevelIdx, UpdLevel) of true -> - Manifest#manifest{levels = UpdLevels, - basement = get_basement(UpdLevels), - manifest_sqn = ManSQN, - pending_deletes = PendingDeletes, - blooms = UpdBlooms}; + Manifest#manifest{ + levels = UpdLevels, + basement = get_basement(UpdLevels), + manifest_sqn = ManSQN, + pending_deletes = PendingDeletes, + blooms = UpdBlooms + }; false -> Basement = max(LevelIdx, Manifest#manifest.basement), - Manifest#manifest{levels = UpdLevels, - basement = Basement, - manifest_sqn = ManSQN, - pending_deletes = PendingDeletes, - blooms = UpdBlooms} + Manifest#manifest{ + levels = UpdLevels, + basement = Basement, + manifest_sqn = ManSQN, + pending_deletes = PendingDeletes, + blooms = UpdBlooms + } end. --spec insert_manifest_entry(manifest(), integer(), integer(), - list()|manifest_entry()) -> manifest(). +-spec insert_manifest_entry( + manifest(), integer(), integer(), list()|manifest_entry()) -> manifest(). %% @doc %% Place a single new manifest entry into a level of the manifest, at a given %% level and manifest sequence number @@ -375,13 +413,15 @@ insert_manifest_entry(Manifest, ManSQN, LevelIdx, Entry) -> UpdLevel = add_entry(LevelIdx, Level, UpdEntry), leveled_log:log(pc019, ["insert", LevelIdx, UpdLevel]), Basement = max(LevelIdx, Manifest#manifest.basement), - Manifest#manifest{levels = array:set(LevelIdx, UpdLevel, Levels), - basement = Basement, - manifest_sqn = ManSQN, - blooms = UpdBlooms}. + Manifest#manifest{ + levels = array:set(LevelIdx, UpdLevel, Levels), + basement = Basement, + manifest_sqn = ManSQN, + blooms = UpdBlooms + }. --spec remove_manifest_entry(manifest(), integer(), integer(), - list()|manifest_entry()) -> manifest(). +-spec remove_manifest_entry( + manifest(), integer(), integer(), list()|manifest_entry()) -> manifest(). %% @doc %% Remove a manifest entry (as it has been merged into the level below) remove_manifest_entry(Manifest, ManSQN, LevelIdx, Entry) -> @@ -391,26 +431,30 @@ remove_manifest_entry(Manifest, ManSQN, LevelIdx, Entry) -> update_blooms(Entry, [], Manifest#manifest.blooms), UpdLevel = remove_entry(LevelIdx, Level, Entry), leveled_log:log(pc019, ["remove", LevelIdx, UpdLevel]), - PendingDeletes = update_pendingdeletes(ManSQN, - Entry, - Manifest#manifest.pending_deletes), + PendingDeletes = + update_pendingdeletes( + ManSQN, Entry, Manifest#manifest.pending_deletes), UpdLevels = array:set(LevelIdx, UpdLevel, Levels), case is_empty(LevelIdx, UpdLevel) of true -> - Manifest#manifest{levels = UpdLevels, - basement = get_basement(UpdLevels), - manifest_sqn = ManSQN, - pending_deletes = PendingDeletes, - blooms = UpdBlooms}; + Manifest#manifest{ + levels = UpdLevels, + basement = get_basement(UpdLevels), + manifest_sqn = ManSQN, + pending_deletes = PendingDeletes, + blooms = UpdBlooms + }; false -> - Manifest#manifest{levels = UpdLevels, - manifest_sqn = ManSQN, - pending_deletes = PendingDeletes, - blooms = UpdBlooms} + Manifest#manifest{ + levels = UpdLevels, + manifest_sqn = ManSQN, + pending_deletes = PendingDeletes, + blooms = UpdBlooms + } end. --spec switch_manifest_entry(manifest(), integer(), integer(), - list()|manifest_entry()) -> manifest(). +-spec switch_manifest_entry( + manifest(), integer(), integer(), list()|manifest_entry()) -> manifest(). %% @doc %% Switch a manifest etry from this level to the level below (i.e when there %% are no overlapping manifest entries in the level below) @@ -421,10 +465,8 @@ switch_manifest_entry(Manifest, ManSQN, SrcLevel, Entry) -> Level = array:get(SrcLevel, Levels), UpdLevel = remove_entry(SrcLevel, Level, Entry), UpdLevels = array:set(SrcLevel, UpdLevel, Levels), - insert_manifest_entry(Manifest#manifest{levels = UpdLevels}, - ManSQN, - SrcLevel + 1, - Entry). + insert_manifest_entry( + Manifest#manifest{levels = UpdLevels}, ManSQN, SrcLevel + 1, Entry). -spec get_manifest_sqn(manifest()) -> integer(). %% @doc @@ -432,8 +474,9 @@ switch_manifest_entry(Manifest, ManSQN, SrcLevel, Entry) -> get_manifest_sqn(Manifest) -> Manifest#manifest.manifest_sqn. --spec key_lookup(manifest(), integer(), leveled_codec:ledger_key()) - -> false|manifest_owner(). +-spec key_lookup( + manifest(), integer(), leveled_codec:ledger_key()) -> + false|manifest_owner(). %% @doc %% For a given key find which manifest entry covers that key at that level, %% returning false if there is no covering manifest entry at that level. @@ -442,9 +485,8 @@ key_lookup(Manifest, LevelIdx, Key) -> true -> false; false -> - key_lookup_level(LevelIdx, - array:get(LevelIdx, Manifest#manifest.levels), - Key) + key_lookup_level( + LevelIdx, array:get(LevelIdx, Manifest#manifest.levels), Key) end. -spec query_manifest( @@ -514,12 +556,12 @@ merge_lookup(Manifest, LevelIdx, StartKey, EndKey) -> %% Hence, the initial implementation is to select files to merge at random mergefile_selector(Manifest, LevelIdx, _Strategy) when LevelIdx =< 1 -> Level = array:get(LevelIdx, Manifest#manifest.levels), - lists:nth(leveled_rand:uniform(length(Level)), Level); + lists:nth(rand:uniform(length(Level)), Level); mergefile_selector(Manifest, LevelIdx, random) -> Level = leveled_tree:to_list( array:get(LevelIdx, Manifest#manifest.levels)), - {_SK, ME} = lists:nth(leveled_rand:uniform(length(Level)), Level), + {_SK, ME} = lists:nth(rand:uniform(length(Level)), Level), ME; mergefile_selector(Manifest, LevelIdx, {grooming, ScoringFun}) -> Level = @@ -527,7 +569,7 @@ mergefile_selector(Manifest, LevelIdx, {grooming, ScoringFun}) -> array:get(LevelIdx, Manifest#manifest.levels)), SelectorFun = fun(_I, Acc) -> - {_SK, ME} = lists:nth(leveled_rand:uniform(length(Level)), Level), + {_SK, ME} = lists:nth(rand:uniform(length(Level)), Level), [ME|Acc] end, Sample = @@ -549,7 +591,7 @@ merge_snapshot(PencillerManifest, ClerkManifest) -> snapshots = PencillerManifest#manifest.snapshots, min_snapshot_sqn = PencillerManifest#manifest.min_snapshot_sqn}. --spec add_snapshot(manifest(), pid()|atom(), integer()) -> manifest(). +-spec add_snapshot(manifest(), pid(), integer()) -> manifest(). %% @doc %% Add a snapshot reference to the manifest, withe rusing the pid or an atom %% known to reference a special process. The timeout should be in seconds, and @@ -589,9 +631,12 @@ release_snapshot(Manifest, Pid) -> end end end, - {SnapList0, MinSnapSQN, Hit} = lists:foldl(FilterFun, - {[], infinity, false}, - Manifest#manifest.snapshots), + {SnapList0, MinSnapSQN, Hit} = + lists:foldl( + FilterFun, + {[], infinity, false}, + Manifest#manifest.snapshots + ), case Hit of false -> leveled_log:log(p0039, [Pid, length(SnapList0), MinSnapSQN]); @@ -600,12 +645,12 @@ release_snapshot(Manifest, Pid) -> end, case SnapList0 of [] -> - Manifest#manifest{snapshots = SnapList0, - min_snapshot_sqn = 0}; - _ -> + Manifest#manifest{snapshots = SnapList0, min_snapshot_sqn = 0}; + _ when is_integer(MinSnapSQN) -> leveled_log:log(p0004, [SnapList0]), - Manifest#manifest{snapshots = SnapList0, - min_snapshot_sqn = MinSnapSQN} + Manifest#manifest{ + snapshots = SnapList0, min_snapshot_sqn = MinSnapSQN + } end. @@ -732,13 +777,46 @@ get_sstpids(Manifest) -> end, lists:foldl(FoldFun, [], lists:seq(0, Manifest#manifest.basement)). +%%%============================================================================ +%%% Manifest Entry +%%%============================================================================ + +-spec new_entry( + leveled_codec:object_key(), + leveled_codec:object_key(), + pid(), + string(), + leveled_ebloom:bloom()|none) -> manifest_entry(). +new_entry(StartKey, EndKey, Owner, FileName, Bloom) -> + #manifest_entry{ + start_key = StartKey, + end_key = EndKey, + owner = Owner, + filename = FileName, + bloom = Bloom + }. + +-spec is_entry(any()) -> boolean(). +is_entry(ME) -> is_record(ME, manifest_entry). + +-spec entry_startkey(manifest_entry()) -> leveled_codec:object_key(). +entry_startkey(ME) -> ME#manifest_entry.start_key. + +-spec entry_endkey(manifest_entry()) -> leveled_codec:object_key(). +entry_endkey(ME) -> ME#manifest_entry.end_key. + +-spec entry_owner(manifest_entry()) -> pid(). +entry_owner(ME) -> ME#manifest_entry.owner. + +-spec entry_filename(manifest_entry()) -> string(). +entry_filename(#manifest_entry{filename = FN}) when ?IS_DEF(FN)-> FN. + %%%============================================================================ %%% Internal Functions %%%============================================================================ - --spec get_manifest_entry({tuple(), manifest_entry()}|manifest_entry()) - -> manifest_entry(). +-spec get_manifest_entry( + {tuple(), manifest_entry()}|manifest_entry()) -> manifest_entry(). %% @doc %% Manifest levels can have entries of two forms, use this if only interested %% in the latter form @@ -778,10 +856,12 @@ load_level(LevelIdx, Level, LoadFun, SQNFun) -> lists:foldr(HigherLevelLoadFun, {[], 0, [], []}, Level); false -> {L0, MaxSQN, Flist, UpdBloomL} = - lists:foldr(LowerLevelLoadFun, - {[], 0, [], []}, - leveled_tree:to_list(Level)), - {leveled_tree:from_orderedlist(L0, ?TREE_TYPE, ?TREE_WIDTH), + lists:foldr( + LowerLevelLoadFun, + {[], 0, [], []}, + leveled_tree:to_list(Level) + ), + {leveled_tree:from_orderedlist(L0, ?TREE_TYPE, ?TREE_WIDTH), MaxSQN, Flist, UpdBloomL} @@ -817,9 +897,12 @@ add_entry(_LevelIdx, Level, []) -> Level; add_entry(LevelIdx, Level, Entries) when is_list(Entries) -> FirstEntry = lists:nth(1, Entries), - PredFun = pred_fun(LevelIdx, - FirstEntry#manifest_entry.start_key, - FirstEntry#manifest_entry.end_key), + PredFun = + pred_fun( + LevelIdx, + FirstEntry#manifest_entry.start_key, + FirstEntry#manifest_entry.end_key + ), case LevelIdx =< 1 of true -> {LHS, RHS} = lists:splitwith(PredFun, Level), @@ -831,9 +914,11 @@ add_entry(LevelIdx, Level, Entries) when is_list(Entries) -> {ME#manifest_entry.end_key, ME} end, Entries0 = lists:map(MapFun, Entries), - leveled_tree:from_orderedlist(lists:append([LHS, Entries0, RHS]), - ?TREE_TYPE, - ?TREE_WIDTH) + leveled_tree:from_orderedlist( + lists:append([LHS, Entries0, RHS]), + ?TREE_TYPE, + ?TREE_WIDTH + ) end. remove_entry(LevelIdx, Level, Entries) -> @@ -861,24 +946,29 @@ remove_section(LevelIdx, Level, FirstEntry, SectionLength) -> false -> {LHS, RHS} = lists:splitwith(PredFun, leveled_tree:to_list(Level)), Post = lists:nthtail(SectionLength, RHS), - leveled_tree:from_orderedlist(lists:append([LHS, Post]), - ?TREE_TYPE, - ?TREE_WIDTH) + leveled_tree:from_orderedlist( + lists:append([LHS, Post]), ?TREE_TYPE, ?TREE_WIDTH) end. replace_entry(LevelIdx, Level, Removals, Additions) when LevelIdx =< 1 -> {SectionLength, FirstEntry} = measure_removals(Removals), - PredFun = pred_fun(LevelIdx, - FirstEntry#manifest_entry.start_key, - FirstEntry#manifest_entry.end_key), + PredFun = + pred_fun( + LevelIdx, + FirstEntry#manifest_entry.start_key, + FirstEntry#manifest_entry.end_key + ), {LHS, RHS} = lists:splitwith(PredFun, Level), Post = lists:nthtail(SectionLength, RHS), lists:append([LHS, Additions, Post]); replace_entry(LevelIdx, Level, Removals, Additions) -> {SectionLength, FirstEntry} = measure_removals(Removals), - PredFun = pred_fun(LevelIdx, - FirstEntry#manifest_entry.start_key, - FirstEntry#manifest_entry.end_key), + PredFun = + pred_fun( + LevelIdx, + FirstEntry#manifest_entry.start_key, + FirstEntry#manifest_entry.end_key + ), {LHS, RHS} = lists:splitwith(PredFun, leveled_tree:to_list(Level)), Post = case RHS of @@ -898,9 +988,7 @@ replace_entry(LevelIdx, Level, Removals, Additions) -> update_pendingdeletes(ManSQN, Removals, PendingDeletes) -> DelFun = fun(E, Acc) -> - dict:store(E#manifest_entry.filename, - {ManSQN, E}, - Acc) + dict:store(E#manifest_entry.filename, {ManSQN, E}, Acc) end, Entries = case is_list(Removals) of @@ -911,10 +999,11 @@ update_pendingdeletes(ManSQN, Removals, PendingDeletes) -> end, lists:foldl(DelFun, PendingDeletes, Entries). --spec update_blooms(list()|manifest_entry(), - list()|manifest_entry(), - any()) - -> {any(), list()}. +-spec update_blooms( + list()|manifest_entry(), + list()|manifest_entry(), + blooms()) + -> {blooms(), list()}. %% @doc %% %% The manifest is a Pid-> Bloom mappping for every Pid, and this needs to @@ -984,11 +1073,12 @@ range_lookup_int(Manifest, LevelIdx, StartKey, EndKey, MakePointerFun) -> true -> []; false -> - range_lookup_level(LevelIdx, - array:get(LevelIdx, - Manifest#manifest.levels), - StartKey, - EndKey) + range_lookup_level( + LevelIdx, + array:get(LevelIdx, Manifest#manifest.levels), + StartKey, + EndKey + ) end, lists:map(MakePointerFun, Range). @@ -999,8 +1089,9 @@ range_lookup_level(LevelIdx, Level, QStartKey, QEndKey) when LevelIdx =< 1 -> end, NotAfterFun = fun(M) -> - not leveled_codec:endkey_passed(QEndKey, - M#manifest_entry.start_key) + not + leveled_codec:endkey_passed( + QEndKey, M#manifest_entry.start_key) end, {_Before, MaybeIn} = lists:splitwith(BeforeFun, Level), {In, _After} = lists:splitwith(NotAfterFun, MaybeIn), @@ -1016,7 +1107,6 @@ range_lookup_level(_LevelIdx, Level, QStartKey, QEndKey) -> ME end, lists:map(MapFun, Range). - get_basement(Levels) -> GetBaseFun = @@ -1030,7 +1120,6 @@ get_basement(Levels) -> end, lists:foldl(GetBaseFun, 0, lists:seq(0, ?MAX_LEVELS)). - filepath(RootPath, manifest) -> MFP = RootPath ++ "/" ++ ?MANIFEST_FP ++ "/", filelib:ensure_dir(MFP), @@ -1043,9 +1132,6 @@ filepath(RootPath, NewMSN, pending_manifest) -> filepath(RootPath, manifest) ++ "nonzero_" ++ integer_to_list(NewMSN) ++ "." ++ ?PENDING_FILEX. - - - open_manifestfile(_RootPath, L) when L == [] orelse L == [0] -> leveled_log:log(p0013, []), new_manifest(); @@ -1056,7 +1142,11 @@ open_manifestfile(RootPath, [TopManSQN|Rest]) -> case erlang:crc32(BinaryOfTerm) of CRC -> leveled_log:log(p0012, [TopManSQN]), - binary_to_term(BinaryOfTerm); + Manifest = binary_to_term(BinaryOfTerm), + Manifest#manifest{ + pending_deletes = new_pending_deletions(), + blooms = new_blooms() + }; _ -> leveled_log:log(p0033, [CurrManFile, "crc wonky"]), open_manifestfile(RootPath, Rest) @@ -1066,6 +1156,9 @@ seconds_now() -> {MegaNow, SecNow, _} = os:timestamp(), MegaNow * 1000000 + SecNow. +new_blooms() -> dict:new(). + +new_pending_deletions() -> dict:new(). %%%============================================================================ %%% Test @@ -1079,36 +1172,54 @@ initial_setup() -> initial_setup(single_change). initial_setup(Changes) -> - E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, - end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, - filename="Z1", - owner="pid_z1", - bloom=none}, - E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"}, - end_key={o, "Bucket1", "K71", null}, - filename="Z2", - owner="pid_z2", - bloom=none}, - E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null}, - end_key={o, "Bucket1", "K993", null}, - filename="Z3", - owner="pid_z3", - bloom=none}, - E4 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, - end_key={i, "Bucket1", {"Idx1", "Fld7"}, "K93"}, - filename="Z4", - owner="pid_z4", - bloom=none}, - E5 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld7"}, "K97"}, - end_key={o, "Bucket1", "K78", null}, - filename="Z5", - owner="pid_z5", - bloom=none}, - E6 = #manifest_entry{start_key={o, "Bucket1", "K81", null}, - end_key={o, "Bucket1", "K996", null}, - filename="Z6", - owner="pid_z6", - bloom=none}, + E1 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld1">>}, <<"K8">>}, + end_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K93">>}, + filename="Z1", + owner=list_to_pid("<0.101.0>"), + bloom=none + }, + E2 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K97">>}, + end_key={o, <<"Bucket1">>, <<"K71">>, null}, + filename="Z2", + owner=list_to_pid("<0.102.0>"), + bloom=none + }, + E3 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K75">>, null}, + end_key={o, <<"Bucket1">>, <<"K993">>, null}, + filename="Z3", + owner=list_to_pid("<0.103.0>"), + bloom=none + }, + E4 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld1">>}, <<"K8">>}, + end_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld7">>}, <<"K93">>}, + filename="Z4", + owner=list_to_pid("<0.104.0>"), + bloom=none + }, + E5 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld7">>}, <<"K97">>}, + end_key={o, <<"Bucket1">>, <<"K78">>, null}, + filename="Z5", + owner=list_to_pid("<0.105.0>"), + bloom=none + }, + E6 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K81">>, null}, + end_key={o, <<"Bucket1">>, <<"K996">>, null}, + filename="Z6", + owner=list_to_pid("<0.106.0>"), + bloom=none + }, initial_setup(Changes, E1, E2, E3, E4, E5, E6). @@ -1137,42 +1248,63 @@ initial_setup(multi_change, E1, E2, E3, E4, E5, E6) -> changeup_setup(Man6) -> - E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, - end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, - filename="Z1", - owner="pid_z1", - bloom=none}, - E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"}, - end_key={o, "Bucket1", "K71", null}, - filename="Z2", - owner="pid_z2", - bloom=none}, - E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null}, - end_key={o, "Bucket1", "K993", null}, - filename="Z3", - owner="pid_z3", - bloom=none}, + E1 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld1">>}, <<"K8">>}, + end_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K93">>}, + filename="Z1", + owner=list_to_pid("<0.101.0>"), + bloom=none + }, + E2 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K97">>}, + end_key={o, <<"Bucket1">>, <<"K71">>, null}, + filename="Z2", + owner=list_to_pid("<0.102.0>"), + bloom=none + }, + E3 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K75">>, null}, + end_key={o, <<"Bucket1">>, <<"K993">>, null}, + filename="Z3", + owner=list_to_pid("<0.103.0>"), + bloom=none + }, - E1_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld4"}, "K8"}, - end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K62"}, - owner="pid_y1", - filename="Y1", - bloom=none}, - E2_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K67"}, - end_key={o, "Bucket1", "K45", null}, - owner="pid_y2", - filename="Y2", - bloom=none}, - E3_2 = #manifest_entry{start_key={o, "Bucket1", "K47", null}, - end_key={o, "Bucket1", "K812", null}, - owner="pid_y3", - filename="Y3", - bloom=none}, - E4_2 = #manifest_entry{start_key={o, "Bucket1", "K815", null}, - end_key={o, "Bucket1", "K998", null}, - owner="pid_y4", - filename="Y4", - bloom=none}, + E1_2 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld4">>}, <<"K8">>}, + end_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K62">>}, + owner=list_to_pid("<0.201.0>"), + filename="Y1", + bloom=none + }, + E2_2 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K67">>}, + end_key={o, <<"Bucket1">>, <<"K45">>, null}, + owner=list_to_pid("<0.202.0>"), + filename="Y2", + bloom=none + }, + E3_2 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K47">>, null}, + end_key={o, <<"Bucket1">>, <<"K812">>, null}, + owner=list_to_pid("<0.203.0>"), + filename="Y3", + bloom=none + }, + E4_2 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K815">>, null}, + end_key={o, <<"Bucket1">>, <<"K998">>, null}, + owner=list_to_pid("<0.204.0>"), + filename="Y4", + bloom=none + }, Man7 = remove_manifest_entry(Man6, 2, 1, E1), Man8 = remove_manifest_entry(Man7, 2, 1, E2), @@ -1211,11 +1343,11 @@ manifest_gc_test() -> keylookup_manifest_test() -> {Man0, Man1, Man2, Man3, _Man4, _Man5, Man6} = initial_setup(), - LK1_1 = {o, "Bucket1", "K711", null}, - LK1_2 = {o, "Bucket1", "K70", null}, - LK1_3 = {o, "Bucket1", "K71", null}, - LK1_4 = {o, "Bucket1", "K75", null}, - LK1_5 = {o, "Bucket1", "K76", null}, + LK1_1 = {o, <<"Bucket1">>, <<"K711">>, null}, + LK1_2 = {o, <<"Bucket1">>, <<"K70">>, null}, + LK1_3 = {o, <<"Bucket1">>, <<"K71">>, null}, + LK1_4 = {o, <<"Bucket1">>, <<"K75">>, null}, + LK1_5 = {o, <<"Bucket1">>, <<"K76">>, null}, ?assertMatch(false, key_lookup(Man0, 1, LK1_1)), ?assertMatch(false, key_lookup(Man1, 1, LK1_1)), @@ -1223,15 +1355,19 @@ keylookup_manifest_test() -> ?assertMatch(false, key_lookup(Man3, 1, LK1_1)), ?assertMatch(false, key_lookup(Man6, 1, LK1_1)), - ?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_2)), - ?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_3)), - ?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_4)), - ?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_5)), + PZ2 = list_to_pid("<0.102.0>"), + PZ3 = list_to_pid("<0.103.0>"), + PZ5 = list_to_pid("<0.105.0>"), + + ?assertMatch(PZ2, key_lookup(Man6, 1, LK1_2)), + ?assertMatch(PZ2, key_lookup(Man6, 1, LK1_3)), + ?assertMatch(PZ3, key_lookup(Man6, 1, LK1_4)), + ?assertMatch(PZ3, key_lookup(Man6, 1, LK1_5)), - ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_2)), - ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_3)), - ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_4)), - ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_5)), + ?assertMatch(PZ5, key_lookup(Man6, 2, LK1_2)), + ?assertMatch(PZ5, key_lookup(Man6, 2, LK1_3)), + ?assertMatch(PZ5, key_lookup(Man6, 2, LK1_4)), + ?assertMatch(PZ5, key_lookup(Man6, 2, LK1_5)), {_Man7, _Man8, _Man9, _Man10, _Man11, _Man12, Man13} = changeup_setup(Man6), @@ -1242,18 +1378,20 @@ keylookup_manifest_test() -> ?assertMatch(false, key_lookup(Man3, 1, LK1_1)), ?assertMatch(false, key_lookup(Man6, 1, LK1_1)), - ?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_2)), - ?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_3)), - ?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_4)), - ?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_5)), + ?assertMatch(PZ2, key_lookup(Man6, 1, LK1_2)), + ?assertMatch(PZ2, key_lookup(Man6, 1, LK1_3)), + ?assertMatch(PZ3, key_lookup(Man6, 1, LK1_4)), + ?assertMatch(PZ3, key_lookup(Man6, 1, LK1_5)), - ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_2)), - ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_3)), - ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_4)), - ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_5)), + ?assertMatch(PZ5, key_lookup(Man6, 2, LK1_2)), + ?assertMatch(PZ5, key_lookup(Man6, 2, LK1_3)), + ?assertMatch(PZ5, key_lookup(Man6, 2, LK1_4)), + ?assertMatch(PZ5, key_lookup(Man6, 2, LK1_5)), - ?assertMatch("pid_y3", key_lookup(Man13, 1, LK1_4)), - ?assertMatch("pid_z5", key_lookup(Man13, 2, LK1_4)). + PY3 = list_to_pid("<0.203.0>"), + + ?assertMatch(PY3, key_lookup(Man13, 1, LK1_4)), + ?assertMatch(PZ5, key_lookup(Man13, 2, LK1_4)). ext_keylookup_manifest_test() -> RP = "test/test_area", @@ -1261,10 +1399,13 @@ ext_keylookup_manifest_test() -> {_Man0, _Man1, _Man2, _Man3, _Man4, _Man5, Man6} = initial_setup(), save_manifest(Man6, RP), - E7 = #manifest_entry{start_key={o, "Bucket1", "K997", null}, - end_key={o, "Bucket1", "K999", null}, - filename="Z7", - owner="pid_z7"}, + E7 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K997">>, null}, + end_key={o, <<"Bucket1">>, <<"K999">>, null}, + filename="Z7", + owner=list_to_pid("<0.107.0>") + }, Man7 = insert_manifest_entry(Man6, 2, 2, E7), save_manifest(Man7, RP), ManOpen1 = open_manifest(RP), @@ -1275,7 +1416,7 @@ ext_keylookup_manifest_test() -> {ok, BytesCopied} = file:copy(Man7FN, Man7FNAlt), {ok, Bin} = file:read_file(Man7FN), ?assertMatch(BytesCopied, byte_size(Bin)), - RandPos = leveled_rand:uniform(bit_size(Bin) - 1), + RandPos = rand:uniform(bit_size(Bin) - 1), <> = Bin, Flipped = BitToFlip bxor 1, ok = file:write_file(Man7FN, @@ -1288,62 +1429,93 @@ ext_keylookup_manifest_test() -> ManOpen2 = open_manifest(RP), ?assertMatch(1, get_manifest_sqn(ManOpen2)), - E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, - end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, - filename="Z1", - owner="pid_z1", - bloom=none}, - E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"}, - end_key={o, "Bucket1", "K71", null}, - filename="Z2", - owner="pid_z2", - bloom=none}, - E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null}, - end_key={o, "Bucket1", "K993", null}, - filename="Z3", - owner="pid_z3", - bloom=none}, + E1 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld1">>}, <<"K8">>}, + end_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K93">>}, + filename="Z1", + owner=list_to_pid("<0.101.0>"), + bloom=none + }, + E2 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K97">>}, + end_key={o, <<"Bucket1">>, <<"K71">>, null}, + filename="Z2", + owner=list_to_pid("<0.102.0>"), + bloom=none + }, + E3 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K75">>, null}, + end_key={o, <<"Bucket1">>, <<"K993">>, null}, + filename="Z3", + owner=list_to_pid("<0.103.0>"), + bloom=none + }, - E1_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld4"}, "K8"}, - end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K62"}, - owner="pid_y1", - filename="Y1", - bloom=none}, - E2_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K67"}, - end_key={o, "Bucket1", "K45", null}, - owner="pid_y2", - filename="Y2", - bloom=none}, - E3_2 = #manifest_entry{start_key={o, "Bucket1", "K47", null}, - end_key={o, "Bucket1", "K812", null}, - owner="pid_y3", - filename="Y3", - bloom=none}, - E4_2 = #manifest_entry{start_key={o, "Bucket1", "K815", null}, - end_key={o, "Bucket1", "K998", null}, - owner="pid_y4", - filename="Y4", - bloom=none}, + E1_2 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld4">>}, <<"K8">>}, + end_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K62">>}, + owner=list_to_pid("<0.201.0>"), + filename="Y1", + bloom=none + }, + E2_2 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K67">>}, + end_key={o, <<"Bucket1">>, <<"K45">>, null}, + owner=list_to_pid("<0.202.0>"), + filename="Y2", + bloom=none + }, + E3_2 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K47">>, null}, + end_key={o, <<"Bucket1">>, <<"K812">>, null}, + owner=list_to_pid("<0.203.0>"), + filename="Y3", + bloom=none + }, + E4_2 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K815">>, null}, + end_key={o, <<"Bucket1">>, <<"K998">>, null}, + owner=list_to_pid("<0.104.0>"), + filename="Y4", + bloom=none + }, Man8 = replace_manifest_entry(ManOpen2, 2, 1, E1, E1_2), Man9 = remove_manifest_entry(Man8, 2, 1, [E2, E3]), Man10 = insert_manifest_entry(Man9, 2, 1, [E2_2, E3_2, E4_2]), ?assertMatch(2, get_manifest_sqn(Man10)), - LK1_4 = {o, "Bucket1", "K75", null}, - ?assertMatch("pid_y3", key_lookup(Man10, 1, LK1_4)), - ?assertMatch("pid_z5", key_lookup(Man10, 2, LK1_4)), + LK1_4 = {o, <<"Bucket1">>, <<"K75">>, null}, + + PY3 = list_to_pid("<0.203.0>"), + PZ5 = list_to_pid("<0.105.0>"), + + ?assertMatch(PY3, key_lookup(Man10, 1, LK1_4)), + ?assertMatch(PZ5, key_lookup(Man10, 2, LK1_4)), - E5 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld7"}, "K97"}, - end_key={o, "Bucket1", "K78", null}, - filename="Z5", - owner="pid_z5", - bloom=none}, - E6 = #manifest_entry{start_key={o, "Bucket1", "K81", null}, - end_key={o, "Bucket1", "K996", null}, - filename="Z6", - owner="pid_z6", - bloom=none}, + E5 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld7">>}, <<"K97">>}, + end_key={o, <<"Bucket1">>, <<"K78">>, null}, + filename="Z5", + owner=PZ5, + bloom=none + }, + E6 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K81">>, null}, + end_key={o, <<"Bucket1">>, <<"K996">>, null}, + filename="Z6", + owner=list_to_pid("<0.106.0>"), + bloom=none + }, Man11 = remove_manifest_entry(Man10, 3, 2, [E5, E6]), ?assertMatch(3, get_manifest_sqn(Man11)), @@ -1351,7 +1523,7 @@ ext_keylookup_manifest_test() -> Man12 = replace_manifest_entry(Man11, 4, 2, E2_2, E5), ?assertMatch(4, get_manifest_sqn(Man12)), - ?assertMatch("pid_z5", key_lookup(Man12, 2, LK1_4)). + ?assertMatch(PZ5, key_lookup(Man12, 2, LK1_4)). rangequery_manifest_test() -> {_Man0, _Man1, _Man2, _Man3, _Man4, _Man5, Man6} = initial_setup(), @@ -1361,50 +1533,60 @@ rangequery_manifest_test() -> {next, ME, _SK} = Pointer, ME#manifest_entry.owner end, + + PZ1 = list_to_pid("<0.101.0>"), + PZ3 = list_to_pid("<0.103.0>"), + PZ5 = list_to_pid("<0.105.0>"), + PZ6 = list_to_pid("<0.106.0>"), + PY1 = list_to_pid("<0.201.0>"), + PY3 = list_to_pid("<0.203.0>"), + PY4 = list_to_pid("<0.204.0>"), - SK1 = {o, "Bucket1", "K711", null}, - EK1 = {o, "Bucket1", "K999", null}, + SK1 = {o, <<"Bucket1">>, <<"K711">>, null}, + EK1 = {o, <<"Bucket1">>, <<"K999">>, null}, RL1_1 = lists:map(PidMapFun, range_lookup(Man6, 1, SK1, EK1)), - ?assertMatch(["pid_z3"], RL1_1), + ?assertMatch([PZ3], RL1_1), RL1_2 = lists:map(PidMapFun, range_lookup(Man6, 2, SK1, EK1)), - ?assertMatch(["pid_z5", "pid_z6"], RL1_2), - SK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, - EK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, + ?assertMatch([PZ5, PZ6], RL1_2), + SK2 = {i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld8">>}, null}, + EK2 = {i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld8">>}, null}, RL2_1 = lists:map(PidMapFun, range_lookup(Man6, 1, SK2, EK2)), - ?assertMatch(["pid_z1"], RL2_1), + ?assertMatch([PZ1], RL2_1), RL2_2 = lists:map(PidMapFun, range_lookup(Man6, 2, SK2, EK2)), - ?assertMatch(["pid_z5"], RL2_2), + ?assertMatch([PZ5], RL2_2), - SK3 = {o, "Bucket1", "K994", null}, - EK3 = {o, "Bucket1", "K995", null}, + SK3 = {o, <<"Bucket1">>, <<"K994">>, null}, + EK3 = {o, <<"Bucket1">>, <<"K995">>, null}, RL3_1 = lists:map(PidMapFun, range_lookup(Man6, 1, SK3, EK3)), ?assertMatch([], RL3_1), RL3_2 = lists:map(PidMapFun, range_lookup(Man6, 2, SK3, EK3)), - ?assertMatch(["pid_z6"], RL3_2), + ?assertMatch([PZ6], RL3_2), {_Man7, _Man8, _Man9, _Man10, _Man11, _Man12, Man13} = changeup_setup(Man6), RL1_1A = lists:map(PidMapFun, range_lookup(Man6, 1, SK1, EK1)), - ?assertMatch(["pid_z3"], RL1_1A), + ?assertMatch([PZ3], RL1_1A), RL2_1A = lists:map(PidMapFun, range_lookup(Man6, 1, SK2, EK2)), - ?assertMatch(["pid_z1"], RL2_1A), + ?assertMatch([PZ1], RL2_1A), RL3_1A = lists:map(PidMapFun, range_lookup(Man6, 1, SK3, EK3)), ?assertMatch([], RL3_1A), RL1_1B = lists:map(PidMapFun, range_lookup(Man13, 1, SK1, EK1)), - ?assertMatch(["pid_y3", "pid_y4"], RL1_1B), + ?assertMatch([PY3, PY4], RL1_1B), RL2_1B = lists:map(PidMapFun, range_lookup(Man13, 1, SK2, EK2)), - ?assertMatch(["pid_y1"], RL2_1B), + ?assertMatch([PY1], RL2_1B), RL3_1B = lists:map(PidMapFun, range_lookup(Man13, 1, SK3, EK3)), - ?assertMatch(["pid_y4"], RL3_1B). + ?assertMatch([PY4], RL3_1B). levelzero_present_test() -> - E0 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, - end_key={o, "Bucket1", "Key996", null}, - filename="Z0", - owner="pid_z0", - bloom=none}, + E0 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld1">>}, <<"K8">>}, + end_key={o, <<"Bucket1">>, <<"Key996">>, null}, + filename="Z0", + owner=list_to_pid("<0.101.0>"), + bloom=none}, Man0 = new_manifest(), ?assertMatch(false, levelzero_present(Man0)), @@ -1426,21 +1608,30 @@ snapshot_release_test() -> PidA3 = spawn(fun() -> ok end), PidA4 = spawn(fun() -> ok end), Man6 = element(7, initial_setup()), - E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, - end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, - filename="Z1", - owner="pid_z1", - bloom=none}, - E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"}, - end_key={o, "Bucket1", "K71", null}, - filename="Z2", - owner="pid_z2", - bloom=none}, - E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null}, - end_key={o, "Bucket1", "K993", null}, - filename="Z3", - owner="pid_z3", - bloom=none}, + E1 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld1">>}, <<"K8">>}, + end_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K93">>}, + filename="Z1", + owner=list_to_pid("<0.101.0>"), + bloom=none + }, + E2 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K97">>}, + end_key={o, <<"Bucket1">>, <<"K71">>, null}, + filename="Z2", + owner=list_to_pid("<0.102.0>"), + bloom=none + }, + E3 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K75">>, null}, + end_key={o, <<"Bucket1">>, <<"K993">>, null}, + filename="Z3", + owner=list_to_pid("<0.103.0>"), + bloom=none + }, Man7 = add_snapshot(Man6, PidA1, 3600), Man8 = remove_manifest_entry(Man7, 2, 1, E1), @@ -1493,39 +1684,79 @@ snapshot_timeout_test() -> potential_issue_test() -> Manifest = - {manifest,{array,9,0,[], - {[], - [{manifest_entry,{o_rkv,"Bucket","Key10",null}, - {o_rkv,"Bucket","Key12949",null}, - "<0.313.0>","./16_1_0.sst", none}, - {manifest_entry,{o_rkv,"Bucket","Key129490",null}, - {o_rkv,"Bucket","Key158981",null}, - "<0.315.0>","./16_1_1.sst", none}, - {manifest_entry,{o_rkv,"Bucket","Key158982",null}, - {o_rkv,"Bucket","Key188472",null}, - "<0.316.0>","./16_1_2.sst", none}], - {idxt,1, - {{[{{o_rkv,"Bucket1","Key1",null}, - {manifest_entry,{o_rkv,"Bucket","Key9083",null}, - {o_rkv,"Bucket1","Key1",null}, - "<0.320.0>","./16_1_6.sst", none}}]}, - {1,{{o_rkv,"Bucket1","Key1",null},1,nil,nil}}}}, - {idxt,0,{{},{0,nil}}}, - {idxt,0,{{},{0,nil}}}, - {idxt,0,{{},{0,nil}}}, - {idxt,0,{{},{0,nil}}}, - {idxt,0,{{},{0,nil}}}, - {idxt,0,{{},{0,nil}}}, - []}}, - 19, [], 0, dict:new(), 2, dict:new()}, - Range1 = range_lookup(Manifest, - 1, - {o_rkv, "Bucket", null, null}, - {o_rkv, "Bucket", null, null}), - Range2 = range_lookup(Manifest, - 2, - {o_rkv, "Bucket", null, null}, - {o_rkv, "Bucket", null, null}), + {manifest, + {array,9,0,[], + { + [], + [ + {manifest_entry, + {o_rkv, <<"Bucket">>, <<"Key10">>, null}, + {o_rkv, <<"Bucket">>, <<"Key12949">>,null}, + list_to_pid("<0.313.0>"), + "./16_1_0.sst", + none + }, + {manifest_entry, + {o_rkv, <<"Bucket">>, <<"Key129490">>, null}, + {o_rkv, <<"Bucket">>, <<"Key158981">>, null}, + list_to_pid("<0.315.0>"), + "./16_1_1.sst", + none + }, + {manifest_entry, + {o_rkv, <<"Bucket">>, <<"Key158982">>, null}, + {o_rkv, <<"Bucket">>, <<"Key188472">>, null}, + list_to_pid("<0.316.0>"), + "./16_1_2.sst", + none + } + ], + { + idxt, + 1, + { + { + [ + {{o_rkv, <<"Bucket1">>, <<"Key1">>, null}, + { + manifest_entry, + {o_rkv, <<"Bucket">>, <<"Key9083">>, null}, + {o_rkv, <<"Bucket1">>, <<"Key1">>, null}, + list_to_pid("<0.320.0>"), + "./16_1_6.sst", + none + } + } + ] + }, + {1, {{o_rkv, <<"Bucket1">> ,<<"Key1">> ,null},1,nil,nil}}}}, + {idxt,0,{{},{0,nil}}}, + {idxt,0,{{},{0,nil}}}, + {idxt,0,{{},{0,nil}}}, + {idxt,0,{{},{0,nil}}}, + {idxt,0,{{},{0,nil}}}, + {idxt,0,{{},{0,nil}}}, + []}}, + 19, + [], + 0, + new_pending_deletions(), + 2, + new_blooms()}, + Range1 = + range_lookup( + Manifest, + 1, + {o_rkv, <<"Bucket">>, null, null}, + {o_rkv, <<"Bucket">>, null, null} + ), + Range2 = + range_lookup( + Manifest, + 2, + {o_rkv, <<"Bucket">>, null, null}, + {o_rkv, <<"Bucket">>, null, null} + ), io:format("Range in Level 1 ~w~n", [Range1]), io:format("Range in Level 2 ~w~n", [Range2]), ?assertMatch(3, length(Range1)), diff --git a/src/leveled_pmem.erl b/src/leveled_pmem.erl index ce7b9e4..db97a99 100644 --- a/src/leveled_pmem.erl +++ b/src/leveled_pmem.erl @@ -41,9 +41,12 @@ cache_full/1 ]). +% Test functions to ignore for equalizer - due to array issues +-eqwalizer({nowarn_function, index_performance_test/0}). + -define(MAX_CACHE_LINES, 31). % Must be less than 128 --type index_array() :: list(array:array())|[]|none. +-type index_array() :: list(array:array(binary()))|none. -export_type([index_array/0]). @@ -58,7 +61,7 @@ cache_full(L0Cache) -> length(L0Cache) == ?MAX_CACHE_LINES. -spec prepare_for_index( - array:array(), leveled_codec:segment_hash()) -> array:array(). + array:array(binary()), leveled_codec:segment_hash()) -> array:array(). %% @doc %% Add the hash of a key to the index. This is 'prepared' in the sense that %% this index is not use until it is loaded into the main index. @@ -73,18 +76,22 @@ prepare_for_index(IndexArray, Hash) -> Bin = array:get(Slot, IndexArray), array:set(Slot, <>, IndexArray). --spec add_to_index(array:array(), index_array(), integer()) -> index_array(). +-spec add_to_index( + array:array(binary()), index_array(), integer()) -> index_array(). %% @doc %% Expand the penciller's current index array with the details from a new %% ledger cache tree sent from the Bookie. The tree will have a cache slot %% which is the index of this ledger_cache in the list of the ledger_caches -add_to_index(LM1Array, L0Index, CacheSlot) when CacheSlot < 128 -> +add_to_index( + LM1Array, L0Index, CacheSlot) + when CacheSlot < 128, L0Index =/= none -> [LM1Array|L0Index]. --spec new_index() -> array:array(). +-spec new_index() -> array:array(binary()). %% @doc %% Create a new index array new_index() -> + % eqwalizer:ignore - array does contain binary() array:new([{size, 256}, {default, <<>>}]). -spec check_index(leveled_codec:segment_hash(), index_array()) @@ -92,7 +99,7 @@ new_index() -> %% @doc %% return a list of positions in the list of cache arrays that may contain the %% key associated with the hash being checked -check_index(Hash, L0Index) -> +check_index(Hash, L0Index) when L0Index =/= none -> {Slot, H0} = split_hash(Hash), {_L, Positions} = lists:foldl( @@ -239,19 +246,14 @@ check_slotlist(Key, _Hash, CheckList, TreeList) -> -include_lib("eunit/include/eunit.hrl"). generate_randomkeys_aslist(Seqn, Count, BucketRangeLow, BucketRangeHigh) -> - lists:ukeysort(1, - generate_randomkeys(Seqn, - Count, - [], - BucketRangeLow, - BucketRangeHigh)). + lists:ukeysort( + 1, + generate_randomkeys(Seqn, Count, [], BucketRangeLow, BucketRangeHigh) + ). generate_randomkeys(Seqn, Count, BucketRangeLow, BucketRangeHigh) -> - KVL = generate_randomkeys(Seqn, - Count, - [], - BucketRangeLow, - BucketRangeHigh), + KVL = + generate_randomkeys(Seqn, Count, [], BucketRangeLow, BucketRangeHigh), leveled_tree:from_orderedlist(lists:ukeysort(1, KVL), ?CACHE_TYPE). generate_randomkeys(_Seqn, 0, Acc, _BucketLow, _BucketHigh) -> @@ -260,32 +262,35 @@ generate_randomkeys(Seqn, Count, Acc, BucketLow, BRange) -> BNumber = lists:flatten( io_lib:format("~4..0B", - [BucketLow + leveled_rand:uniform(BRange)])), + [BucketLow + rand:uniform(BRange)])), KNumber = - lists:flatten(io_lib:format("~4..0B", [leveled_rand:uniform(1000)])), - {K, V} = {{o, "Bucket" ++ BNumber, "Key" ++ KNumber, null}, - {Seqn, {active, infinity}, null}}, - generate_randomkeys(Seqn + 1, - Count - 1, - [{K, V}|Acc], - BucketLow, - BRange). + lists:flatten(io_lib:format("~4..0B", [rand:uniform(1000)])), + {K, V} = + { + {o, + list_to_binary("Bucket" ++ BNumber), + list_to_binary("Key" ++ KNumber), + null}, + {Seqn, {active, infinity}, null} + }, + generate_randomkeys(Seqn + 1, Count - 1, [{K, V}|Acc], BucketLow, BRange). compare_method_test() -> - R = lists:foldl(fun(_X, {LedgerSQN, L0Size, L0TreeList}) -> - LM1 = generate_randomkeys(LedgerSQN + 1, - 2000, 1, 500), - add_to_cache( - L0Size, - {LM1, LedgerSQN + 1, LedgerSQN + 2000}, - LedgerSQN, - L0TreeList, - true) - end, - {0, 0, []}, - lists:seq(1, 16)), - + R = + lists:foldl( + fun(_X, {LedgerSQN, L0Size, L0TreeList}) -> + LM1 = generate_randomkeys(LedgerSQN + 1, 2000, 1, 500), + add_to_cache( + L0Size, + {LM1, LedgerSQN + 1, LedgerSQN + 2000}, + LedgerSQN, + L0TreeList, + true) + end, + {0, 0, []}, + lists:seq(1, 16)), + {SQN, Size, TreeList} = R, ?assertMatch(32000, SQN), ?assertMatch(true, Size =< 32000), @@ -310,51 +315,62 @@ compare_method_test() -> end end, - S0 = lists:foldl(fun({Key, _V}, Acc) -> - R0 = lists:foldl(FindKeyFun(Key), - {false, not_found}, - TreeList), - [R0|Acc] end, - [], - TestList), + S0 = + lists:foldl( + fun({Key, _V}, Acc) -> + R0 = + lists:foldl( + FindKeyFun(Key), {false, not_found}, TreeList), + [R0|Acc] + end, + [], + TestList) + , PosList = lists:seq(1, length(TreeList)), - S1 = lists:foldl(fun({Key, _V}, Acc) -> - R0 = check_levelzero(Key, PosList, TreeList), - [R0|Acc] - end, - [], - TestList), + S1 = + lists:foldl( + fun({Key, _V}, Acc) -> + R0 = check_levelzero(Key, PosList, TreeList), + [R0|Acc] + end, + [], + TestList + ), ?assertMatch(S0, S1), - StartKey = {o, "Bucket0100", null, null}, - EndKey = {o, "Bucket0200", null, null}, + StartKey = {o, <<"Bucket0100">>, null, null}, + EndKey = {o, <<"Bucket0200">>, null, null}, SWa = os:timestamp(), FetchFun = fun(Slot) -> lists:nth(Slot, TreeList) end, DumpList = to_list(length(TreeList), FetchFun), - Q0 = lists:foldl(fun({K, V}, Acc) -> - P = leveled_codec:endkey_passed(EndKey, K), - case {K, P} of - {K, false} when K >= StartKey -> - [{K, V}|Acc]; - _ -> - Acc - end - end, - [], - DumpList), + Q0 = + lists:foldl( + fun({K, V}, Acc) -> + P = leveled_codec:endkey_passed(EndKey, K), + case {K, P} of + {K, false} when K >= StartKey -> + [{K, V}|Acc]; + _ -> + Acc + end + end, + [], + DumpList + ), Tree = leveled_tree:from_orderedlist(lists:ukeysort(1, Q0), ?CACHE_TYPE), Sz0 = leveled_tree:tsize(Tree), - io:format("Crude method took ~w microseconds resulting in tree of " ++ - "size ~w~n", - [timer:now_diff(os:timestamp(), SWa), Sz0]), + io:format( + "Crude method took ~w microseconds resulting in tree of size ~w~n", + [timer:now_diff(os:timestamp(), SWa), Sz0] + ), SWb = os:timestamp(), Q1 = merge_trees(StartKey, EndKey, TreeList, leveled_tree:empty(?CACHE_TYPE)), Sz1 = length(Q1), - io:format("Merge method took ~w microseconds resulting in tree of " ++ - "size ~w~n", - [timer:now_diff(os:timestamp(), SWb), Sz1]), + io:format( + "Merge method took ~w microseconds resulting in tree of size ~w~n", + [timer:now_diff(os:timestamp(), SWb), Sz1]), ?assertMatch(Sz0, Sz1). with_index_test_() -> diff --git a/src/leveled_rand.erl b/src/leveled_rand.erl deleted file mode 100644 index fedaf20..0000000 --- a/src/leveled_rand.erl +++ /dev/null @@ -1,28 +0,0 @@ -%% Generalized random module that offers a backwards compatible API -%% around some of the changes in rand, crypto and for time units. - --module(leveled_rand). - -%% API --export([ - uniform/0, - uniform/1, - seed/0, - rand_bytes/1 - ]). - -%%%=================================================================== -%%% New (r19+) rand style functions -%%%=================================================================== -uniform() -> - rand:uniform(). - -uniform(N) -> - rand:uniform(N). - -seed() -> - ok. - -rand_bytes(Size) -> - crypto:strong_rand_bytes(Size). - diff --git a/src/leveled_runner.erl b/src/leveled_runner.erl index 32fe5f1..9737162 100644 --- a/src/leveled_runner.erl +++ b/src/leveled_runner.erl @@ -39,8 +39,7 @@ -define(CHECKJOURNAL_PROB, 0.2). -type key_range() - :: {leveled_codec:ledger_key()|null, - leveled_codec:ledger_key()|null}. + :: {leveled_codec:query_key(), leveled_codec:query_key()}. -type foldacc() :: any(). % Can't currently be specific about what an acc might be @@ -56,15 +55,15 @@ :: fun((leveled_codec:key(), leveled_codec:key()) -> accumulate|pass). -type snap_fun() - :: fun(() -> {ok, pid(), pid()|null}). + :: fun(() -> {ok, pid(), pid()|null, fun(() -> ok)}). -type runner_fun() :: fun(() -> foldacc()). --type acc_fun() - :: fun((leveled_codec:key(), any(), foldacc()) -> foldacc()). +-type objectacc_fun() + :: fun((leveled_codec:object_key(), any(), foldacc()) -> foldacc()). -type mp() :: {re_pattern, term(), term(), term(), term()}. --export_type([acc_fun/0, mp/0]). +-export_type([fold_keys_fun/0, mp/0]). %%%============================================================================ %%% External functions @@ -76,8 +75,8 @@ %% @doc %% Fold over a bucket accumulating the count of objects and their total sizes bucket_sizestats(SnapFun, Bucket, Tag) -> - StartKey = leveled_codec:to_ledgerkey(Bucket, null, Tag), - EndKey = leveled_codec:to_ledgerkey(Bucket, null, Tag), + StartKey = leveled_codec:to_querykey(Bucket, null, Tag), + EndKey = leveled_codec:to_querykey(Bucket, null, Tag), AccFun = accumulate_size(), Runner = fun() -> @@ -132,7 +131,7 @@ bucket_list(SnapFun, Tag, FoldBucketsFun, InitAcc, MaxBuckets) -> -spec index_query(snap_fun(), {leveled_codec:ledger_key(), leveled_codec:ledger_key(), - {boolean(), undefined|mp()|iodata()}}, + {boolean(), undefined|mp()}}, {fold_keys_fun(), foldacc()}) -> {async, runner_fun()}. %% @doc @@ -165,7 +164,7 @@ index_query(SnapFun, {StartKey, EndKey, TermHandling}, FoldAccT) -> -spec bucketkey_query(snap_fun(), leveled_codec:tag(), leveled_codec:key()|null, - key_range(), + {leveled_codec:single_key()|null, leveled_codec:single_key()|null}, {fold_keys_fun(), foldacc()}, leveled_codec:regular_expression()) -> {async, runner_fun()}. @@ -175,8 +174,8 @@ bucketkey_query(SnapFun, Tag, Bucket, {StartKey, EndKey}, {FoldKeysFun, InitAcc}, TermRegex) -> - SK = leveled_codec:to_ledgerkey(Bucket, StartKey, Tag), - EK = leveled_codec:to_ledgerkey(Bucket, EndKey, Tag), + SK = leveled_codec:to_querykey(Bucket, StartKey, Tag), + EK = leveled_codec:to_querykey(Bucket, EndKey, Tag), AccFun = accumulate_keys(FoldKeysFun, TermRegex), Runner = fun() -> @@ -203,8 +202,8 @@ bucketkey_query(SnapFun, Tag, Bucket, FunAcc) -> %% @doc %% Fold over the keys under a given Tag accumulating the hashes hashlist_query(SnapFun, Tag, JournalCheck) -> - StartKey = leveled_codec:to_ledgerkey(null, null, Tag), - EndKey = leveled_codec:to_ledgerkey(null, null, Tag), + StartKey = leveled_codec:to_querykey(null, null, Tag), + EndKey = leveled_codec:to_querykey(null, null, Tag), Runner = fun() -> {ok, LedgerSnapshot, JournalSnapshot, AfterFun} = SnapFun(), @@ -217,10 +216,11 @@ hashlist_query(SnapFun, Tag, JournalCheck) -> end, {async, Runner}. --spec tictactree(snap_fun(), - {leveled_codec:tag(), leveled_codec:key(), tuple()}, - boolean(), atom(), fold_filter_fun()) - -> {async, runner_fun()}. +-spec tictactree( + snap_fun(), + {leveled_codec:tag(), leveled_codec:key(), tuple()}, + boolean(), leveled_tictac:tree_size(), fold_filter_fun()) + -> {async, runner_fun()}. %% @doc %% Return a merkle tree from the fold, directly accessing hashes cached in the %% metadata @@ -246,14 +246,14 @@ tictactree(SnapFun, {Tag, Bucket, Query}, JournalCheck, TreeSize, Filter) -> case Tag of ?IDX_TAG -> {IdxFld, StartIdx, EndIdx} = Query, - KeyDefFun = fun leveled_codec:to_ledgerkey/5, + KeyDefFun = fun leveled_codec:to_querykey/5, {KeyDefFun(Bucket, null, ?IDX_TAG, IdxFld, StartIdx), KeyDefFun(Bucket, null, ?IDX_TAG, IdxFld, EndIdx), EnsureKeyBinaryFun}; _ -> {StartOKey, EndOKey} = Query, - {leveled_codec:to_ledgerkey(Bucket, StartOKey, Tag), - leveled_codec:to_ledgerkey(Bucket, EndOKey, Tag), + {leveled_codec:to_querykey(Bucket, StartOKey, Tag), + leveled_codec:to_querykey(Bucket, EndOKey, Tag), fun(K, H) -> V = {is_hash, H}, EnsureKeyBinaryFun(K, V) @@ -279,8 +279,8 @@ tictactree(SnapFun, {Tag, Bucket, Query}, JournalCheck, TreeSize, Filter) -> %% function to each proxy object foldheads_allkeys(SnapFun, Tag, FoldFun, JournalCheck, SegmentList, LastModRange, MaxObjectCount) -> - StartKey = leveled_codec:to_ledgerkey(null, null, Tag), - EndKey = leveled_codec:to_ledgerkey(null, null, Tag), + StartKey = leveled_codec:to_querykey(null, null, Tag), + EndKey = leveled_codec:to_querykey(null, null, Tag), foldobjects(SnapFun, Tag, [{StartKey, EndKey}], @@ -298,8 +298,8 @@ foldheads_allkeys(SnapFun, Tag, FoldFun, JournalCheck, %% @doc %% Fold over all objects for a given tag foldobjects_allkeys(SnapFun, Tag, FoldFun, key_order) -> - StartKey = leveled_codec:to_ledgerkey(null, null, Tag), - EndKey = leveled_codec:to_ledgerkey(null, null, Tag), + StartKey = leveled_codec:to_querykey(null, null, Tag), + EndKey = leveled_codec:to_querykey(null, null, Tag), foldobjects(SnapFun, Tag, [{StartKey, EndKey}], @@ -335,7 +335,7 @@ foldobjects_allkeys(SnapFun, Tag, FoldObjectsFun, sqn_order) -> _ -> {VBin, _VSize} = ExtractFun(JVal), {Obj, _IdxSpecs} = - leveled_codec:split_inkvalue(VBin), + leveled_codec:revert_value_from_journal(VBin), ToLoop = case SQN of MaxSQN -> stop; @@ -353,11 +353,16 @@ foldobjects_allkeys(SnapFun, Tag, FoldObjectsFun, sqn_order) -> Folder = fun() -> - {ok, LedgerSnapshot, JournalSnapshot, AfterFun} = SnapFun(), - {ok, JournalSQN} = leveled_inker:ink_getjournalsqn(JournalSnapshot), + {ok, LedgerSnapshot, JournalSnapshot, AfterFun} = + case SnapFun() of + {ok, LS, JS, AF} when is_pid(JS) -> + {ok, LS, JS, AF} + end, + {ok, JournalSQN} = + leveled_inker:ink_getjournalsqn(JournalSnapshot), IsValidFun = fun(Bucket, Key, SQN) -> - LedgerKey = leveled_codec:to_ledgerkey(Bucket, Key, Tag), + LedgerKey = leveled_codec:to_objectkey(Bucket, Key, Tag), CheckSQN = leveled_penciller:pcl_checksequencenumber( LedgerSnapshot, LedgerKey, SQN), @@ -438,9 +443,9 @@ foldheads_bybucket(SnapFun, %% and passing those objects into the fold function foldobjects_byindex(SnapFun, {Tag, Bucket, Field, FromTerm, ToTerm}, FoldFun) -> StartKey = - leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, Field, FromTerm), + leveled_codec:to_querykey(Bucket, null, ?IDX_TAG, Field, FromTerm), EndKey = - leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, Field, ToTerm), + leveled_codec:to_querykey(Bucket, null, ?IDX_TAG, Field, ToTerm), foldobjects(SnapFun, Tag, [{StartKey, EndKey}], @@ -457,38 +462,39 @@ foldobjects_byindex(SnapFun, {Tag, Bucket, Field, FromTerm, ToTerm}, FoldFun) -> get_nextbucket(_NextB, _NextK, _Tag, _LS, BKList, {Limit, Limit}) -> lists:reverse(BKList); get_nextbucket(NextBucket, NextKey, Tag, LedgerSnapshot, BKList, {C, L}) -> - StartKey = leveled_codec:to_ledgerkey(NextBucket, NextKey, Tag), - EndKey = leveled_codec:to_ledgerkey(null, null, Tag), + StartKey = leveled_codec:to_querykey(NextBucket, NextKey, Tag), + EndKey = leveled_codec:to_querykey(null, null, Tag), ExtractFun = fun(LK, V, _Acc) -> {leveled_codec:from_ledgerkey(LK), V} end, - R = leveled_penciller:pcl_fetchnextkey(LedgerSnapshot, - StartKey, - EndKey, - ExtractFun, - null), + R = + leveled_penciller:pcl_fetchnextkey( + LedgerSnapshot, StartKey, EndKey, ExtractFun, null), case R of {1, null} -> leveled_log:log(b0008,[]), BKList; - {0, {{B, K}, _V}} -> + {0, {{B, K}, _V}} when is_binary(B); is_tuple(B) -> leveled_log:log(b0009,[B]), - get_nextbucket(leveled_codec:next_key(B), - null, - Tag, - LedgerSnapshot, - [{B, K}|BKList], - {C + 1, L}) + get_nextbucket( + leveled_codec:next_key(B), + null, + Tag, + LedgerSnapshot, + [{B, K}|BKList], + {C + 1, L} + ) end. --spec foldobjects(snap_fun(), - atom(), - list(), - fold_objects_fun()|{fold_objects_fun(), foldacc()}, - false|{true, boolean()}, false|list(integer())) -> - {async, runner_fun()}. +-spec foldobjects( + snap_fun(), + atom(), + list(), + fold_objects_fun()|{fold_objects_fun(), foldacc()}, + false|{true, boolean()}, false|list(integer())) + -> {async, runner_fun()}. foldobjects(SnapFun, Tag, KeyRanges, FoldObjFun, DeferredFetch, SegmentList) -> foldobjects(SnapFun, Tag, KeyRanges, FoldObjFun, DeferredFetch, SegmentList, false, false). @@ -534,14 +540,16 @@ foldobjects(SnapFun, Tag, KeyRanges, FoldObjFun, DeferredFetch, FoldFun, JournalSnapshot, Tag, DeferredFetch), FoldFunGen = fun({StartKey, EndKey}, FoldAcc) -> - leveled_penciller:pcl_fetchkeysbysegment(LedgerSnapshot, - StartKey, - EndKey, - AccFun, - FoldAcc, - SegmentList, - LastModRange, - LimitByCount) + leveled_penciller:pcl_fetchkeysbysegment( + LedgerSnapshot, + StartKey, + EndKey, + AccFun, + FoldAcc, + SegmentList, + LastModRange, + LimitByCount + ) end, ListFoldFun = fun(KeyRange, Acc) -> @@ -567,9 +575,7 @@ accumulate_hashes(JournalCheck, InkerClone) -> fun(B, K, H, Acc) -> [{B, K, H}|Acc] end, - get_hashaccumulator(JournalCheck, - InkerClone, - AddKeyFun). + get_hashaccumulator(JournalCheck, InkerClone, AddKeyFun). accumulate_tree(FilterFun, JournalCheck, InkerClone, HashFun) -> AddKeyFun = @@ -581,15 +587,13 @@ accumulate_tree(FilterFun, JournalCheck, InkerClone, HashFun) -> Tree end end, - get_hashaccumulator(JournalCheck, - InkerClone, - AddKeyFun). + get_hashaccumulator(JournalCheck, InkerClone, AddKeyFun). get_hashaccumulator(JournalCheck, InkerClone, AddKeyFun) -> AccFun = fun(LK, V, Acc) -> {B, K, H} = leveled_codec:get_keyandobjhash(LK, V), - Check = leveled_rand:uniform() < ?CHECKJOURNAL_PROB, + Check = rand:uniform() < ?CHECKJOURNAL_PROB, case JournalCheck and Check of true -> case check_presence(LK, V, InkerClone) of @@ -604,11 +608,11 @@ get_hashaccumulator(JournalCheck, InkerClone, AddKeyFun) -> end, AccFun. --spec accumulate_objects(fold_objects_fun(), - pid()|null, - leveled_codec:tag(), - false|{true, boolean()}) - -> acc_fun(). +-spec accumulate_objects + (fold_objects_fun(), pid(), leveled_head:object_tag(), false|{true, boolean()}) + -> objectacc_fun(); + (fold_objects_fun(), null, leveled_head:headonly_tag(), {true, false}) + -> objectacc_fun(). accumulate_objects(FoldObjectsFun, InkerClone, Tag, DeferredFetch) -> AccFun = fun(LK, V, Acc) -> @@ -630,24 +634,23 @@ accumulate_objects(FoldObjectsFun, InkerClone, Tag, DeferredFetch) -> {B0, K0, _T0} -> {B0, K0} end, - JK = {leveled_codec:to_ledgerkey(B, K, Tag), SQN}, + JK = {leveled_codec:to_objectkey(B, K, Tag), SQN}, case DeferredFetch of - {true, JournalCheck} -> + {true, JournalCheck} when MD =/= null -> ProxyObj = leveled_codec:return_proxy(Tag, MD, InkerClone, JK), - case JournalCheck of - true -> + case {JournalCheck, InkerClone} of + {true, InkerClone} when is_pid(InkerClone) -> InJournal = - leveled_inker:ink_keycheck(InkerClone, - LK, - SQN), + leveled_inker:ink_keycheck( + InkerClone, LK, SQN), case InJournal of probably -> FoldObjectsFun(B, K, ProxyObj, Acc); missing -> Acc end; - false -> + {false, _} -> FoldObjectsFun(B, K, ProxyObj, Acc) end; false -> @@ -730,10 +733,10 @@ throw_test() -> fun() -> error end, - ?assertMatch({ok, ['1']}, - wrap_runner(CompletedFolder, AfterAction)), - ?assertException(throw, stop_fold, - wrap_runner(StoppedFolder, AfterAction)). + ?assertMatch({ok, ['1']}, wrap_runner(CompletedFolder, AfterAction)), + ?assertException( + throw, stop_fold, wrap_runner(StoppedFolder, AfterAction) + ). -endif. diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index 5647274..8aa21a9 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -63,6 +63,9 @@ -include("leveled.hrl"). +% Test functions to ignore for equalizer +-eqwalizer({nowarn_function, fetch_status_test/0}). + -define(LOOK_SLOTSIZE, 128). % Maximum of 128 -define(LOOK_BLOCKSIZE, {24, 32}). % 4x + y = ?LOOK_SLOTSIZE -define(NOLOOK_SLOTSIZE, 256). @@ -83,7 +86,6 @@ -define(FLIPPER32, 4294967295). -define(DOUBLESIZE_LEVEL, 3). -define(INDEX_MODDATE, true). --define(TOMB_COUNT, true). -define(USE_SET_FOR_SPEED, 32). -define(STARTUP_TIMEOUT, 10000). -define(MIN_HASH, 32768). @@ -126,7 +128,7 @@ sst_gettombcount/1, sst_close/1]). --export([sst_newmerge/10]). +-export([sst_newmerge/9]). -export([tune_seglist/1, extract_hash/1, segment_checker/1]). @@ -170,7 +172,7 @@ range_endpoint()}. -type expandable_pointer() :: slot_pointer()|sst_pointer()|sst_closed_pointer(). --type expanded_pointer() +-type maybe_expanded_pointer() :: leveled_codec:ledger_kv()|expandable_pointer(). -type expanded_slot() :: {binary(), non_neg_integer(), range_endpoint(), range_endpoint()}. @@ -179,7 +181,12 @@ -type sst_options() :: #sst_options{}. -type binary_slot() - :: {binary(), binary(), list(integer()), leveled_codec:ledger_key()}. + :: { + binary(), + binary(), + list(leveled_codec:segment_hash()), + leveled_codec:ledger_key() + }. -type sst_summary() :: #summary{}. -type blockindex_cache() @@ -199,27 +206,38 @@ | false. -type fetch_levelzero_fun() :: fun((pos_integer(), leveled_penciller:levelzero_returnfun()) -> ok). +-type extract_hash() :: non_neg_integer()|no_lookup. + +-record(read_state, + { + handle :: file:io_device(), + blockindex_cache :: blockindex_cache()|redacted, + fetch_cache :: fetch_cache()|redacted, + level :: leveled_pmanifest:lsm_level(), + filter_fun :: summary_filter() + } +). + +-type read_state() :: #read_state{}. -record(state, - {summary, - handle :: file:fd() | undefined, - penciller :: pid() | undefined | false, - root_path, - filename, - blockindex_cache :: - blockindex_cache() | undefined | redacted, - compression_method = native :: press_method(), - index_moddate = ?INDEX_MODDATE :: boolean(), - starting_pid :: pid()|undefined, - fetch_cache = no_cache :: fetch_cache() | redacted, - new_slots :: list()|undefined, - deferred_startup_tuple :: tuple()|undefined, - level :: leveled_pmanifest:lsm_level()|undefined, - tomb_count = not_counted - :: non_neg_integer()|not_counted, - high_modified_date :: non_neg_integer()|undefined, - filter_fun :: summary_filter() | undefined, - monitor = {no_monitor, 0} :: leveled_monitor:monitor()}). + { + summary, + penciller :: pid() | undefined | false, + root_path, + filename, + read_state :: read_state() | undefined, + compression_method = native :: press_method(), + index_moddate = ?INDEX_MODDATE :: boolean(), + starting_pid :: pid()|undefined, + new_slots :: list()|undefined, + deferred_startup_tuple :: tuple()|undefined, + tomb_count = not_counted + :: non_neg_integer()|not_counted, + high_modified_date :: non_neg_integer()|undefined, + monitor = {no_monitor, 0} :: leveled_monitor:monitor() + } +). -record(build_timings, {slot_hashlist = 0 :: integer(), @@ -238,8 +256,9 @@ -spec sst_open( string(), string(), sst_options(), leveled_pmanifest:lsm_level()) - -> {ok, pid(), - {leveled_codec:ledger_key(), leveled_codec:ledger_key()}, + -> + {ok, pid(), + {leveled_codec:object_key(), leveled_codec:object_key()}, binary()}. %% @doc %% Open an SST file at a given path and filename. The first and last keys @@ -261,7 +280,8 @@ sst_open(RootPath, Filename, OptsSST, Level) -> list(leveled_codec:ledger_kv()), integer(), sst_options()) -> {ok, pid(), - {leveled_codec:ledger_key(), leveled_codec:ledger_key()}, + {leveled_codec:object_key(), + leveled_codec:object_key()}, binary()}. %% @doc %% Start a new SST file at the assigned level passing in a list of Key, Value @@ -299,8 +319,8 @@ sst_new(RootPath, Filename, Level, KVList, MaxSQN, OptsSST, IndexModDate) -> -> empty|{ok, pid(), {{list(leveled_codec:ledger_kv()), list(leveled_codec:ledger_kv())}, - leveled_codec:ledger_key(), - leveled_codec:ledger_key()}, + leveled_codec:object_key(), + leveled_codec:object_key()}, binary()}. %% @doc %% Start a new SST file at the assigned level passing in a two lists of @@ -318,11 +338,11 @@ sst_newmerge(RootPath, Filename, MaxSQN, OptsSST) when Level > 0 -> sst_newmerge(RootPath, Filename, KVL1, KVL2, IsBasement, Level, - MaxSQN, OptsSST, ?INDEX_MODDATE, ?TOMB_COUNT). + MaxSQN, OptsSST, ?INDEX_MODDATE). sst_newmerge(RootPath, Filename, KVL1, KVL2, IsBasement, Level, - MaxSQN, OptsSST, IndexModDate, TombCount) -> + MaxSQN, OptsSST, IndexModDate) -> OptsSST0 = update_options(OptsSST, Level), {Rem1, Rem2, SlotList, FK, CountOfTombs} = merge_lists( @@ -330,8 +350,8 @@ sst_newmerge(RootPath, Filename, KVL2, {IsBasement, Level}, OptsSST0, - IndexModDate, - TombCount), + IndexModDate + ), case SlotList of [] -> empty; @@ -348,7 +368,9 @@ sst_newmerge(RootPath, Filename, MaxSQN, OptsSST0, IndexModDate, - CountOfTombs, self()}, + CountOfTombs, + self() + }, infinity), {ok, Pid, {{Rem1, Rem2}, SK, EK}, Bloom} end. @@ -387,8 +409,9 @@ sst_newlevelzero( {ok, Pid, noreply}. --spec sst_get(pid(), leveled_codec:ledger_key()) - -> leveled_codec:ledger_kv()|not_present. +-spec sst_get( + pid(), leveled_codec:object_key()) -> + leveled_codec:ledger_kv()|not_present. %% @doc %% Return a Key, Value pair matching a Key or not_present if the Key is not in %% the store. The segment_hash function is used to accelerate the seeking of @@ -396,17 +419,18 @@ sst_newlevelzero( sst_get(Pid, LedgerKey) -> sst_get(Pid, LedgerKey, leveled_codec:segment_hash(LedgerKey)). --spec sst_get(pid(), leveled_codec:ledger_key(), leveled_codec:segment_hash()) - -> leveled_codec:ledger_kv()|not_present. +-spec sst_get( + pid(), leveled_codec:object_key(), leveled_codec:segment_hash()) + -> leveled_codec:ledger_kv()|not_present. %% @doc %% Return a Key, Value pair matching a Key or not_present if the Key is not in %% the store (with the magic hash precalculated). sst_get(Pid, LedgerKey, Hash) -> gen_statem:call(Pid, {get_kv, LedgerKey, Hash, undefined}, infinity). --spec sst_getsqn(pid(), - leveled_codec:ledger_key(), - leveled_codec:segment_hash()) -> leveled_codec:sqn()|not_present. +-spec sst_getsqn( + pid(), leveled_codec:object_key(), leveled_codec:segment_hash()) + -> leveled_codec:sqn()|not_present. %% @doc %% Return a SQN for the key or not_present if the key is not in %% the store (with the magic hash precalculated). @@ -424,7 +448,7 @@ sst_getmaxsequencenumber(Pid) -> list(expandable_pointer()), pos_integer(), segment_check_fun(), - non_neg_integer()) -> list(expanded_pointer()). + non_neg_integer()) -> list(maybe_expanded_pointer()). %% @doc %% Expand out a list of pointer to return a list of Keys and Values with a %% tail of pointers (once the ScanWidth has been satisfied). @@ -469,7 +493,7 @@ sst_deleteconfirmed(Pid) -> gen_statem:cast(Pid, close). -spec sst_checkready(pid()) -> - {ok, string(), leveled_codec:ledger_key(), leveled_codec:ledger_key()}. + {ok, string(), leveled_codec:object_key(), leveled_codec:object_key()}. %% @doc %% If a file has been set to be built, check that it has been built. Returns %% the filename and the {startKey, EndKey} for the manifest. @@ -507,14 +531,17 @@ starting({call, From}, leveled_log:save(OptsSST#sst_options.log_options), Monitor = OptsSST#sst_options.monitor, {UpdState, Bloom} = - read_file(Filename, - State#state{root_path=RootPath}, - OptsSST#sst_options.pagecache_level >= Level), + read_file( + Filename, + State#state{root_path=RootPath}, + OptsSST#sst_options.pagecache_level >= Level, + undefined, + Level + ), Summary = UpdState#state.summary, {next_state, reader, - UpdState#state{ - level = Level, fetch_cache = new_cache(Level), monitor = Monitor}, + UpdState#state{monitor = Monitor}, [{reply, From, {ok, {Summary#summary.first_key, Summary#summary.last_key}, @@ -532,7 +559,7 @@ starting({call, From}, PressMethod = OptsSST#sst_options.press_method, {Length, SlotIndex, BlockEntries, SlotsBin, Bloom} = build_all_slots(SlotList), - {_, BlockIndex, HighModDate} = + {_, BlockIndexCache, HighModDate} = update_blockindex_cache( BlockEntries, new_blockindex_cache(Length), @@ -549,7 +576,10 @@ starting({call, From}, read_file( ActualFilename, State#state{root_path=RootPath}, - OptsSST#sst_options.pagecache_level >= Level), + OptsSST#sst_options.pagecache_level >= Level, + BlockIndexCache, + Level + ), Summary = UpdState#state.summary, leveled_log:log_timer( sst08, [ActualFilename, Level, Summary#summary.max_sqn], SW), @@ -557,11 +587,8 @@ starting({call, From}, {next_state, reader, UpdState#state{ - blockindex_cache = BlockIndex, high_modified_date = HighModDate, starting_pid = StartingPID, - level = Level, - fetch_cache = new_cache(Level), monitor = Monitor}, [{reply, From, @@ -575,9 +602,7 @@ starting({call, From}, {sst_newlevelzero, RootPath, Filename, IdxModDate}, {next_state, starting, State#state{ - deferred_startup_tuple = DeferredStartupTuple, - level = 0, - fetch_cache = new_cache(0)}, + deferred_startup_tuple = DeferredStartupTuple}, [{reply, From, ok}]}; starting({call, From}, close, State) -> %% No file should have been created, so nothing to close. @@ -592,6 +617,7 @@ starting(cast, complete_l0startup, State) -> State#state.deferred_startup_tuple, SW0 = os:timestamp(), FetchedSlots = State#state.new_slots, + FetchedSlots =/= undefined orelse error(invalid_type), leveled_log:save(OptsSST#sst_options.log_options), Monitor = OptsSST#sst_options.monitor, PressMethod = OptsSST#sst_options.press_method, @@ -607,7 +633,7 @@ starting(cast, complete_l0startup, State) -> SW2 = os:timestamp(), {SlotCount, SlotIndex, BlockEntries, SlotsBin,Bloom} = build_all_slots(SlotList), - {_, BlockIndex, HighModDate} = + {_, BlockIndexCache, HighModDate} = update_blockindex_cache( BlockEntries, new_blockindex_cache(SlotCount), @@ -632,7 +658,10 @@ starting(cast, complete_l0startup, State) -> root_path=RootPath, new_slots=undefined, % Important to empty this from state deferred_startup_tuple=undefined}, - true), + true, + BlockIndexCache, + 0 + ), Summary = UpdState#state.summary, Time4 = timer:now_diff(os:timestamp(), SW4), @@ -655,16 +684,15 @@ starting(cast, complete_l0startup, State) -> {next_state, reader, UpdState#state{ - blockindex_cache = BlockIndex, high_modified_date = HighModDate, monitor = Monitor}}; starting(cast, {sst_returnslot, FetchedSlot, FetchFun, SlotCount}, State) -> FetchedSlots = - case FetchedSlot of - none -> - []; + case {FetchedSlot, State#state.new_slots} of + {FS, PreviousSlots} when FS =/= none, is_list(PreviousSlots) -> + [FetchedSlot|PreviousSlots]; _ -> - [FetchedSlot|State#state.new_slots] + [] end, case length(FetchedSlots) == SlotCount of true -> @@ -685,7 +713,9 @@ starting(cast, {sst_returnslot, FetchedSlot, FetchFun, SlotCount}, State) -> State#state{new_slots = FetchedSlots}} end. -reader({call, From}, {get_kv, LedgerKey, Hash, Filter}, State) -> +reader({call, From}, + {get_kv, LedgerKey, Hash, Filter}, + State = #state{read_state = RS}) when ?IS_DEF(RS)-> % Get a KV value and potentially take sample timings Monitor = case Filter of @@ -694,65 +724,69 @@ reader({call, From}, {get_kv, LedgerKey, Hash, Filter}, State) -> _ -> {no_monitor, 0} end, - {KeyValue, BIC, HMD, FC} = + FetchResult = fetch( - LedgerKey, Hash, + LedgerKey, + Hash, State#state.summary, State#state.compression_method, State#state.high_modified_date, State#state.index_moddate, - State#state.filter_fun, - State#state.blockindex_cache, - State#state.fetch_cache, - State#state.handle, - State#state.level, + RS#read_state.filter_fun, + RS#read_state.blockindex_cache, + RS#read_state.fetch_cache, + RS#read_state.handle, + RS#read_state.level, Monitor), - Result = + FilterFun = case Filter of - undefined -> - KeyValue; - F -> - F(KeyValue) + undefined -> fun(KV) -> KV end; + F when is_function(F) -> F end, - case {BIC, HMD, FC} of - {no_update, no_update, no_update} -> - {keep_state_and_data, [{reply, From, Result}]}; - {no_update, no_update, FC} -> + case FetchResult of + {KV, no_update, no_update, no_update} -> + {keep_state_and_data, [{reply, From, FilterFun(KV)}]}; + {KV, no_update, no_update, FC} -> + RS0 = RS#read_state{fetch_cache = FC}, {keep_state, - State#state{fetch_cache = FC}, - [{reply, From, Result}]}; - {BIC, undefined, no_update} -> + State#state{read_state = RS0}, + [{reply, From, FilterFun(KV)}]}; + {KV, BIC, undefined, no_update} when BIC =/= no_update -> + RS0 = RS#read_state{blockindex_cache = BIC}, {keep_state, - State#state{blockindex_cache = BIC}, - [{reply, From, Result}]}; - {BIC, HMD, no_update} -> + State#state{read_state = RS0}, + [{reply, From, FilterFun(KV)}]}; + {KV, BIC, HMD, no_update} when BIC =/= no_update, HMD =/= no_update -> + RS0 = RS#read_state{blockindex_cache = BIC}, {keep_state, - State#state{blockindex_cache = BIC, high_modified_date = HMD}, - [hibernate, {reply, From, Result}]} + State#state{read_state = RS0, high_modified_date = HMD}, + [hibernate, {reply, From, FilterFun(KV)}]} end; reader({call, From}, {fetch_range, StartKey, EndKey, LowLastMod}, - State) -> + State = #state{read_state = RS}) when ?IS_DEF(RS) -> SlotsToPoint = fetch_range( StartKey, EndKey, State#state.summary, - State#state.filter_fun, + RS#read_state.filter_fun, check_modified( State#state.high_modified_date, LowLastMod, State#state.index_moddate) ), {keep_state_and_data, [{reply, From, SlotsToPoint}]}; -reader({call, From}, {get_slots, SlotList, SegChecker, LowLastMod}, State) -> +reader({call, From}, + {get_slots, SlotList, SegChecker, LowLastMod}, + State = #state{read_state = RS}) when ?IS_DEF(RS) -> PressMethod = State#state.compression_method, IdxModDate = State#state.index_moddate, {NeedBlockIdx, SlotBins} = read_slots( - State#state.handle, + RS#read_state.handle, SlotList, - {SegChecker, LowLastMod, State#state.blockindex_cache}, + {SegChecker, LowLastMod, RS#read_state.blockindex_cache}, State#state.compression_method, State#state.index_moddate), {keep_state_and_data, @@ -780,15 +814,22 @@ reader({call, From}, background_complete, State) -> reader({call, From}, get_tomb_count, State) -> {keep_state_and_data, [{reply, From, State#state.tomb_count}]}; -reader({call, From}, close, State) -> - ok = file:close(State#state.handle), +reader({call, From}, + close, + State = #state{read_state = RS}) when ?IS_DEF(RS) -> + ok = file:close(RS#read_state.handle), {stop_and_reply, normal, [{reply, From, ok}], State}; -reader(cast, {switch_levels, NewLevel}, State) -> +reader(cast, + {switch_levels, NewLevel}, + State = #state{read_state = RS}) when ?IS_DEF(RS) -> {keep_state, State#state{ - level = NewLevel, - fetch_cache = new_cache(NewLevel) + read_state = + RS#read_state{ + fetch_cache = new_cache(NewLevel), + level = NewLevel + } }, [hibernate]}; reader(info, {update_blockindex_cache, BIC}, State) -> @@ -799,7 +840,10 @@ reader(info, bic_complete, State) -> % fragmentation leveled_log:log(sst14, [State#state.filename]), {keep_state_and_data, [hibernate]}; -reader(info, start_complete, State) -> +reader( + info, + start_complete, + #state{starting_pid = StartingPid}) when ?IS_DEF(StartingPid) -> % The SST file will be started by a clerk, but the clerk may be shut down % prior to the manifest being updated about the existence of this SST file. % If there is no activity after startup, check the clerk is still alive and @@ -807,7 +851,7 @@ reader(info, start_complete, State) -> % If the clerk has crashed, the penciller will restart at the latest % manifest, and so this file sill be restarted if and only if it is still % part of the store - case is_process_alive(State#state.starting_pid) of + case is_process_alive(StartingPid) of true -> {keep_state_and_data, []}; false -> @@ -815,7 +859,9 @@ reader(info, start_complete, State) -> end. -delete_pending({call, From}, {get_kv, LedgerKey, Hash, Filter}, State) -> +delete_pending({call, From}, + {get_kv, LedgerKey, Hash, Filter}, + State = #state{read_state = RS}) when ?IS_DEF(RS) -> {KeyValue, _BIC, _HMD, _FC} = fetch( LedgerKey, Hash, @@ -823,11 +869,11 @@ delete_pending({call, From}, {get_kv, LedgerKey, Hash, Filter}, State) -> State#state.compression_method, State#state.high_modified_date, State#state.index_moddate, - State#state.filter_fun, - State#state.blockindex_cache, - State#state.fetch_cache, - State#state.handle, - State#state.level, + RS#read_state.filter_fun, + RS#read_state.blockindex_cache, + RS#read_state.fetch_cache, + RS#read_state.handle, + RS#read_state.level, {no_monitor, 0}), Result = case Filter of @@ -840,13 +886,13 @@ delete_pending({call, From}, {get_kv, LedgerKey, Hash, Filter}, State) -> delete_pending( {call, From}, {fetch_range, StartKey, EndKey, LowLastMod}, - State) -> + State = #state{read_state = RS}) when ?IS_DEF(RS) -> SlotsToPoint = fetch_range( StartKey, EndKey, State#state.summary, - State#state.filter_fun, + RS#read_state.filter_fun, check_modified( State#state.high_modified_date, LowLastMod, @@ -856,29 +902,35 @@ delete_pending( delete_pending( {call, From}, {get_slots, SlotList, SegChecker, LowLastMod}, - State) -> + State = #state{read_state = RS}) when ?IS_DEF(RS) -> PressMethod = State#state.compression_method, IdxModDate = State#state.index_moddate, {_NeedBlockIdx, SlotBins} = read_slots( - State#state.handle, + RS#read_state.handle, SlotList, - {SegChecker, LowLastMod, State#state.blockindex_cache}, + {SegChecker, LowLastMod, RS#read_state.blockindex_cache}, PressMethod, IdxModDate), {keep_state_and_data, [{reply, From, {false, SlotBins, PressMethod, IdxModDate}}, ?DELETE_TIMEOUT]}; -delete_pending({call, From}, close, State) -> +delete_pending( + {call, From}, + close, + State = #state{read_state = RS}) when ?IS_DEF(RS) -> leveled_log:log(sst07, [State#state.filename]), - ok = file:close(State#state.handle), + ok = file:close(RS#read_state.handle), ok = file:delete( filename:join(State#state.root_path, State#state.filename)), {stop_and_reply, normal, [{reply, From, ok}], State}; -delete_pending(cast, close, State) -> +delete_pending( + cast, + close, + State = #state{read_state = RS}) when ?IS_DEF(RS) -> leveled_log:log(sst07, [State#state.filename]), - ok = file:close(State#state.handle), + ok = file:close(RS#read_state.handle), ok = file:delete( filename:join(State#state.root_path, State#state.filename)), @@ -892,27 +944,32 @@ delete_pending(timeout, _, State) -> case State#state.penciller of false -> ok = leveled_sst:sst_deleteconfirmed(self()); - PCL -> + PCL when is_pid(PCL) -> FN = State#state.filename, ok = leveled_penciller:pcl_confirmdelete(PCL, FN, self()) - end, + end, % If the next thing is another timeout - may be long-running snapshot, so % back-off - {keep_state_and_data, [leveled_rand:uniform(10) * ?DELETE_TIMEOUT]}. + {keep_state_and_data, [rand:uniform(10) * ?DELETE_TIMEOUT]}. -handle_update_blockindex_cache(BIC, State) -> +handle_update_blockindex_cache( + BIC, + State = #state{read_state = RS}) when ?IS_DEF(RS) -> {NeedBlockIdx, BlockIndexCache, HighModDate} = update_blockindex_cache( BIC, - State#state.blockindex_cache, + RS#read_state.blockindex_cache, State#state.high_modified_date, State#state.index_moddate), case NeedBlockIdx of true -> {keep_state, State#state{ - blockindex_cache = BlockIndexCache, - high_modified_date = HighModDate}}; + read_state = + RS#read_state{blockindex_cache = BlockIndexCache}, + high_modified_date = HighModDate + } + }; false -> keep_state_and_data end. @@ -927,14 +984,17 @@ code_change(_OldVsn, StateName, State, _Extra) -> format_status(Status) -> - case maps:get(reason, Status, normal) of - terminate -> - State = maps:get(state, Status), + case {maps:get(reason, Status, normal), maps:get(state, Status)} of + {terminate, State = #state{read_state = RS}} when ?IS_DEF(RS) -> maps:update( state, State#state{ - blockindex_cache = redacted, - fetch_cache = redacted}, + read_state = + RS#read_state{ + blockindex_cache = redacted, + fetch_cache = redacted + } + }, Status ); _ -> @@ -949,7 +1009,7 @@ format_status(Status) -> -spec expand_list_by_pointer( expandable_pointer(), list(expandable_pointer()), - pos_integer()) -> list(expanded_pointer()). + pos_integer()) -> list(maybe_expanded_pointer()). %% @doc %% Expand a list of pointers, maybe ending up with a list of keys and values %% with a tail of pointers @@ -966,7 +1026,7 @@ expand_list_by_pointer(Pointer, Tail, Width) -> list(expandable_pointer()), pos_integer(), segment_check_fun(), - non_neg_integer()) -> list(expanded_pointer()). + non_neg_integer()) -> list(maybe_expanded_pointer()). %% @doc %% With filters (as described in expand_list_by_pointer/3 expand_list_by_pointer( @@ -975,17 +1035,7 @@ expand_list_by_pointer( {PotentialPointers, Remainder} = lists:split(min(Width - 1, length(Tail)), Tail), {LocalPointers, OtherPointers} = - lists:partition( - fun(Pointer) -> - case Pointer of - {pointer, SSTPid, _S, _SK, _EK} -> - true; - _ -> - false - end - end, - PotentialPointers - ), + split_localpointers(SSTPid, PotentialPointers), sst_getfilteredslots( SSTPid, [{pointer, SSTPid, Slot, StartKey, EndKey}|LocalPointers], @@ -994,18 +1044,30 @@ expand_list_by_pointer( OtherPointers ++ Remainder ); expand_list_by_pointer( - {next, ManEntry, StartKey, EndKey}, - Tail, _Width, _SegChecker, LowLastMod) -> + {next, ME, StartKey, EndKey}, Tail, _Width, _SegChecker, LowLastMod + ) -> % The first pointer is a pointer to a file - expand_list_by_pointer will % in this case convert this into list of pointers within that SST file % i.e. of the form {pointer, SSTPid, Slot, StartKey, EndKey} % This can then be further expanded by calling again to % expand_list_by_pointer - SSTPid = ManEntry#manifest_entry.owner, + SSTPid = leveled_pmanifest:entry_owner(ME), leveled_log:log(sst10, [SSTPid, is_process_alive(SSTPid)]), ExpPointer = sst_getfilteredrange(SSTPid, StartKey, EndKey, LowLastMod), ExpPointer ++ Tail. +-spec split_localpointers( + pid(), list(expandable_pointer())) -> + {list(slot_pointer()), list(expandable_pointer())}. +split_localpointers(LocalPid, PotentialPointers) -> + lists:partition( + fun({pointer, PID, _S, _SK, _EK}) when PID == LocalPid -> + true; + (_) -> + false + end, + PotentialPointers + ). -spec sst_getfilteredrange( pid(), @@ -1112,7 +1174,7 @@ segment_checker(HashList) when is_list(HashList) -> Max = lists:max(HashList), case length(HashList) > ?USE_SET_FOR_SPEED of true -> - HashSet = sets:from_list(HashList), + HashSet = sets:from_list(HashList, [{version, 2}]), {Min, Max, fun(H) -> sets:is_element(H, HashSet) end}; false -> {Min, Max, fun(H) -> lists:member(H, HashList) end} @@ -1128,7 +1190,7 @@ sqn_only(KV) -> leveled_codec:strip_to_seqonly(KV). -spec extract_hash( - leveled_codec:segment_hash()) -> non_neg_integer()|no_lookup. + leveled_codec:segment_hash()) -> extract_hash(). extract_hash({SegHash, _ExtraHash}) when is_integer(SegHash) -> tune_hash(SegHash); extract_hash(NotHash) -> @@ -1141,7 +1203,7 @@ new_cache(Level) -> no_cache -> no_cache; CacheSize -> - array:new([{size, CacheSize}]) + array:new([{size, CacheSize}, {default, none}]) end. -spec cache_hash(leveled_codec:segment_hash(), non_neg_integer()) -> @@ -1171,23 +1233,59 @@ cache_size(6) -> cache_size(_LowerLevel) -> no_cache. --spec fetch_from_cache( +-spec get_from_fetchcache( cache_hash(), - fetch_cache()) -> undefined|leveled_codec:ledger_kv(). -fetch_from_cache(_CacheHash, no_cache) -> - undefined; -fetch_from_cache(CacheHash, Cache) -> + fetch_cache()) -> none|leveled_codec:ledger_kv(). +get_from_fetchcache(_CacheHash, no_cache) -> + none; +get_from_fetchcache(CacheHash, Cache) when is_integer(CacheHash) -> + % When defining array can use array:array/1 but this will lead to type + % issues when using array:new + % eqwalizer:ignore array:get(CacheHash, Cache). -spec add_to_cache( - non_neg_integer(), + non_neg_integer()|no_cache, leveled_codec:ledger_kv(), fetch_cache()) -> fetch_cache(). -add_to_cache(_CacheHash, _KV, no_cache) -> - no_cache; -add_to_cache(CacheHash, KV, FetchCache) -> - array:set(CacheHash, KV, FetchCache). +add_to_cache( + CacheHash, KV, FetchCache) + when is_integer(CacheHash), FetchCache =/= no_cache -> + array:set(CacheHash, KV, FetchCache); +add_to_cache(_CacheHash, _KV, _FetchCache) -> + no_cache. +-spec get_from_blockcache(pos_integer(), blockindex_cache()) -> binary()|none. +get_from_blockcache(SlotID, BIC) when is_integer(SlotID) -> + case array:get(SlotID - 1, element(2, BIC)) of + CBI when is_binary(CBI) -> + CBI; + none -> + none + end. + +-spec add_to_blockcache( + pos_integer(), blockindex_cache(), binary(), non_neg_integer()|false) -> + blockindex_cache(). +add_to_blockcache(SlotID, {Cnt, Cache, HighLMD}, Block, LMD) -> + { + Cnt + 1, + array:set(SlotID - 1, binary:copy(Block), Cache), + case LMD of + LMD when is_integer(LMD), LMD > HighLMD -> + LMD; + _ -> + HighLMD + end + }. + +-spec size_of_blockcache(blockindex_cache()) -> non_neg_integer(). +size_of_blockcache(BIC) -> + array:size(element(2, BIC)). + +-spec new_blockindex_cache(pos_integer()) -> blockindex_cache(). +new_blockindex_cache(Size) -> + {0, array:new([{size, Size}, {default, none}]), 0}. -spec tune_hash(non_neg_integer()) -> ?MIN_HASH..?MAX_HASH. %% @doc @@ -1220,44 +1318,40 @@ update_options(OptsSST, Level) -> maxslots_level(Level, OptsSST#sst_options.max_sstslots), OptsSST#sst_options{press_method = PressMethod0, max_sstslots = MaxSlots0}. --spec new_blockindex_cache(pos_integer()) -> blockindex_cache(). -new_blockindex_cache(Size) -> - {0, array:new([{size, Size}, {default, none}]), 0}. - -spec updatebic_foldfun(boolean()) -> - fun(({integer(), binary()}, blockindex_cache()) -> blockindex_cache()). + fun(({integer(), binary()|none}, blockindex_cache()) -> blockindex_cache()). updatebic_foldfun(HMDRequired) -> - fun(CacheEntry, {AccCount, Cache, AccHMD}) -> + fun(CacheEntry, BIC) -> case CacheEntry of {ID, Header} when is_binary(Header) -> - case array:get(ID - 1, Cache) of + case get_from_blockcache(ID, BIC) of none -> - H0 = binary:copy(Header), - AccHMD0 = + LMD = case HMDRequired of true -> - max(AccHMD, - element(2, extract_header(H0, true))); + {_, ExtractLMD, _} = + extract_header(Header, true), + ExtractLMD; false -> - AccHMD + false end, - {AccCount + 1, array:set(ID - 1, H0, Cache), AccHMD0}; + add_to_blockcache(ID, BIC, Header, LMD); _ -> - {AccCount, Cache, AccHMD} + BIC end; _ -> - {AccCount, Cache, AccHMD} + BIC end end. -spec update_blockindex_cache( - list({integer(), binary()}), + list({integer(), binary()|none}), blockindex_cache(), non_neg_integer()|undefined, boolean()) -> {boolean(), blockindex_cache(), non_neg_integer()|undefined}. update_blockindex_cache(Entries, BIC, HighModDate, IdxModDate) -> - case {element(1, BIC), array:size(element(2, BIC))} of + case {element(1, BIC), size_of_blockcache(BIC)} of {N, N} -> {false, BIC, HighModDate}; {N, S} when N < S -> @@ -1318,7 +1412,7 @@ fetch(LedgerKey, Hash, Slot = lookup_slot(LedgerKey, Summary#summary.index, FilterFun), SlotID = Slot#slot_index_value.slot_id, - CachedBlockIdx = array:get(SlotID - 1, element(2, BIC)), + CachedBlockIdx = get_from_blockcache(SlotID, BIC), case extract_header(CachedBlockIdx, IndexModDate) of none -> @@ -1340,14 +1434,17 @@ fetch(LedgerKey, Hash, {Result, BIC0, HMD0, no_update}; {BlockLengths, _LMD, PosBin} -> PosList = - find_pos(PosBin, segment_checker(extract_hash(Hash))), + case extract_hash(Hash) of + HashExtract when is_integer(HashExtract) -> + find_pos(PosBin, segment_checker(HashExtract)) + end, case PosList of [] -> maybelog_fetch_timing(Monitor, Level, not_found, SW0), {not_present, no_update, no_update, no_update}; _ -> CacheHash = cache_hash(Hash, Level), - case fetch_from_cache(CacheHash, FetchCache) of + case get_from_fetchcache(CacheHash, FetchCache) of {LedgerKey, V} -> maybelog_fetch_timing( Monitor, Level, fetch_cache, SW0), @@ -1355,29 +1452,32 @@ fetch(LedgerKey, Hash, _ -> StartPos = Slot#slot_index_value.start_position, Result = - check_blocks( + check_blocks_matchkey( PosList, {Handle, StartPos}, BlockLengths, byte_size(PosBin), LedgerKey, PressMethod, - IndexModDate, - not_present), + IndexModDate + ), case Result of not_present -> maybelog_fetch_timing( Monitor, Level, not_found, SW0), {not_present, no_update, no_update, no_update}; - _ -> + {LK, LV} -> FetchCache0 = add_to_cache( - CacheHash, Result, FetchCache), + CacheHash, {LK, LV}, FetchCache), maybelog_fetch_timing( Monitor, Level, slot_cachedblock, SW0), - {Result, - no_update, no_update, FetchCache0} + {{LK, LV}, + no_update, + no_update, + FetchCache0 + } end end end @@ -1461,7 +1561,7 @@ write_file(RootPath, Filename, SummaryBin, SlotsBin, false), FinalName. -read_file(Filename, State, LoadPageCache) -> +read_file(Filename, State, LoadPageCache, BIC, Level) -> {Handle, FileVersion, SummaryBin} = open_reader( filename:join(State#state.root_path, Filename), @@ -1469,19 +1569,33 @@ read_file(Filename, State, LoadPageCache) -> UpdState0 = imp_fileversion(FileVersion, State), {Summary, Bloom, SlotList, TombCount} = read_table_summary(SummaryBin, UpdState0#state.tomb_count), - BlockIndexCache = new_blockindex_cache(Summary#summary.size), - UpdState1 = UpdState0#state{blockindex_cache = BlockIndexCache}, {SlotIndex, FilterFun} = from_list( SlotList, Summary#summary.first_key, Summary#summary.last_key), UpdSummary = Summary#summary{index = SlotIndex}, leveled_log:log( sst03, [Filename, Summary#summary.size, Summary#summary.max_sqn]), - {UpdState1#state{summary = UpdSummary, - handle = Handle, - filename = Filename, - tomb_count = TombCount, - filter_fun = FilterFun}, + ReadState = + #read_state{ + handle = Handle, + blockindex_cache = + case BIC of + undefined -> + new_blockindex_cache(Summary#summary.size); + _ -> + BIC + end, + fetch_cache = new_cache(Level), + level = Level, + filter_fun = FilterFun + }, + + {UpdState0#state{ + summary = UpdSummary, + filename = Filename, + tomb_count = TombCount, + read_state = ReadState + }, Bloom}. gen_fileversion(PressMethod, IdxModDate, CountOfTombs) -> @@ -1678,7 +1792,7 @@ serialise_block(Term, none) -> CRC32 = hmac(Bin), <>. --spec deserialise_block(binary(), press_method()) -> any(). +-spec deserialise_block(binary(), press_method()) -> list(leveled_codec:ledger_kv()). %% @doc %% Convert binary to term %% Function split out to make it easier to experiment with different @@ -1688,21 +1802,27 @@ serialise_block(Term, none) -> deserialise_block(Bin, PressMethod) when byte_size(Bin) > 4 -> BinS = byte_size(Bin) - 4, <> = Bin, - case hmac(TermBin) of - CRC32 -> - deserialise_checkedblock(TermBin, PressMethod); - _ -> + try + CRC32 = hmac(TermBin), + deserialise_checkedblock(TermBin, PressMethod) + catch + _Exception:_Reason -> [] end; deserialise_block(_Bin, _PM) -> []. -deserialise_checkedblock(Bin, lz4) -> - {ok, Bin0} = lz4:unpack(Bin), - binary_to_term(Bin0); -deserialise_checkedblock(Bin, zstd) -> - binary_to_term(zstd:decompress(Bin)); -deserialise_checkedblock(Bin, _Other) -> +deserialise_checkedblock(Bin, lz4) when is_binary(Bin) -> + case lz4:unpack(Bin) of + {ok, Bin0} when is_binary(Bin0) -> + binary_to_term(Bin0) + end; +deserialise_checkedblock(Bin, zstd) when is_binary(Bin) -> + case zstd:decompress(Bin) of + Bin0 when is_binary(Bin0) -> + binary_to_term(Bin0) + end; +deserialise_checkedblock(Bin, _Other) when is_binary(Bin) -> % native or none can be treated the same binary_to_term(Bin). @@ -1766,7 +1886,8 @@ null_filter(Key) -> Key. key_filter({_Tag, _Bucket, Key, null}) -> Key. -spec term_filter(leveled_codec:ledger_key()) -> leveled_codec:slimmed_key(). -term_filter({_Tag, _Bucket, {_Field, Term}, Key}) -> {Term, Key}. +term_filter({_Tag, _Bucket, {_Field, Term}, Key}) -> + {Term, Key}. -spec key_prefix_filter( pos_integer(), binary()) -> @@ -1868,12 +1989,13 @@ lookup_slots(StartKey, EndKey, Tree, FilterFun) -> {binary(), non_neg_integer(), list(leveled_codec:segment_hash()), - leveled_codec:last_moddate()}. + non_neg_integer()}. %% @doc %% Fold function use to accumulate the position information needed to %% populate the summary of the slot -accumulate_positions([], Acc) -> - Acc; +accumulate_positions( + [], {PosBin, NoHashCount, HashAcc, LMDAcc}) when is_integer(LMDAcc) -> + {PosBin, NoHashCount, HashAcc, LMDAcc}; accumulate_positions([{K, V}|T], {PosBin, NoHashCount, HashAcc, LMDAcc}) -> {_SQN, H1, LMD} = leveled_codec:strip_to_indexdetails({K, V}), LMDAcc0 = take_max_lastmoddate(LMD, LMDAcc), @@ -2053,52 +2175,95 @@ generate_binary_slot( {{Header, SlotBin, HashL, LastKey}, BuildTimings3}. --spec check_blocks(list(integer()), - binary()|{file:io_device(), integer()}, - binary(), - integer(), - leveled_codec:ledger_key()|false, - %% if false the acc is a list, and if true - %% Acc will be initially not_present, and may - %% result in a {K, V} tuple - press_method(), - boolean(), - list()|not_present) -> - list(leveled_codec:ledger_kv())| - not_present|leveled_codec:ledger_kv(). +-spec check_blocks_allkeys( + list(integer()), + binary()|{file:io_device(), integer()}, + binary(), + integer(), + press_method(), + boolean(), + list()) -> + list(leveled_codec:ledger_kv()). %% @doc %% Acc should start as not_present if LedgerKey is a key, and a list if %% LedgerKey is false -check_blocks([], _BlockPointer, _BlockLengths, _PosBinLength, - _LedgerKeyToCheck, _PressMethod, _IdxModDate, not_present) -> - not_present; -check_blocks([], _BlockPointer, _BlockLengths, _PosBinLength, - _LedgerKeyToCheck, _PressMethod, _IdxModDate, Acc) -> +check_blocks_allkeys([], _BP, _BLs, _PBL, _PM, _IMD, Acc) -> lists:reverse(Acc); -check_blocks([Pos|Rest], BlockPointer, BlockLengths, PosBinLength, - LedgerKeyToCheck, PressMethod, IdxModDate, Acc) -> +check_blocks_allkeys( + [Pos|Rest], + BlockPointer, + BlockLengths, + PosBinLength, + PressMethod, + IdxModDate, + Acc) -> {BlockNumber, BlockPos} = revert_position(Pos), BlockBin = - read_block(BlockPointer, - BlockLengths, - PosBinLength, - BlockNumber, - additional_offset(IdxModDate)), - Result = spawn_check_block(BlockPos, BlockBin, PressMethod), - case {Result, LedgerKeyToCheck} of + read_block( + BlockPointer, + BlockLengths, + PosBinLength, + BlockNumber, + additional_offset(IdxModDate) + ), + case spawn_check_block(BlockPos, BlockBin, PressMethod) of + {K, V} -> + check_blocks_allkeys( + Rest, + BlockPointer, + BlockLengths, + PosBinLength, + PressMethod, + IdxModDate, + [{K, V}|Acc] + ) + end. + +-spec check_blocks_matchkey( + list(integer()), + binary()|{file:io_device(), integer()}, + binary(), + integer(), + leveled_codec:ledger_key(), + press_method(), + boolean()) -> + not_present|leveled_codec:ledger_kv(). +%% @doc +%% Acc should start as not_present if LedgerKey is a key, and a list if +%% LedgerKey is false +check_blocks_matchkey([], _BP, _BLs, _PBL, _LKTC, _PM, _IMD) -> + not_present; +check_blocks_matchkey( + [Pos|Rest], + BlockPointer, + BlockLengths, + PosBinLength, + LedgerKeyToCheck, + PressMethod, + IdxModDate) -> + {BlockNumber, BlockPos} = revert_position(Pos), + BlockBin = + read_block(BlockPointer, + BlockLengths, + PosBinLength, + BlockNumber, + additional_offset(IdxModDate) + ), + CheckResult = spawn_check_block(BlockPos, BlockBin, PressMethod), + case {CheckResult, LedgerKeyToCheck} of {{K, V}, K} -> {K, V}; - {{K, V}, false} -> - check_blocks(Rest, BlockPointer, - BlockLengths, PosBinLength, - LedgerKeyToCheck, PressMethod, IdxModDate, - [{K, V}|Acc]); _ -> - check_blocks(Rest, BlockPointer, - BlockLengths, PosBinLength, - LedgerKeyToCheck, PressMethod, IdxModDate, - Acc) - end. + check_blocks_matchkey( + Rest, + BlockPointer, + BlockLengths, + PosBinLength, + LedgerKeyToCheck, + PressMethod, + IdxModDate + ) +end. -spec spawn_check_block(non_neg_integer(), binary(), press_method()) -> not_present|leveled_codec:ledger_kv(). @@ -2204,7 +2369,7 @@ read_slots(Handle, SlotList, {SegChecker, LowLastMod, BlockIndexCache}, BinMapFun = fun(Pointer, {NeededBlockIdx, Acc}) -> {SP, _L, ID, SK, EK} = pointer_mapfun(Pointer), - CachedHeader = array:get(ID - 1, element(2, BlockIndexCache)), + CachedHeader = get_from_blockcache(ID, BlockIndexCache), case extract_header(CachedHeader, IdxModDate) of none -> % If there is an attempt to use the seg list query and the @@ -2271,9 +2436,15 @@ checkblocks_segandrange( PressMethod, IdxModDate, SegChecker, {StartKey, EndKey}) -> PositionList = find_pos(BlockIdx, SegChecker), KVL = - check_blocks( - PositionList, SlotOrHandle, BlockLengths, byte_size(BlockIdx), - false, PressMethod, IdxModDate, []), + check_blocks_allkeys( + PositionList, + SlotOrHandle, + BlockLengths, + byte_size(BlockIdx), + PressMethod, + IdxModDate, + [] + ), in_range(KVL, StartKey, EndKey). @@ -2371,18 +2542,26 @@ extract_header(Header, false) -> <> = Header, {BlockLengths, 0, PosBinIndex}. +-spec binaryslot_get( + binary(), + leveled_codec:ledger_key(), + leveled_codec:segment_hash(), + press_method(), + boolean()) -> {not_present|leveled_codec:ledger_kv(), binary()|none}. binaryslot_get(FullBin, Key, Hash, PressMethod, IdxModDate) -> case crc_check_slot(FullBin) of {Header, Blocks} -> {BlockLengths, _LMD, PosBinIndex} = extract_header(Header, IdxModDate), PosList = - find_pos(PosBinIndex, segment_checker(extract_hash(Hash))), + case extract_hash(Hash) of + HashExtract when is_integer(HashExtract) -> + find_pos(PosBinIndex, segment_checker(HashExtract)) + end, {fetch_value(PosList, BlockLengths, Blocks, Key, PressMethod), Header}; crc_wonky -> - {not_present, - none} + {not_present, none} end. -spec binaryslot_tolist( @@ -2390,7 +2569,7 @@ binaryslot_get(FullBin, Key, Hash, PressMethod, IdxModDate) -> press_method(), boolean(), list(leveled_codec:ledger_kv()|expandable_pointer())) - -> list(leveled_codec:ledger_kv()). + -> list(leveled_codec:ledger_kv()|expandable_pointer()). binaryslot_tolist(FullBin, PressMethod, IdxModDate, InitAcc) -> case crc_check_slot(FullBin) of {Header, Blocks} -> @@ -2426,7 +2605,7 @@ binaryslot_tolist(FullBin, PressMethod, IdxModDate, InitAcc) -> segment_check_fun(), list(leveled_codec:ledger_kv()|expandable_pointer()) ) -> - {list(leveled_codec:ledger_kv()), + {list(leveled_codec:ledger_kv()|expandable_pointer()), list({integer(), binary()})|none}. %% @doc %% Must return a trimmed and reversed list of results in the range @@ -2642,6 +2821,12 @@ block_offsetandlength(BlockLengths, BlockID) -> {B1L + B2L + B3L + B4L, B5L} end. +-spec fetch_value( + list(non_neg_integer()), + binary(), + binary(), + leveled_codec:ledger_key(), + press_method()) -> not_present|leveled_codec:ledger_kv(). fetch_value([], _BlockLengths, _Blocks, _Key, _PressMethod) -> not_present; fetch_value([Pos|Rest], BlockLengths, Blocks, Key, PressMethod) -> @@ -2791,12 +2976,12 @@ split_lists(KVList1, SlotLists, N, PressMethod, IdxModDate) -> split_lists(KVListRem, [SlotD|SlotLists], N - 1, PressMethod, IdxModDate). -spec merge_lists( - list(expanded_pointer()), - list(expanded_pointer()), + list(maybe_expanded_pointer()), + list(maybe_expanded_pointer()), {boolean(), non_neg_integer()}, - sst_options(), boolean(), boolean()) -> - {list(expanded_pointer()), - list(expanded_pointer()), + sst_options(), boolean()) -> + {list(maybe_expanded_pointer()), + list(maybe_expanded_pointer()), list(binary_slot()), leveled_codec:ledger_key()|null, non_neg_integer()}. @@ -2805,9 +2990,7 @@ split_lists(KVList1, SlotLists, N, PressMethod, IdxModDate) -> %% provided may include pointers to fetch more Keys/Values from the source %% file merge_lists( - KVList1, KVList2, {IsBase, L}, SSTOpts, IndexModDate, SaveTombCount) -> - InitTombCount = - case SaveTombCount of true -> 0; false -> not_counted end, + KVList1, KVList2, {IsBase, L}, SSTOpts, IndexModDate) -> BuildTimings = case IsBase orelse lists:member(L, ?LOG_BUILDTIMINGS_LEVELS) of true -> @@ -2816,16 +2999,23 @@ merge_lists( no_timing end, merge_lists( - KVList1, KVList2, - {IsBase, L}, [], null, 0, - SSTOpts#sst_options.max_sstslots, SSTOpts#sst_options.press_method, - IndexModDate, InitTombCount, - BuildTimings). + KVList1, + KVList2, + {IsBase, L}, + [], + null, + 0, + SSTOpts#sst_options.max_sstslots, + SSTOpts#sst_options.press_method, + IndexModDate, + 0, + BuildTimings + ). -spec merge_lists( - list(expanded_pointer()), - list(expanded_pointer()), + list(maybe_expanded_pointer()), + list(maybe_expanded_pointer()), {boolean(), non_neg_integer()}, list(binary_slot()), leveled_codec:ledger_key()|null, @@ -2833,11 +3023,11 @@ merge_lists( non_neg_integer(), press_method(), boolean(), - non_neg_integer()|not_counted, + non_neg_integer(), build_timings()) -> - {list(expanded_pointer()), list(expanded_pointer()), + {list(maybe_expanded_pointer()), list(maybe_expanded_pointer()), list(binary_slot()), leveled_codec:ledger_key()|null, - non_neg_integer()|not_counted}. + non_neg_integer()}. merge_lists(KVL1, KVL2, LI, SlotList, FirstKey, MaxSlots, MaxSlots, _PressMethod, _IdxModDate, CountOfTombs, T0) -> @@ -2890,16 +3080,17 @@ merge_lists(KVL1, KVL2, LI, SlotList, FirstKey, SlotCount, MaxSlots, T2) end. --spec form_slot(list(expanded_pointer()), - list(expanded_pointer()), - {boolean(), non_neg_integer()}, - lookup|no_lookup, - non_neg_integer(), - list(leveled_codec:ledger_kv()), - leveled_codec:ledger_key()|null) -> - {list(expanded_pointer()), list(expanded_pointer()), - {lookup|no_lookup, list(leveled_codec:ledger_kv())}, - leveled_codec:ledger_key()}. +-spec form_slot( + list(maybe_expanded_pointer()), + list(maybe_expanded_pointer()), + {boolean(), non_neg_integer()}, + lookup|no_lookup, + non_neg_integer(), + list(leveled_codec:ledger_kv()), + leveled_codec:ledger_key()|null) -> + {list(maybe_expanded_pointer()), list(maybe_expanded_pointer()), + {lookup|no_lookup, list(leveled_codec:ledger_kv())}, + leveled_codec:ledger_key()|null}. %% @doc %% Merge together Key Value lists to provide a reverse-ordered slot of KVs form_slot([], [], _LI, Type, _Size, Slot, FK) -> @@ -2932,7 +3123,7 @@ form_slot(KVList1, KVList2, LevelInfo, no_lookup, Size, Slot, FK) -> FK0); lookup -> case Size >= ?LOOK_SLOTSIZE of - true -> + true when FK =/= null -> {KVList1, KVList2, {no_lookup, Slot}, FK}; false -> form_slot( @@ -2950,22 +3141,28 @@ form_slot(KVList1, KVList2, LevelInfo, no_lookup, Size, Slot, FK) -> end. -spec key_dominates( - list(expanded_pointer()), - list(expanded_pointer()), - {boolean()|undefined, leveled_pmanifest:lsm_level()}) + list(maybe_expanded_pointer()), + list(maybe_expanded_pointer()), + {boolean(), leveled_pmanifest:lsm_level()}) -> {{next_key, leveled_codec:ledger_kv()}|skipped_key, - list(expanded_pointer()), - list(expanded_pointer())}. + list(maybe_expanded_pointer()), + list(maybe_expanded_pointer())}. key_dominates([{pointer, SSTPid, Slot, StartKey, all}|T1], KL2, Level) -> key_dominates( expand_list_by_pointer( + % As the head is a pointer, the tail must be pointers too + % So eqwalizer is wrong that this may be + % [leveled_codec:ledger_kv()] + % eqwalizer:ignore {pointer, SSTPid, Slot, StartKey, all}, T1, ?MERGE_SCANWIDTH), KL2, Level); key_dominates([{next, ManEntry, StartKey}|T1], KL2, Level) -> key_dominates( expand_list_by_pointer( + % See above + % eqwalizer:ignore {next, ManEntry, StartKey, all}, T1, ?MERGE_SCANWIDTH), KL2, Level); @@ -2973,12 +3170,16 @@ key_dominates(KL1, [{pointer, SSTPid, Slot, StartKey, all}|T2], Level) -> key_dominates( KL1, expand_list_by_pointer( + % See above + % eqwalizer:ignore {pointer, SSTPid, Slot, StartKey, all}, T2, ?MERGE_SCANWIDTH), Level); key_dominates(KL1, [{next, ManEntry, StartKey}|T2], Level) -> key_dominates( KL1, expand_list_by_pointer( + % See above + % eqwalizer:ignore {next, ManEntry, StartKey, all}, T2, ?MERGE_SCANWIDTH), Level); key_dominates( @@ -2988,7 +3189,7 @@ key_dominates( [{K1, V1}|Rest1], [{K2, _V2}|_T2]=Rest2, {false, _TS}) when K1 < K2 -> {{next_key, {K1, V1}}, Rest1, Rest2}; key_dominates(KL1, KL2, Level) -> - case key_dominates_expanded(KL1, KL2) of + case key_dominates_comparison(KL1, KL2) of {{next_key, NKV}, Rest1, Rest2} -> case leveled_codec:maybe_reap_expiredkey(NKV, Level) of true -> @@ -3000,25 +3201,27 @@ key_dominates(KL1, KL2, Level) -> {skipped_key, Rest1, Rest2} end. --spec key_dominates_expanded( - list(expanded_pointer()), list(expanded_pointer())) +-spec key_dominates_comparison( + list(maybe_expanded_pointer()), + list(maybe_expanded_pointer())) + % first item in each list must be leveled_codec:ledger_kv() -> {{next_key, leveled_codec:ledger_kv()}|skipped_key, - list(expanded_pointer()), - list(expanded_pointer())}. -key_dominates_expanded([H1|T1], []) -> - {{next_key, H1}, T1, []}; -key_dominates_expanded([], [H2|T2]) -> - {{next_key, H2}, [], T2}; -key_dominates_expanded([{K1, _V1}|_T1]=LHL, [{K2, V2}|T2]) when K2 < K1 -> + list(maybe_expanded_pointer()), + list(maybe_expanded_pointer())}. +key_dominates_comparison([{K1, V1}|T1], []) -> + {{next_key, {K1, V1}}, T1, []}; +key_dominates_comparison([], [{K2, V2}|T2]) -> + {{next_key, {K2, V2}}, [], T2}; +key_dominates_comparison([{K1, _V1}|_T1]=LHL, [{K2, V2}|T2]) when K2 < K1 -> {{next_key, {K2, V2}}, LHL, T2}; -key_dominates_expanded([{K1, V1}|T1], [{K2, _V2}|_T2]=RHL) when K1 < K2 -> +key_dominates_comparison([{K1, V1}|T1], [{K2, _V2}|_T2]=RHL) when K1 < K2 -> {{next_key, {K1, V1}}, T1, RHL}; -key_dominates_expanded([H1|T1], [H2|T2]) -> - case leveled_codec:key_dominates(H1, H2) of +key_dominates_comparison([{K1, V1}|T1], [{K2, V2}|T2]) -> + case leveled_codec:key_dominates({K1, V1}, {K2, V2}) of true -> - {skipped_key, [H1|T1], T2}; + {skipped_key, [{K1, V1}|T1], T2}; false -> - {skipped_key, T1, [H2|T2]} + {skipped_key, T1, [{K2, V2}|T2]} end. @@ -3079,8 +3282,11 @@ log_buildtimings(Timings, LI) -> erlang:timestamp()|no_timing) -> ok. maybelog_fetch_timing(_Monitor, _Level, _Type, no_timing) -> ok; -maybelog_fetch_timing({Pid, _SlotFreq}, Level, Type, SW) -> - {TS1, _} = leveled_monitor:step_time(SW), +maybelog_fetch_timing({Pid, _SlotFreq}, Level, Type, SW) when is_pid(Pid), SW =/= no_timing -> + TS1 = + case leveled_monitor:step_time(SW) of + {TS, _NextSW} when is_integer(TS) -> TS + end, leveled_monitor:add_stat(Pid, {sst_fetch_update, Level, Type, TS1}). %%%============================================================================ @@ -3112,7 +3318,7 @@ sst_getkvrange(Pid, StartKey, EndKey, ScanWidth) -> range_endpoint(), integer(), segment_check_fun(), - non_neg_integer()) -> list(leveled_codec:ledger_kv()|slot_pointer()). + non_neg_integer()) -> list(maybe_expanded_pointer()). %% @doc %% Get a range of {Key, Value} pairs as a list between StartKey and EndKey %% (inclusive). The ScanWidth is the maximum size of the range, a pointer @@ -3145,7 +3351,7 @@ testsst_new(RootPath, Filename, #sst_options{press_method=PressMethod, log_options=leveled_log:get_opts()}, sst_newmerge(RootPath, Filename, KVL1, KVL2, IsBasement, Level, MaxSQN, - OptsSST, false, false). + OptsSST, false). generate_randomkeys(Seqn, Count, BucketRangeLow, BucketRangeHigh) -> generate_randomkeys(Seqn, @@ -3157,16 +3363,21 @@ generate_randomkeys(Seqn, Count, BucketRangeLow, BucketRangeHigh) -> generate_randomkeys(_Seqn, 0, Acc, _BucketLow, _BucketHigh) -> Acc; generate_randomkeys(Seqn, Count, Acc, BucketLow, BRange) -> - BRand = leveled_rand:uniform(BRange), + BRand = rand:uniform(BRange), BNumber = lists:flatten(io_lib:format("B~6..0B", [BucketLow + BRand])), KNumber = - lists:flatten(io_lib:format("K~8..0B", [leveled_rand:uniform(1000000)])), - LK = leveled_codec:to_ledgerkey("Bucket" ++ BNumber, "Key" ++ KNumber, o), - Chunk = leveled_rand:rand_bytes(64), - {_B, _K, MV, _H, _LMs} = - leveled_codec:generate_ledgerkv(LK, Seqn, Chunk, 64, infinity), + lists:flatten(io_lib:format("K~8..0B", [rand:uniform(1000000)])), + LK = + leveled_codec:to_objectkey( + list_to_binary("Bucket" ++ BNumber), + list_to_binary("Key" ++ KNumber), + o + ), + Chunk = crypto:strong_rand_bytes(64), + MV = leveled_codec:convert_to_ledgerv(LK, Seqn, Chunk, 64, infinity), MD = element(4, MV), + MD =/= null orelse error(bad_type), ?assertMatch(undefined, element(3, MD)), MD0 = [{magic_md, [<<0:32/integer>>, base64:encode(Chunk)]}], MV0 = setelement(4, MV, setelement(3, MD, MD0)), @@ -3176,14 +3387,13 @@ generate_randomkeys(Seqn, Count, Acc, BucketLow, BRange) -> BucketLow, BRange). - generate_indexkeys(Count) -> generate_indexkeys(Count, []). generate_indexkeys(0, IndexList) -> IndexList; generate_indexkeys(Count, IndexList) -> - Changes = generate_indexkey(leveled_rand:uniform(8000), Count), + Changes = generate_indexkey(rand:uniform(8000), Count), generate_indexkeys(Count - 1, IndexList ++ Changes). generate_indexkey(Term, Count) -> @@ -3255,7 +3465,7 @@ tombcount_tester(Level) -> KL2 = generate_indexkeys(N div 2), FlipToTombFun = fun({K, V}) -> - case leveled_rand:uniform(10) of + case rand:uniform(10) of X when X > 5 -> {K, setelement(2, V, tomb)}; _ -> @@ -3277,18 +3487,11 @@ tombcount_tester(Level) -> {RP, Filename} = {?TEST_AREA, "tombcount_test"}, OptsSST = - #sst_options{press_method=native, - log_options=leveled_log:get_opts()}, - {ok, SST1, _KD, _BB} = sst_newmerge(RP, Filename, - KVL1, KVL2, false, Level, - N, OptsSST, false, false), - ?assertMatch(not_counted, sst_gettombcount(SST1)), - ok = sst_close(SST1), - ok = file:delete(filename:join(RP, Filename ++ ".sst")), - - {ok, SST2, _KD1, _BB1} = sst_newmerge(RP, Filename, - KVL1, KVL2, false, Level, - N, OptsSST, false, true), + #sst_options{ + press_method=native, log_options=leveled_log:get_opts()}, + {ok, SST2, _KD1, _BB1} = + sst_newmerge( + RP, Filename, KVL1, KVL2, false, Level, N, OptsSST, true), ?assertMatch(ExpectedCount, sst_gettombcount(SST2)), ok = sst_close(SST2), @@ -3299,37 +3502,42 @@ form_slot_test() -> % If a skip key happens, mustn't switch to loookup by accident as could be % over the expected size SkippingKV = - {{o, "B1", "K9999", null}, {9999, tomb, {1234568, 1234567}, {}}}, + { + {o, <<"B1">>, <<"K9999">>, null}, + {9999, tomb, {1234568, 1234567}, {}} + }, Slot = - [{{o, "B1", "K5", null}, + [{{o, <<"B1">>, <<"K5">>, null}, {5, {active, infinity}, {99234568, 99234567}, {}}}], R1 = form_slot([SkippingKV], [], {true, 99999999}, no_lookup, ?LOOK_SLOTSIZE + 1, Slot, - {o, "B1", "K5", null}), - ?assertMatch({[], [], {no_lookup, Slot}, {o, "B1", "K5", null}}, R1). + {o, <<"B1">>, <<"K5">>, null}), + ?assertMatch( + {[], [], {no_lookup, Slot}, {o, <<"B1">>, <<"K5">>, null}}, + R1 + ). merge_tombstonelist_test() -> % Merge lists with nothing but tombstones, and file at basement level SkippingKV1 = - {{o, "B1", "K9995", null}, {9995, tomb, {1234568, 1234567}, {}}}, + {{o, <<"B1">>, <<"K9995">>, null}, {9995, tomb, {1234568, 1234567}, {}}}, SkippingKV2 = - {{o, "B1", "K9996", null}, {9996, tomb, {1234568, 1234567}, {}}}, + {{o, <<"B1">>, <<"K9996">>, null}, {9996, tomb, {1234568, 1234567}, {}}}, SkippingKV3 = - {{o, "B1", "K9997", null}, {9997, tomb, {1234568, 1234567}, {}}}, + {{o, <<"B1">>, <<"K9997">>, null}, {9997, tomb, {1234568, 1234567}, {}}}, SkippingKV4 = - {{o, "B1", "K9998", null}, {9998, tomb, {1234568, 1234567}, {}}}, + {{o, <<"B1">>, <<"K9998">>, null}, {9998, tomb, {1234568, 1234567}, {}}}, SkippingKV5 = - {{o, "B1", "K9999", null}, {9999, tomb, {1234568, 1234567}, {}}}, + {{o, <<"B1">>, <<"K9999">>, null}, {9999, tomb, {1234568, 1234567}, {}}}, R = merge_lists([SkippingKV1, SkippingKV3, SkippingKV5], [SkippingKV2, SkippingKV4], {true, 9999999}, #sst_options{press_method = native, max_sstslots = 256}, - ?INDEX_MODDATE, - true), + ?INDEX_MODDATE), ?assertMatch({[], [], [], null, 0}, R). @@ -3530,15 +3738,17 @@ indexed_list_mixedkeys_bitflip_test() -> TestKey2 = element(1, lists:nth(33, KVL1)), MH1 = leveled_codec:segment_hash(TestKey1), MH2 = leveled_codec:segment_hash(TestKey2), - + test_binary_slot(SlotBin, TestKey1, MH1, lists:nth(1, KVL1)), test_binary_slot(SlotBin, TestKey2, MH2, lists:nth(33, KVL1)), ToList = binaryslot_tolist(SlotBin, native, ?INDEX_MODDATE), ?assertMatch(Keys, ToList), - [Pos1] = find_pos(PosBin, segment_checker(extract_hash(MH1))), - [Pos2] = find_pos(PosBin, segment_checker(extract_hash(MH2))), + EH1 = case extract_hash(MH1) of Int1 when is_integer(Int1) -> Int1 end, + EH2 = case extract_hash(MH2) of Int2 when is_integer(Int2) -> Int2 end, + [Pos1] = find_pos(PosBin, segment_checker(EH1)), + [Pos2] = find_pos(PosBin, segment_checker(EH2)), {BN1, _BP1} = revert_position(Pos1), {BN2, _BP2} = revert_position(Pos2), {Offset1, Length1} = block_offsetandlength(Header, BN1), @@ -3593,7 +3803,7 @@ indexed_list_mixedkeys_bitflip_test() -> flip_byte(Binary, Offset, Length) -> - Byte1 = leveled_rand:uniform(Length) + Offset - 1, + Byte1 = rand:uniform(Length) + Offset - 1, <> = Binary, case A of 0 -> @@ -3639,15 +3849,16 @@ size_tester(KVL1, KVL2, N) -> io:format(user, "~nStarting ... test with ~w keys ~n", [N]), {RP, Filename} = {?TEST_AREA, "doublesize_test"}, - OptsSST = + Opts = #sst_options{press_method=native, log_options=leveled_log:get_opts()}, - {ok, SST1, _KD, _BB} = sst_newmerge(RP, Filename, - KVL1, KVL2, false, ?DOUBLESIZE_LEVEL, - N, OptsSST, false, false), + {ok, SST1, _KD, _BB} = + sst_newmerge( + RP, Filename, KVL1, KVL2, false, ?DOUBLESIZE_LEVEL, N, Opts, false + ), ok = sst_close(SST1), {ok, SST2, _SKEK, Bloom} = - sst_open(RP, Filename ++ ".sst", OptsSST, ?DOUBLESIZE_LEVEL), + sst_open(RP, Filename ++ ".sst", Opts, ?DOUBLESIZE_LEVEL), FetchFun = fun({K, V}) -> {K0, V0} = sst_get(SST2, K), @@ -3696,8 +3907,20 @@ merge_tester(NewFunS, NewFunM) -> ?assertMatch(ExpFK2, FK2), ?assertMatch(ExpLK1, LK1), ?assertMatch(ExpLK2, LK2), - ML1 = [{next, #manifest_entry{owner = P1}, FK1}], - ML2 = [{next, #manifest_entry{owner = P2}, FK2}], + DSK = {o, <<"B">>, <<"SK">>, null}, + DEK = {o, <<"E">>, <<"EK">>, null}, + ML1 = + [{ + next, + leveled_pmanifest:new_entry(DSK, DEK, P1, "P1", none), + FK1 + }], + ML2 = + [{ + next, + leveled_pmanifest:new_entry(DSK, DEK, P2, "P2", none), + FK2 + }], NewR = NewFunM(?TEST_AREA, "level2_merge", ML1, ML2, false, 2, N * 2, native), {ok, P3, {{Rem1, Rem2}, FK3, LK3}, _Bloom3} = NewR, @@ -3795,9 +4018,14 @@ simple_persisted_rangesegfilter_tester(SSTNewFun) -> GetSegFun = fun(LK) -> - extract_hash( - leveled_codec:strip_to_segmentonly( - lists:keyfind(LK, 1, KVList1))) + case lists:keyfind(LK, 1, KVList1) of + LKV when LKV =/= false -> + extract_hash( + leveled_codec:strip_to_segmentonly( + LKV + ) + ) + end end, SegList = lists:map(GetSegFun, @@ -4042,15 +4270,18 @@ fetch_status_test() -> testsst_new(RP, Filename, 1, KVList1, length(KVList1), native), {status, Pid, {module, gen_statem}, SItemL} = sys:get_status(Pid), {data,[{"State", {reader, S}}]} = lists:nth(3, lists:nth(5, SItemL)), - true = is_integer(array:size(S#state.fetch_cache)), - true = is_integer(array:size(element(2, S#state.blockindex_cache))), + RS = S#state.read_state, + true = is_integer(array:size(RS#read_state.fetch_cache)), + true = is_integer(array:size(element(2, RS#read_state.blockindex_cache))), Status = format_status(#{reason => terminate, state => S}), ST = maps:get(state, Status), - ?assertMatch(redacted, ST#state.blockindex_cache), - ?assertMatch(redacted, ST#state.fetch_cache), + RST = ST#state.read_state, + ?assertMatch(redacted, RST#read_state.blockindex_cache), + ?assertMatch(redacted, RST#read_state.fetch_cache), NormStatus = format_status(#{reason => normal, state => S}), NST = maps:get(state, NormStatus), - ?assert(is_integer(array:size(NST#state.fetch_cache))), + RNST = NST#state.read_state, + ?assert(is_integer(array:size(RNST#read_state.fetch_cache))), ok = sst_close(Pid), ok = file:delete(filename:join(RP, Filename ++ ".sst")). @@ -4206,27 +4437,31 @@ check_binary_references(Pid) -> TotalBinMem. key_dominates_test() -> - KV1 = {{o, "Bucket", "Key1", null}, {5, {active, infinity}, 0, []}}, - KV2 = {{o, "Bucket", "Key3", null}, {6, {active, infinity}, 0, []}}, - KV3 = {{o, "Bucket", "Key2", null}, {3, {active, infinity}, 0, []}}, - KV4 = {{o, "Bucket", "Key4", null}, {7, {active, infinity}, 0, []}}, - KV5 = {{o, "Bucket", "Key1", null}, {4, {active, infinity}, 0, []}}, - KV6 = {{o, "Bucket", "Key1", null}, {99, {tomb, 999}, 0, []}}, - KV7 = {{o, "Bucket", "Key1", null}, {99, tomb, 0, []}}, + MakeKVFun = + fun(Key, SQN, Status) -> + {{o, <<"Bucket">>, Key, null}, {SQN, Status, 0, null, []}} + end, + KV1 = MakeKVFun(<<"Key1">>, 5, {active, infinity}), + KV2 = MakeKVFun(<<"Key3">>, 6, {active, infinity}), + KV3 = MakeKVFun(<<"Key2">>, 3, {active, infinity}), + KV4 = MakeKVFun(<<"Key4">>, 7, {active, infinity}), + KV5 = MakeKVFun(<<"Key1">>, 4, {active, infinity}), + KV6 = MakeKVFun(<<"Key1">>, 99, {tomb, 999}), + KV7 = MakeKVFun(<<"Key1">>, 99, tomb), KL1 = [KV1, KV2], KL2 = [KV3, KV4], ?assertMatch({{next_key, KV1}, [KV2], KL2}, - key_dominates(KL1, KL2, {undefined, 1})), + key_dominates(KL1, KL2, {false, 1})), ?assertMatch({{next_key, KV1}, KL2, [KV2]}, - key_dominates(KL2, KL1, {undefined, 1})), + key_dominates(KL2, KL1, {false, 1})), ?assertMatch({skipped_key, KL2, KL1}, - key_dominates([KV5|KL2], KL1, {undefined, 1})), + key_dominates([KV5|KL2], KL1, {false, 1})), ?assertMatch({{next_key, KV1}, [KV2], []}, - key_dominates(KL1, [], {undefined, 1})), + key_dominates(KL1, [], {false, 1})), ?assertMatch({skipped_key, [KV6|KL2], [KV2]}, - key_dominates([KV6|KL2], KL1, {undefined, 1})), + key_dominates([KV6|KL2], KL1, {false, 1})), ?assertMatch({{next_key, KV6}, KL2, [KV2]}, - key_dominates([KV6|KL2], [KV2], {undefined, 1})), + key_dominates([KV6|KL2], [KV2], {false, 1})), ?assertMatch({skipped_key, [KV6|KL2], [KV2]}, key_dominates([KV6|KL2], KL1, {true, 1})), ?assertMatch({skipped_key, [KV6|KL2], [KV2]}, @@ -4248,9 +4483,9 @@ key_dominates_test() -> ?assertMatch({skipped_key, [], []}, key_dominates([], [KV7], {true, 1})), ?assertMatch({skipped_key, [KV7|KL2], [KV2]}, - key_dominates([KV7|KL2], KL1, {undefined, 1})), + key_dominates([KV7|KL2], KL1, {false, 1})), ?assertMatch({{next_key, KV7}, KL2, [KV2]}, - key_dominates([KV7|KL2], [KV2], {undefined, 1})), + key_dominates([KV7|KL2], [KV2], {false, 1})), ?assertMatch({skipped_key, [KV7|KL2], [KV2]}, key_dominates([KV7|KL2], KL1, {true, 1})), ?assertMatch({skipped_key, KL2, [KV2]}, @@ -4277,11 +4512,9 @@ hashmatching_bytreesize_test() -> B, list_to_binary("Key" ++ integer_to_list(X)), null}, - LKV = leveled_codec:generate_ledgerkv(LK, - X, - V, - byte_size(V), - {active, infinity}), + LKV = + leveled_codec:generate_ledgerkv( + LK, X, V, byte_size(V), infinity), {_Bucket, _Key, MetaValue, _Hashes, _LastMods} = LKV, {LK, MetaValue} end, @@ -4340,8 +4573,8 @@ corrupted_block_rangetester(PressMethod, TestCount) -> KVL1 = lists:ukeysort(1, generate_randomkeys(1, N, 1, 2)), RandomRangesFun = fun(_X) -> - SKint = leveled_rand:uniform(90) + 1, - EKint = min(N, leveled_rand:uniform(N - SKint)), + SKint = rand:uniform(90) + 1, + EKint = min(N, rand:uniform(N - SKint)), SK = element(1, lists:nth(SKint, KVL1)), EK = element(1, lists:nth(EKint, KVL1)), {SK, EK} @@ -4354,7 +4587,7 @@ corrupted_block_rangetester(PressMethod, TestCount) -> B5 = serialise_block(lists:sublist(KVL1, 81, 20), PressMethod), CorruptBlockFun = fun(Block) -> - case leveled_rand:uniform(10) < 2 of + case rand:uniform(10) < 2 of true -> flip_byte(Block, 0 , byte_size(Block)); false -> @@ -4408,18 +4641,20 @@ corrupted_block_fetch_tester(PressMethod) -> CheckFun = fun(N, {AccHit, AccMiss}) -> - PosL = [min(0, leveled_rand:uniform(N - 2)), N - 1], + PosL = [min(0, rand:uniform(N - 2)), N - 1], {LK, LV} = lists:nth(N, KVL1), {BlockLengths, 0, PosBinIndex} = extract_header(Header, false), - R = check_blocks(PosL, - CorruptSlotBin, - BlockLengths, - byte_size(PosBinIndex), - LK, - PressMethod, - false, - not_present), + R = + check_blocks_matchkey( + PosL, + CorruptSlotBin, + BlockLengths, + byte_size(PosBinIndex), + LK, + PressMethod, + false + ), case R of not_present -> {AccHit, AccMiss + 1}; @@ -5004,7 +5239,11 @@ range_key_lestthanprefix_test() -> ok = file:delete(filename:join(?TEST_AREA, FileName ++ ".sst")). size_summary(P) -> - Summary = element(2, element(2, sys:get_state(P))), + Summary = + case sys:get_state(P) of + State when is_tuple(State) -> + element(2, element(2, State)) + end, true = is_record(Summary, summary), erts_debug:flat_size(Summary). @@ -5016,14 +5255,12 @@ print_compare_size(Type, OptimisedSize, UnoptimisedSize) -> % Reduced by at least a quarter ?assert(OptimisedSize < (UnoptimisedSize - (UnoptimisedSize div 4))). - single_key_test() -> FileName = "single_key_test", Field = <<"t1_bin">>, - LK = leveled_codec:to_ledgerkey(<<"Bucket0">>, <<"Key0">>, ?STD_TAG), - Chunk = leveled_rand:rand_bytes(16), - {_B, _K, MV, _H, _LMs} = - leveled_codec:generate_ledgerkv(LK, 1, Chunk, 16, infinity), + LK = leveled_codec:to_objectkey(<<"Bucket0">>, <<"Key0">>, ?STD_TAG), + Chunk = crypto:strong_rand_bytes(16), + MV = leveled_codec:convert_to_ledgerv(LK, 1, Chunk, 16, infinity), OptsSST = #sst_options{press_method=native, log_options=leveled_log:get_opts()}, @@ -5075,14 +5312,14 @@ strange_range_test() -> #sst_options{press_method=native, log_options=leveled_log:get_opts()}, - FK = leveled_codec:to_ledgerkey({<<"T0">>, <<"B0">>}, <<"K0">>, ?RIAK_TAG), - LK = leveled_codec:to_ledgerkey({<<"T0">>, <<"B0">>}, <<"K02">>, ?RIAK_TAG), - EK = leveled_codec:to_ledgerkey({<<"T0">>, <<"B0">>}, <<"K0299">>, ?RIAK_TAG), + FK = leveled_codec:to_objectkey({<<"T0">>, <<"B0">>}, <<"K0">>, ?RIAK_TAG), + LK = leveled_codec:to_objectkey({<<"T0">>, <<"B0">>}, <<"K02">>, ?RIAK_TAG), + EK = leveled_codec:to_objectkey({<<"T0">>, <<"B0">>}, <<"K0299">>, ?RIAK_TAG), KL1 = lists:map( fun(I) -> - leveled_codec:to_ledgerkey( + leveled_codec:to_objectkey( {<<"T0">>, <<"B0">>}, list_to_binary("K00" ++ integer_to_list(I)), ?RIAK_TAG) @@ -5091,7 +5328,7 @@ strange_range_test() -> KL2 = lists:map( fun(I) -> - leveled_codec:to_ledgerkey( + leveled_codec:to_objectkey( {<<"T0">>, <<"B0">>}, list_to_binary("K02" ++ integer_to_list(I)), ?RIAK_TAG) @@ -5114,8 +5351,8 @@ strange_range_test() -> {ok, P1, {FK, EK}, _Bloom1} = sst_new(?TEST_AREA, FileName, 1, KVL, 6000, OptsSST), - ?assertMatch(LK, element(1, sst_get(P1, LK))), - ?assertMatch(FK, element(1, sst_get(P1, FK))), + ?assertMatch({LK, _}, sst_get(P1, LK)), + ?assertMatch({FK, _}, sst_get(P1, FK)), ok = sst_close(P1), ok = file:delete(filename:join(?TEST_AREA, FileName ++ ".sst")), @@ -5171,7 +5408,7 @@ start_sst_fun(ProcessToInform) -> blocks_required_test() -> B = <<"Bucket">>, Idx = <<"idx_bin">>, - Chunk = leveled_rand:rand_bytes(32), + Chunk = crypto:strong_rand_bytes(32), KeyFun = fun(I) -> list_to_binary(io_lib:format("B~6..0B", [I])) @@ -5194,7 +5431,7 @@ blocks_required_test() -> element( 3, leveled_codec:generate_ledgerkv( - IdxKey(I), I, null, 0, infinity)) + IdxKey(I), I, <<>>, 0, infinity)) end, Block1L = lists:map(fun(I) -> {IdxKey(I), IdxValue(I)} end, lists:seq(1, 16)), diff --git a/src/leveled_tictac.erl b/src/leveled_tictac.erl index 5eb201d..6bdfc33 100644 --- a/src/leveled_tictac.erl +++ b/src/leveled_tictac.erl @@ -101,7 +101,7 @@ width :: integer(), segment_count :: integer(), level1 :: level1_map(), - level2 :: array:array() + level2 :: array:array(binary()) }). -type level1_map() :: #{non_neg_integer() => binary()}|binary(). @@ -161,6 +161,8 @@ new_tree(TreeID, Size, UseMap) -> width = Width, segment_count = Width * ?L2_CHUNKSIZE, level1 = Lv1Init, + % array values are indeed all binaries + % eqwalizer:ignore level2 = Lv2Init }. @@ -196,13 +198,16 @@ import_tree(ExportedTree) -> [{<<"level1">>, L1Base64}, {<<"level2">>, {struct, L2List}}]} = ExportedTree, L1Bin = base64:decode(L1Base64), - Sizes = lists:map(fun(SizeTag) -> {SizeTag, get_size(SizeTag)} end, - ?VALID_SIZES), + Sizes = + lists:map( + fun(SizeTag) -> {SizeTag, get_size(SizeTag)} end, + ?VALID_SIZES + ), Width = byte_size(L1Bin) div ?HASH_SIZE, {Size, _Width} = lists:keyfind(Width, 2, Sizes), %% assert that side is indeed the provided width true = get_size(Size) == Width, - Lv2Init = array:new([{size, Width}]), + Lv2Init = array:new([{size, Width}, {default, ?EMPTY}]), FoldFun = fun({X, EncodedL2SegBin}, L2Array) -> L2SegBin = zlib:uncompress(base64:decode(EncodedL2SegBin)), @@ -216,6 +221,8 @@ import_tree(ExportedTree) -> width = Width, segment_count = Width * ?L2_CHUNKSIZE, level1 = to_level1_map(L1Bin), + % array values are indeed all binaries + % eqwalizer:ignore level2 = Lv2 }. @@ -229,12 +236,14 @@ import_tree(ExportedTree) -> add_kv(TicTacTree, Key, Value, BinExtractFun) -> add_kv(TicTacTree, Key, Value, BinExtractFun, false). --spec add_kv( - tictactree(), term(), term(), bin_extract_fun(), boolean()) - -> tictactree()|{tictactree(), integer()}. +-spec add_kv + (tictactree(), term(), term(), bin_extract_fun(), true) -> + {tictactree(), integer()}; + (tictactree(), term(), term(), bin_extract_fun(), false) -> + tictactree(). %% @doc %% add_kv with ability to return segment ID of Key added -add_kv(TicTacTree, Key, Value, BinExtractFun, ReturnSegment) -> +add_kv(TicTacTree, Key, Value, BinExtractFun, true) -> {BinK, BinV} = BinExtractFun(Key, Value), {SegHash, SegChangeHash} = tictac_hash(BinK, BinV), Segment = get_segment(SegHash, TicTacTree#tictactree.segment_count), @@ -249,12 +258,9 @@ add_kv(TicTacTree, Key, Value, BinExtractFun, ReturnSegment) -> replace_segment( SegLeaf1Upd, SegLeaf2Upd, L1Extract, L2Extract, TicTacTree ), - case ReturnSegment of - true -> - {UpdatedTree, Segment}; - false -> - UpdatedTree - end. + {UpdatedTree, Segment}; +add_kv(TicTacTree, Key, Value, BinExtractFun, false) -> + element(1, add_kv(TicTacTree, Key, Value, BinExtractFun, true)). -spec alter_segment(integer(), integer(), tictactree()) -> tictactree(). %% @doc @@ -373,16 +379,13 @@ get_segment(Hash, TreeSize) -> %% has already been taken. If the value is not a pre-extracted hash just use %% erlang:phash2. If an exportable hash of the value is required this should %% be managed through the add_kv ExtractFun providing a pre-prepared Hash. -tictac_hash(BinKey, Val) when is_binary(BinKey) -> +tictac_hash( + BinKey, {is_hash, HashedVal}) + when is_binary(BinKey), is_integer(HashedVal) -> {HashKeyToSeg, AltHashKey} = keyto_doublesegment32(BinKey), - HashVal = - case Val of - {is_hash, HashedVal} -> - HashedVal; - _ -> - erlang:phash2(Val) - end, - {HashKeyToSeg, AltHashKey bxor HashVal}. + {HashKeyToSeg, AltHashKey bxor HashedVal}; +tictac_hash(BinKey, ValToHash) when is_binary(BinKey) -> + tictac_hash(BinKey, {is_hash, erlang:phash2(ValToHash)}). -spec keyto_doublesegment32( binary()) -> {non_neg_integer(), non_neg_integer()}. @@ -961,7 +964,7 @@ timing_test() -> timing_tester(KeyCount, SegCount, SmallSize, LargeSize) -> SegList = lists:map(fun(_C) -> - leveled_rand:uniform(get_size(SmallSize) * ?L2_CHUNKSIZE - 1) + rand:uniform(get_size(SmallSize) * ?L2_CHUNKSIZE - 1) end, lists:seq(1, SegCount)), KeyToSegFun = diff --git a/src/leveled_tree.erl b/src/leveled_tree.erl index d824c9f..15cfde2 100644 --- a/src/leveled_tree.erl +++ b/src/leveled_tree.erl @@ -29,9 +29,7 @@ -define(SKIP_WIDTH, 16). -type tree_type() :: tree|idxt|skpl. --type leveled_tree() :: {tree_type(), - integer(), % length - any()}. +-type leveled_tree() :: {tree_type(), integer(), any()}. -export_type([leveled_tree/0]). @@ -104,7 +102,7 @@ match(Key, {tree, _L, Tree}) -> {_NK, SL, _Iter} -> lookup_match(Key, SL) end; -match(Key, {idxt, _L, {TLI, IDX}}) -> +match(Key, {idxt, _L, {TLI, IDX}}) when is_tuple(TLI) -> Iter = tree_iterator_from(Key, IDX), case tree_next(Iter) of none -> @@ -136,7 +134,7 @@ search(Key, {tree, _L, Tree}, StartKeyFun) -> {K, V} end end; -search(Key, {idxt, _L, {TLI, IDX}}, StartKeyFun) -> +search(Key, {idxt, _L, {TLI, IDX}}, StartKeyFun) when is_tuple(TLI) -> Iter = tree_iterator_from(Key, IDX), case tree_next(Iter) of none -> @@ -235,14 +233,13 @@ to_list({tree, _L, Tree}) -> Acc ++ SL end, lists:foldl(FoldFun, [], tree_to_list(Tree)); -to_list({idxt, _L, {TLI, _IDX}}) -> +to_list({idxt, _L, {TLI, _IDX}}) when is_tuple(TLI) -> lists:append(tuple_to_list(TLI)); -to_list({skpl, _L, SkipList}) -> +to_list({skpl, _L, SkipList}) when is_list(SkipList) -> FoldFun = fun({_M, SL}, Acc) -> [SL|Acc] end, - Lv1List = lists:reverse(lists:foldl(FoldFun, [], SkipList)), Lv0List = lists:reverse(lists:foldl(FoldFun, [], lists:append(Lv1List))), lists:append(Lv0List). @@ -580,13 +577,13 @@ generate_randomkeys(Seqn, Count, BucketRangeLow, BucketRangeHigh) -> generate_randomkeys(_Seqn, 0, Acc, _BucketLow, _BucketHigh) -> Acc; generate_randomkeys(Seqn, Count, Acc, BucketLow, BRange) -> - BRand = leveled_rand:uniform(BRange), + BRand = rand:uniform(BRange), BNumber = lists:flatten( io_lib:format("K~4..0B", [BucketLow + BRand])), KNumber = lists:flatten( - io_lib:format("K~8..0B", [leveled_rand:uniform(1000)])), + io_lib:format("K~8..0B", [rand:uniform(1000)])), {K, V} = {{o_kv, {<<"btype">>, list_to_binary("Bucket" ++ BNumber)}, @@ -608,7 +605,7 @@ generate_simplekeys(Seqn, Count, Acc) -> KNumber = list_to_binary( lists:flatten( - io_lib:format("K~8..0B", [leveled_rand:uniform(100000)]))), + io_lib:format("K~8..0B", [rand:uniform(100000)]))), generate_simplekeys(Seqn + 1, Count - 1, [{KNumber, Seqn}|Acc]). @@ -958,14 +955,13 @@ search_range_idx_test() -> {o_rkv,"Bucket1","Key1",null}, "<0.320.0>","./16_1_6.sst", none}}]}, {1,{{o_rkv,"Bucket1","Key1",null},1,nil,nil}}}}, - StartKeyFun = - fun(ME) -> - ME#manifest_entry.start_key - end, - R = search_range({o_rkv, "Bucket", null, null}, - {o_rkv, "Bucket", null, null}, - Tree, - StartKeyFun), + R = + search_range( + {o_rkv, "Bucket", null, null}, + {o_rkv, "Bucket", null, null}, + Tree, + fun leveled_pmanifest:entry_startkey/1 + ), ?assertMatch(1, length(R)). -endif. diff --git a/src/leveled_util.erl b/src/leveled_util.erl index ac8c439..630e9f4 100644 --- a/src/leveled_util.erl +++ b/src/leveled_util.erl @@ -23,7 +23,7 @@ %% Credit to %% https://github.com/afiskon/erlang-uuid-v4/blob/master/src/uuid.erl generate_uuid() -> - <> = leveled_rand:rand_bytes(16), + <> = crypto:strong_rand_bytes(16), L = io_lib:format("~8.16.0b-~4.16.0b-4~3.16.0b-~4.16.0b-~12.16.0b", [A, B, C band 16#0fff, D band 16#3fff bor 16#8000, E]), binary_to_list(list_to_binary(L)). diff --git a/test/end_to_end/appdefined_SUITE.erl b/test/end_to_end/appdefined_SUITE.erl index 4c508a0..906249a 100644 --- a/test/end_to_end/appdefined_SUITE.erl +++ b/test/end_to_end/appdefined_SUITE.erl @@ -73,7 +73,7 @@ application_defined_tag_tester(KeyCount, Tag, Functions, ExpectMD) -> [{bespoke_tag1, retain}, {bespoke_tag2, retain}]}, {override_functions, Functions}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), - Value = leveled_rand:rand_bytes(512), + Value = crypto:strong_rand_bytes(512), MapFun = fun(C) -> {C, object_generator(C, Value)} @@ -119,7 +119,7 @@ application_defined_tag_tester(KeyCount, Tag, Functions, ExpectMD) -> object_generator(Count, V) -> Hash = erlang:phash2({count, V}), - Random = leveled_rand:uniform(1000), + Random = rand:uniform(1000), Key = list_to_binary(leveled_util:generate_uuid()), Bucket = <<"B">>, {Bucket, diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 22c7b91..13beb8d 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -55,15 +55,19 @@ simple_put_fetch_head_delete(_Config) -> simple_test_withlog(LogLevel, ForcedLogs) -> RootPath = testutil:reset_filestructure(), - StartOpts1 = [{root_path, RootPath}, - {sync_strategy, testutil:sync_strategy()}, - {log_level, LogLevel}, - {forced_logs, ForcedLogs}], + StartOpts1 = + [ + {root_path, RootPath}, + {sync_strategy, testutil:sync_strategy()}, + {log_level, LogLevel}, + {forced_logs, ForcedLogs}, + {max_pencillercachesize, 200} + ], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = testutil:book_riakput(Bookie1, TestObject, TestSpec), testutil:check_forobject(Bookie1, TestObject), - testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"), + testutil:check_formissingobject(Bookie1, <<"Bucket1">>, <<"Key2">>), ok = leveled_bookie:book_close(Bookie1), StartOpts2 = [{root_path, RootPath}, {max_journalsize, 3000000}, @@ -78,29 +82,49 @@ simple_test_withlog(LogLevel, ForcedLogs) -> ChkList1 = lists:sublist(lists:sort(ObjList1), 100), testutil:check_forlist(Bookie2, ChkList1), testutil:check_forobject(Bookie2, TestObject), - testutil:check_formissingobject(Bookie2, "Bucket1", "Key2"), - ok = leveled_bookie:book_put(Bookie2, "Bucket1", "Key2", "Value2", - [{add, "Index1", "Term1"}]), - {ok, "Value2"} = leveled_bookie:book_get(Bookie2, "Bucket1", "Key2"), - {ok, {62888926, S, undefined}} = - leveled_bookie:book_head(Bookie2, "Bucket1", "Key2"), - true = (S == 58) or (S == 60), + testutil:check_formissingobject(Bookie2, <<"Bucket1">>, <<"Key2">>), + ok = + leveled_bookie:book_put( + Bookie2, + <<"Bucket1">>, + <<"Key2">>, + <<"Value2">>, + [{add, <<"Index1">>, <<"Term1">>}] + ), + {ok, <<"Value2">>} = + leveled_bookie:book_get(Bookie2, <<"Bucket1">>, <<"Key2">>), + {ok, {2220864, S, undefined}} = + leveled_bookie:book_head(Bookie2, <<"Bucket1">>, <<"Key2">>), + true = (S == 63) or (S == 65), % After OTP 26 the object is 58 bytes not 60 - testutil:check_formissingobject(Bookie2, "Bucket1", "Key2"), - ok = leveled_bookie:book_put(Bookie2, "Bucket1", "Key2", <<"Value2">>, - [{remove, "Index1", "Term1"}, - {add, "Index1", <<"Term2">>}]), - {ok, <<"Value2">>} = leveled_bookie:book_get(Bookie2, "Bucket1", "Key2"), + testutil:check_formissingobject(Bookie2, <<"Bucket1">>, <<"Key2">>), + ok = + leveled_bookie:book_put( + Bookie2, + <<"Bucket1">>, + <<"Key2">>, + <<"Value2">>, + [{remove, <<"Index1">>, <<"Term1">>}, + {add, <<"Index1">>, <<"Term2">>}] + ), + {ok, <<"Value2">>} = + leveled_bookie:book_get(Bookie2, <<"Bucket1">>, <<"Key2">>), ok = leveled_bookie:book_close(Bookie2), {ok, Bookie3} = leveled_bookie:book_start(StartOpts2), - {ok, <<"Value2">>} = leveled_bookie:book_get(Bookie3, "Bucket1", "Key2"), - ok = leveled_bookie:book_delete(Bookie3, "Bucket1", "Key2", - [{remove, "Index1", "Term1"}]), - not_found = leveled_bookie:book_get(Bookie3, "Bucket1", "Key2"), - not_found = leveled_bookie:book_head(Bookie3, "Bucket1", "Key2"), + {ok, <<"Value2">>} = + leveled_bookie:book_get(Bookie3, <<"Bucket1">>, <<"Key2">>), + ok = + leveled_bookie:book_delete( + Bookie3, + <<"Bucket1">>, + <<"Key2">>, + [{remove, <<"Index1">>, <<"Term1">>}] + ), + not_found = leveled_bookie:book_get(Bookie3, <<"Bucket1">>, <<"Key2">>), + not_found = leveled_bookie:book_head(Bookie3, <<"Bucket1">>, <<"Key2">>), ok = leveled_bookie:book_close(Bookie3), {ok, Bookie4} = leveled_bookie:book_start(StartOpts2), - not_found = leveled_bookie:book_get(Bookie4, "Bucket1", "Key2"), + not_found = leveled_bookie:book_get(Bookie4, <<"Bucket1">>, <<"Key2">>), ok = leveled_bookie:book_destroy(Bookie4). many_put_fetch_head(_Config) -> @@ -168,7 +192,7 @@ many_put_fetch_head(_Config) -> not_found = leveled_bookie:book_sqn(Bookie3, testutil:get_bucket(TestObject), testutil:get_key(TestObject)), - testutil:check_formissingobject(Bookie3, "Bookie1", "MissingKey0123"), + testutil:check_formissingobject(Bookie3, <<"Bookie1">>, <<"MissingKey0123">>), ok = leveled_bookie:book_destroy(Bookie3). bigjournal_littlejournal(_Config) -> @@ -181,7 +205,7 @@ bigjournal_littlejournal(_Config) -> {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), ObjL1 = testutil:generate_objects(100, 1, [], - leveled_rand:rand_bytes(10000), + crypto:strong_rand_bytes(10000), fun() -> [] end, <<"B">>), testutil:riakload(Bookie1, ObjL1), ok = leveled_bookie:book_close(Bookie1), @@ -189,7 +213,7 @@ bigjournal_littlejournal(_Config) -> {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), ObjL2 = testutil:generate_objects(10, 1000, [], - leveled_rand:rand_bytes(10000), + crypto:strong_rand_bytes(10000), fun() -> [] end, <<"B">>), testutil:riakload(Bookie2, ObjL2), testutil:check_forlist(Bookie2, ObjL1), @@ -214,7 +238,7 @@ bigsst_littlesst(_Config) -> 100000, 1, [], - leveled_rand:rand_bytes(100), + crypto:strong_rand_bytes(100), fun() -> [] end, <<"B">>) ), @@ -260,13 +284,16 @@ journal_compaction_tester(Restart, WRP) -> ChkList1 = lists:sublist(lists:sort(ObjList1), 10000), testutil:check_forlist(Bookie0, ChkList1), testutil:check_forobject(Bookie0, TestObject), - {B2, K2, V2, Spec2, MD} = {"Bucket2", - "Key2", - "Value2", - [], - [{"MDK2", "MDV2"}]}, - {TestObject2, TestSpec2} = testutil:generate_testobject(B2, K2, - V2, Spec2, MD), + {B2, K2, V2, Spec2, MD} = + { + <<"Bucket2">>, + <<"Key2">>, + <<"Value2">>, + [], + [{<<"MDK2">>, <<"MDV2">>}] + }, + {TestObject2, TestSpec2} = + testutil:generate_testobject(B2, K2, V2, Spec2, MD), ok = testutil:book_riakput(Bookie0, TestObject2, TestSpec2), ok = leveled_bookie:book_compactjournal(Bookie0, 30000), testutil:check_forlist(Bookie0, ChkList1), @@ -277,13 +304,15 @@ journal_compaction_tester(Restart, WRP) -> testutil:check_forobject(Bookie0, TestObject2), %% Delete some of the objects ObjListD = testutil:generate_objects(10000, 2), - lists:foreach(fun({_R, O, _S}) -> - testutil:book_riakdelete(Bookie0, - testutil:get_bucket(O), - testutil:get_key(O), - []) - end, - ObjListD), + lists:foreach( + fun({_R, O, _S}) -> + testutil:book_riakdelete(Bookie0, + testutil:get_bucket(O), + testutil:get_key(O), + []) + end, + ObjListD + ), %% Now replace all the other objects ObjList2 = testutil:generate_objects(40000, 10002), @@ -539,11 +568,11 @@ fetchput_snapshot(_Config) -> % smaller due to replacements and files deleting % This is dependent on the sleep though (yuk) - {B1Size, B1Count} = testutil:check_bucket_stats(Bookie2, "Bucket1"), + {B1Size, B1Count} = testutil:check_bucket_stats(Bookie2, <<"Bucket1">>), true = B1Size > 0, true = B1Count == 1, - {B1Size, B1Count} = testutil:check_bucket_stats(Bookie2, "Bucket1"), - {BSize, BCount} = testutil:check_bucket_stats(Bookie2, "Bucket"), + {B1Size, B1Count} = testutil:check_bucket_stats(Bookie2, <<"Bucket1">>), + {BSize, BCount} = testutil:check_bucket_stats(Bookie2, <<"Bucket">>), true = BSize > 0, true = BCount == 180000, @@ -622,82 +651,78 @@ load_and_count(JournalSize, BookiesMemSize, PencillerMemSize) -> testutil:check_forobject(Bookie1, TestObject), io:format("Loading initial small objects~n"), G1 = fun testutil:generate_smallobjects/2, - lists:foldl(fun(_X, Acc) -> - testutil:load_objects(5000, - [Acc + 2], - Bookie1, - TestObject, - G1), - {_S, Count} = - testutil:check_bucket_stats(Bookie1, "Bucket"), - if - Acc + 5000 == Count -> - ok - end, - Acc + 5000 end, - 0, - lists:seq(1, 20)), + lists:foldl( + fun(_X, Acc) -> + testutil:load_objects( + 5000, [Acc + 2], Bookie1, TestObject, G1), + {_S, Count} = + testutil:check_bucket_stats(Bookie1, <<"Bucket">>), + if + Acc + 5000 == Count -> + ok + end, + Acc + 5000 end, + 0, + lists:seq(1, 20) + ), testutil:check_forobject(Bookie1, TestObject), io:format("Loading larger compressible objects~n"), G2 = fun testutil:generate_compressibleobjects/2, - lists:foldl(fun(_X, Acc) -> - testutil:load_objects(5000, - [Acc + 2], - Bookie1, - TestObject, - G2), - {_S, Count} = - testutil:check_bucket_stats(Bookie1, "Bucket"), - if - Acc + 5000 == Count -> - ok - end, - Acc + 5000 end, - 100000, - lists:seq(1, 20)), + lists:foldl( + fun(_X, Acc) -> + testutil:load_objects( + 5000, [Acc + 2], Bookie1, TestObject, G2), + {_S, Count} = + testutil:check_bucket_stats(Bookie1, <<"Bucket">>), + if + Acc + 5000 == Count -> + ok + end, + Acc + 5000 end, + 100000, + lists:seq(1, 20) + ), testutil:check_forobject(Bookie1, TestObject), io:format("Replacing small objects~n"), - lists:foldl(fun(_X, Acc) -> - testutil:load_objects(5000, - [Acc + 2], - Bookie1, - TestObject, - G1), - {_S, Count} = - testutil:check_bucket_stats(Bookie1, "Bucket"), - if - Count == 200000 -> - ok - end, - Acc + 5000 end, - 0, - lists:seq(1, 20)), + lists:foldl( + fun(_X, Acc) -> + testutil:load_objects( + 5000, [Acc + 2], Bookie1, TestObject, G1), + {_S, Count} = + testutil:check_bucket_stats(Bookie1, <<"Bucket">>), + if + Count == 200000 -> + ok + end, + Acc + 5000 end, + 0, + lists:seq(1, 20) + ), testutil:check_forobject(Bookie1, TestObject), io:format("Loading more small objects~n"), io:format("Now with unused snapshot so deletions are blocked~n"), {ok, PclClone, null} = leveled_bookie:book_snapshot(Bookie1, ledger, undefined, true), - lists:foldl(fun(_X, Acc) -> - testutil:load_objects(5000, - [Acc + 2], - Bookie1, - TestObject, - G2), - {_S, Count} = - testutil:check_bucket_stats(Bookie1, "Bucket"), - if - Acc + 5000 == Count -> - ok - end, - Acc + 5000 end, - 200000, - lists:seq(1, 20)), + lists:foldl( + fun(_X, Acc) -> + testutil:load_objects( + 5000, [Acc + 2], Bookie1, TestObject, G2), + {_S, Count} = + testutil:check_bucket_stats(Bookie1, <<"Bucket">>), + if + Acc + 5000 == Count -> + ok + end, + Acc + 5000 end, + 200000, + lists:seq(1, 20) + ), testutil:check_forobject(Bookie1, TestObject), ok = leveled_penciller:pcl_close(PclClone), - {_S, 300000} = testutil:check_bucket_stats(Bookie1, "Bucket"), + {_S, 300000} = testutil:check_bucket_stats(Bookie1, <<"Bucket">>), ok = leveled_bookie:book_close(Bookie1), {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), - {_, 300000} = testutil:check_bucket_stats(Bookie2, "Bucket"), + {_, 300000} = testutil:check_bucket_stats(Bookie2, <<"Bucket">>), ok = leveled_bookie:book_close(Bookie2), @@ -722,21 +747,19 @@ load_and_count_withdelete(_Config) -> testutil:check_forobject(Bookie1, TestObject), io:format("Loading initial small objects~n"), G1 = fun testutil:generate_smallobjects/2, - lists:foldl(fun(_X, Acc) -> - testutil:load_objects(5000, - [Acc + 2], - Bookie1, - TestObject, - G1), - {_S, Count} = testutil:check_bucket_stats(Bookie1, - "Bucket"), - if - Acc + 5000 == Count -> - ok - end, - Acc + 5000 end, - 0, - lists:seq(1, 20)), + lists:foldl( + fun(_X, Acc) -> + testutil:load_objects( + 5000, [Acc + 2], Bookie1, TestObject, G1), + {_S, Count} = testutil:check_bucket_stats(Bookie1, <<"Bucket">>), + if + Acc + 5000 == Count -> + ok + end, + Acc + 5000 end, + 0, + lists:seq(1, 20) + ), testutil:check_forobject(Bookie1, TestObject), {BucketD, KeyD} = {testutil:get_bucket(TestObject), testutil:get_key(TestObject)}, @@ -746,21 +769,19 @@ load_and_count_withdelete(_Config) -> {_, 0} = testutil:check_bucket_stats(Bookie1, BucketD), io:format("Loading larger compressible objects~n"), G2 = fun testutil:generate_compressibleobjects/2, - lists:foldl(fun(_X, Acc) -> - testutil:load_objects(5000, - [Acc + 2], - Bookie1, - no_check, - G2), - {_S, Count} = testutil:check_bucket_stats(Bookie1, - "Bucket"), - if - Acc + 5000 == Count -> - ok - end, - Acc + 5000 end, - 100000, - lists:seq(1, 20)), + lists:foldl( + fun(_X, Acc) -> + testutil:load_objects( + 5000, [Acc + 2], Bookie1, no_check, G2), + {_S, Count} = testutil:check_bucket_stats(Bookie1, <<"Bucket">>), + if + Acc + 5000 == Count -> + ok + end, + Acc + 5000 end, + 100000, + lists:seq(1, 20) + ), not_found = testutil:book_riakget(Bookie1, BucketD, KeyD), ok = leveled_bookie:book_close(Bookie1), @@ -780,11 +801,8 @@ space_clear_ondelete(_Config) -> {sync_strategy, testutil:sync_strategy()}], {ok, Book1} = leveled_bookie:book_start(StartOpts1), G2 = fun testutil:generate_compressibleobjects/2, - testutil:load_objects(20000, - [uuid, uuid, uuid, uuid], - Book1, - no_check, - G2), + testutil:load_objects( + 20000, [uuid, uuid, uuid, uuid], Book1, no_check, G2), FoldKeysFun = fun(B, K, Acc) -> [{B, K}|Acc] end, @@ -808,10 +826,9 @@ space_clear_ondelete(_Config) -> FoldObjectsFun = fun(B, K, ObjBin, Acc) -> [{B, K, erlang:phash2(ObjBin)}|Acc] end, - {async, HTreeF1} = leveled_bookie:book_objectfold(Book1, - ?RIAK_TAG, - {FoldObjectsFun, []}, - false), + {async, HTreeF1} = + leveled_bookie:book_objectfold( + Book1, ?RIAK_TAG, {FoldObjectsFun, []}, false), % This query does not Snap PreFold - and so will not prevent % pending deletes from prompting actual deletes @@ -822,32 +839,34 @@ space_clear_ondelete(_Config) -> % Delete the keys SW2 = os:timestamp(), - lists:foreach(fun({Bucket, Key}) -> - testutil:book_riakdelete(Book1, - Bucket, - Key, - []) - end, - KL1), - io:format("Deletion took ~w microseconds for 80K keys~n", - [timer:now_diff(os:timestamp(), SW2)]), - - + lists:foreach( + fun({Bucket, Key}) -> + testutil:book_riakdelete(Book1, Bucket, Key, []) + end, + KL1), + io:format( + "Deletion took ~w microseconds for 80K keys~n", + [timer:now_diff(os:timestamp(), SW2)]), ok = leveled_bookie:book_compactjournal(Book1, 30000), F = fun leveled_bookie:book_islastcompactionpending/1, - lists:foldl(fun(X, Pending) -> - case Pending of - false -> - false; - true -> - io:format("Loop ~w waiting for journal " - ++ "compaction to complete~n", [X]), - timer:sleep(20000), - F(Book1) - end end, - true, - lists:seq(1, 15)), + lists:foldl( + fun(X, Pending) -> + case Pending of + false -> + false; + true -> + io:format( + "Loop ~w waiting for journal " + "compaction to complete~n", + [X] + ), + timer:sleep(20000), + F(Book1) + end + end, + true, + lists:seq(1, 15)), io:format("Waiting for journal deletes - blocked~n"), timer:sleep(20000), @@ -1113,7 +1132,7 @@ many_put_fetch_switchcompression_tester(CompressionMethod) -> %% Change method back again {ok, Bookie3} = leveled_bookie:book_start(StartOpts1), - testutil:check_formissingobject(Bookie3, "Bookie1", "MissingKey0123"), + testutil:check_formissingobject(Bookie3, <<"Bookie1">>, "MissingKey0123"), lists:foreach( fun(CL) -> ok = testutil:check_forlist(Bookie3, CL) end, CL2s), lists:foreach( @@ -1244,10 +1263,12 @@ bigpcl_bucketlist(_Config) -> MapFun = fun(B) -> - testutil:generate_objects(ObjectCount, 1, [], - leveled_rand:rand_bytes(100), - fun() -> [] end, - B) + testutil:generate_objects( + ObjectCount, 1, [], + crypto:strong_rand_bytes(100), + fun() -> [] end, + B + ) end, ObjLofL = lists:map(MapFun, BucketList), lists:foreach(fun(ObjL) -> testutil:riakload(Bookie1, ObjL) end, ObjLofL), @@ -1263,11 +1284,15 @@ bigpcl_bucketlist(_Config) -> FBAccT = {BucketFold, sets:new()}, {async, BucketFolder1} = - leveled_bookie:book_headfold(Bookie1, - ?RIAK_TAG, - {bucket_list, BucketList}, - FBAccT, - false, false, false), + leveled_bookie:book_headfold( + Bookie1, + ?RIAK_TAG, + {bucket_list, BucketList}, + FBAccT, + false, + false, + false + ), {FoldTime1, BucketList1} = timer:tc(BucketFolder1, []), true = BucketCount == sets:size(BucketList1), @@ -1276,11 +1301,15 @@ bigpcl_bucketlist(_Config) -> {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), {async, BucketFolder2} = - leveled_bookie:book_headfold(Bookie2, - ?RIAK_TAG, - {bucket_list, BucketList}, - FBAccT, - false, false, false), + leveled_bookie:book_headfold( + Bookie2, + ?RIAK_TAG, + {bucket_list, BucketList}, + FBAccT, + false, + false, + false + ), {FoldTime2, BucketList2} = timer:tc(BucketFolder2, []), true = BucketCount == sets:size(BucketList2), diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index 15236f6..855aad8 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -57,7 +57,7 @@ expiring_indexes(_Config) -> Indexes9 = testutil:get_randomindexes_generator(2), TempRiakObjects = testutil:generate_objects( - KeyCount, binary_uuid, [], V9, Indexes9, "riakBucket"), + KeyCount, binary_uuid, [], V9, Indexes9, <<"riakBucket">>), IBKL1 = testutil:stdload_expiring(Bookie1, KeyCount, Future), lists:foreach( @@ -147,11 +147,13 @@ expiring_indexes(_Config) -> Bookie1, B0, K0, 5, <<"value">>, leveled_util:integer_now() + 10), timer:sleep(1000), {async, Folder2} = IndexFold(), - leveled_bookie:book_indexfold(Bookie1, - B0, - {FoldFun, InitAcc}, - {<<"temp_int">>, 5, 8}, - {true, undefined}), + leveled_bookie:book_indexfold( + Bookie1, + B0, + {FoldFun, InitAcc}, + {<<"temp_int">>, 5, 8}, + {true, undefined} + ), QR2 = Folder2(), io:format("Query with additional entry length ~w~n", [length(QR2)]), true = lists:sort(QR2) == lists:sort([{5, B0, K0}|LoadedEntriesInRange]), @@ -208,11 +210,9 @@ breaking_folds(_Config) -> {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), ObjectGen = testutil:get_compressiblevalue_andinteger(), IndexGen = testutil:get_randomindexes_generator(8), - ObjL1 = testutil:generate_objects(KeyCount, - binary_uuid, - [], - ObjectGen, - IndexGen), + ObjL1 = + testutil:generate_objects( + KeyCount, binary_uuid, [], ObjectGen, IndexGen), testutil:riakload(Bookie1, ObjL1), % Find all keys index, and then same again but stop at a midpoint using a @@ -261,7 +261,6 @@ breaking_folds(_Config) -> io:format("Index fold with result size ~w~n", [length(KeyList2)]), true = KeyCount div 2 == length(KeyList2), - HeadFoldFun = fun(_B, K, PO, Acc) -> {proxy_object, _MDBin, Size, _FF} = binary_to_term(PO), @@ -287,10 +286,14 @@ breaking_folds(_Config) -> end end, {async, HeadFolderToMidK} = - leveled_bookie:book_headfold(Bookie1, - ?RIAK_TAG, - {FoldThrowFun(HeadFoldFun), []}, - true, true, false), + leveled_bookie:book_headfold( + Bookie1, + ?RIAK_TAG, + {FoldThrowFun(HeadFoldFun), []}, + true, + true, + false + ), KeySizeList2 = lists:reverse(CatchingFold(HeadFolderToMidK)), io:format("Head fold with result size ~w~n", [length(KeySizeList2)]), true = KeyCount div 2 == length(KeySizeList2), @@ -300,21 +303,25 @@ breaking_folds(_Config) -> [{K,byte_size(V)}|Acc] end, {async, ObjectFolderKO} = - leveled_bookie:book_objectfold(Bookie1, - ?RIAK_TAG, - {ObjFoldFun, []}, - false, - key_order), + leveled_bookie:book_objectfold( + Bookie1, + ?RIAK_TAG, + {ObjFoldFun, []}, + false, + key_order + ), ObjSizeList1 = lists:reverse(ObjectFolderKO()), io:format("Obj fold with result size ~w~n", [length(ObjSizeList1)]), true = KeyCount == length(ObjSizeList1), {async, ObjFolderToMidK} = - leveled_bookie:book_objectfold(Bookie1, - ?RIAK_TAG, - {FoldThrowFun(ObjFoldFun), []}, - false, - key_order), + leveled_bookie:book_objectfold( + Bookie1, + ?RIAK_TAG, + {FoldThrowFun(ObjFoldFun), []}, + false, + key_order + ), ObjSizeList2 = lists:reverse(CatchingFold(ObjFolderToMidK)), io:format("Object fold with result size ~w~n", [length(ObjSizeList2)]), true = KeyCount div 2 == length(ObjSizeList2), @@ -324,11 +331,13 @@ breaking_folds(_Config) -> % that was terminated by reaching a point in the key range .. as results % will not be passed to the fold function in key order {async, ObjectFolderSO} = - leveled_bookie:book_objectfold(Bookie1, - ?RIAK_TAG, - {ObjFoldFun, []}, - false, - sqn_order), + leveled_bookie:book_objectfold( + Bookie1, + ?RIAK_TAG, + {ObjFoldFun, []}, + false, + sqn_order + ), ObjSizeList1_SO = lists:reverse(ObjectFolderSO()), io:format("Obj fold with result size ~w~n", [length(ObjSizeList1_SO)]), true = KeyCount == length(ObjSizeList1_SO), @@ -346,33 +355,26 @@ breaking_folds(_Config) -> end end, {async, ObjFolderTo1K} = - leveled_bookie:book_objectfold(Bookie1, - ?RIAK_TAG, - {FoldThrowThousandFun(ObjFoldFun), []}, - false, - sqn_order), + leveled_bookie:book_objectfold( + Bookie1, + ?RIAK_TAG, + {FoldThrowThousandFun(ObjFoldFun), []}, + false, + sqn_order + ), ObjSizeList2_SO = lists:reverse(CatchingFold(ObjFolderTo1K)), io:format("Object fold with result size ~w~n", [length(ObjSizeList2_SO)]), true = 1000 == length(ObjSizeList2_SO), - ObjL2 = testutil:generate_objects(10, - binary_uuid, - [], - ObjectGen, - IndexGen, - "B2"), - ObjL3 = testutil:generate_objects(10, - binary_uuid, - [], - ObjectGen, - IndexGen, - "B3"), - ObjL4 = testutil:generate_objects(10, - binary_uuid, - [], - ObjectGen, - IndexGen, - "B4"), + ObjL2 = + testutil:generate_objects( + 10, binary_uuid, [], ObjectGen, IndexGen, <<"B2">>), + ObjL3 = + testutil:generate_objects( + 10, binary_uuid, [], ObjectGen, IndexGen, <<"B3">>), + ObjL4 = + testutil:generate_objects( + 10, binary_uuid, [], ObjectGen, IndexGen, <<"B4">>), testutil:riakload(Bookie1, ObjL2), testutil:riakload(Bookie1, ObjL3), testutil:riakload(Bookie1, ObjL4), @@ -396,20 +398,16 @@ breaking_folds(_Config) -> end, {async, StopAt3BucketFolder} = - leveled_bookie:book_bucketlist(Bookie1, - ?RIAK_TAG, - {StopAt3Fun, []}, - all), + leveled_bookie:book_bucketlist( + Bookie1, ?RIAK_TAG, {StopAt3Fun, []}, all), BucketListSA3 = lists:reverse(CatchingFold(StopAt3BucketFolder)), io:format("bucket list with result ~w~n", [BucketListSA3]), true = [<<"B2">>, <<"B3">>] == BucketListSA3, - ok = leveled_bookie:book_close(Bookie1), testutil:reset_filestructure(). - single_object_with2i(_Config) -> % Load a single object with an integer and a binary % index and query for it @@ -429,36 +427,40 @@ single_object_with2i(_Config) -> {async, IdxFolder1} = leveled_bookie:book_indexfold( Bookie1, - "Bucket1", + <<"Bucket1">>, {fun testutil:foldkeysfun/3, []}, {list_to_binary("binary_bin"), <<99:32/integer>>, <<101:32/integer>>}, {true, undefined}), R1 = IdxFolder1(), io:format("R1 of ~w~n", [R1]), - true = [{<<100:32/integer>>,"Key1"}] == R1, + true = [{<<100:32/integer>>, <<"Key1">>}] == R1, - IdxQ2 = {index_query, - "Bucket1", - {fun testutil:foldkeysfun/3, []}, - {list_to_binary("integer_int"), - 99, 101}, - {true, undefined}}, + IdxQ2 = + { + index_query, + <<"Bucket1">>, + {fun testutil:foldkeysfun/3, []}, + {list_to_binary("integer_int"), 99, 101}, + {true, undefined} + }, {async, IdxFolder2} = leveled_bookie:book_returnfolder(Bookie1, IdxQ2), R2 = IdxFolder2(), io:format("R2 of ~w~n", [R2]), - true = [{100,"Key1"}] == R2, + true = [{100, <<"Key1">>}] == R2, - IdxQ3 = {index_query, - {"Bucket1", "Key1"}, - {fun testutil:foldkeysfun/3, []}, - {list_to_binary("integer_int"), - 99, 101}, - {true, undefined}}, + IdxQ3 = + { + index_query, + {<<"Bucket1">>, <<"Key1">>}, + {fun testutil:foldkeysfun/3, []}, + {list_to_binary("integer_int"), 99, 101}, + {true, undefined} + }, {async, IdxFolder3} = leveled_bookie:book_returnfolder(Bookie1, IdxQ3), R3 = IdxFolder3(), io:format("R2 of ~w~n", [R3]), - true = [{100,"Key1"}] == R3, + true = [{100, <<"Key1">>}] == R3, ok = leveled_bookie:book_close(Bookie1), testutil:reset_filestructure(). @@ -473,7 +475,7 @@ small_load_with2i(_Config) -> {TestObject, TestSpec} = testutil:generate_testobject(), ok = testutil:book_riakput(Bookie1, TestObject, TestSpec), testutil:check_forobject(Bookie1, TestObject), - testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"), + testutil:check_formissingobject(Bookie1, <<"Bucket1">>, <<"Key2">>), testutil:check_forobject(Bookie1, TestObject), ObjectGen = testutil:get_compressiblevalue_andinteger(), IndexGen = testutil:get_randomindexes_generator(8), @@ -486,58 +488,60 @@ small_load_with2i(_Config) -> testutil:check_forobject(Bookie1, TestObject), % Find all keys index, and then just the last key - IdxQ1 = {index_query, - "Bucket", - {fun testutil:foldkeysfun/3, []}, - {<<"idx1_bin">>, <<"#">>, <<"|">>}, - {true, undefined}}, + IdxQ1 = + { + index_query, + <<"Bucket">>, + {fun testutil:foldkeysfun/3, []}, + {<<"idx1_bin">>, <<"#">>, <<"|">>}, + {true, undefined} + }, {async, IdxFolder} = leveled_bookie:book_returnfolder(Bookie1, IdxQ1), KeyList1 = lists:usort(IdxFolder()), true = 10000 == length(KeyList1), {LastTerm, LastKey} = lists:last(KeyList1), - IdxQ2 = {index_query, - {"Bucket", LastKey}, - {fun testutil:foldkeysfun/3, []}, - {<<"idx1_bin">>, LastTerm, <<"|">>}, - {false, undefined}}, + IdxQ2 = + { + index_query, + {<<"Bucket">>, LastKey}, + {fun testutil:foldkeysfun/3, []}, + {<<"idx1_bin">>, LastTerm, <<"|">>}, + {false, undefined} + }, {async, IdxFolderLK} = leveled_bookie:book_returnfolder(Bookie1, IdxQ2), KeyList2 = lists:usort(IdxFolderLK()), io:format("List should be last key ~w ~w~n", [LastKey, KeyList2]), true = 1 == length(KeyList2), %% Delete the objects from the ChkList removing the indexes - lists:foreach(fun({_RN, Obj, Spc}) -> - DSpc = lists:map(fun({add, F, T}) -> - {remove, F, T} - end, - Spc), - {B, K} = - {testutil:get_bucket(Obj), testutil:get_key(Obj)}, - testutil:book_riakdelete(Bookie1, B, K, DSpc) - end, - ChkList1), + lists:foreach( + fun({_RN, Obj, Spc}) -> + DSpc = + lists:map(fun({add, F, T}) -> {remove, F, T} end, Spc), + {B, K} = {testutil:get_bucket(Obj), testutil:get_key(Obj)}, + testutil:book_riakdelete(Bookie1, B, K, DSpc) + end, + ChkList1 + ), %% Get the Buckets Keys and Hashes for the whole bucket - FoldObjectsFun = fun(B, K, V, Acc) -> [{B, K, erlang:phash2(V)}|Acc] - end, + FoldObjectsFun = + fun(B, K, V, Acc) -> [{B, K, erlang:phash2(V)}|Acc] end, - {async, HTreeF1} = leveled_bookie:book_objectfold(Bookie1, - ?RIAK_TAG, - {FoldObjectsFun, []}, - false), + {async, HTreeF1} = + leveled_bookie:book_objectfold( + Bookie1, ?RIAK_TAG, {FoldObjectsFun, []}, false), KeyHashList1 = HTreeF1(), - {async, HTreeF2} = leveled_bookie:book_objectfold(Bookie1, - ?RIAK_TAG, - "Bucket", - all, - {FoldObjectsFun, []}, - false), + {async, HTreeF2} = + leveled_bookie:book_objectfold( + Bookie1, ?RIAK_TAG, <<"Bucket">>, all, {FoldObjectsFun, []}, false + ), KeyHashList2 = HTreeF2(), {async, HTreeF3} = leveled_bookie:book_objectfold( Bookie1, ?RIAK_TAG, - "Bucket", + <<"Bucket">>, {<<"idx1_bin">>, <<"#">>, <<"|">>}, {FoldObjectsFun, []}, false), @@ -546,12 +550,13 @@ small_load_with2i(_Config) -> true = 9900 == length(KeyHashList2), true = 9900 == length(KeyHashList3), - SumIntFun = fun(_B, _K, Obj, Acc) -> - {I, _Bin} = testutil:get_value(Obj), - Acc + I - end, + SumIntFun = + fun(_B, _K, Obj, Acc) -> + {I, _Bin} = testutil:get_value(Obj), + Acc + I + end, BucketObjQ = - {foldobjects_bybucket, ?RIAK_TAG, "Bucket", all, {SumIntFun, 0}, true}, + {foldobjects_bybucket, ?RIAK_TAG, <<"Bucket">>, all, {SumIntFun, 0}, true}, {async, Sum1} = leveled_bookie:book_returnfolder(Bookie1, BucketObjQ), Total1 = Sum1(), io:format("Total from summing all I is ~w~n", [Total1]), @@ -596,21 +601,18 @@ query_count(_Config) -> BucketBin = list_to_binary("Bucket"), {TestObject, TestSpec} = testutil:generate_testobject( - BucketBin, term_to_binary("Key1"), "Value1", [], [{"MDK1", "MDV1"}]), + BucketBin, term_to_binary("Key1"), <<"Value1">>, [], [{<<"MDK1">>, <<"MDV1">>}]), ok = testutil:book_riakput(Book1, TestObject, TestSpec), testutil:check_forobject(Book1, TestObject), - testutil:check_formissingobject(Book1, "Bucket1", "Key2"), + testutil:check_formissingobject(Book1, <<"Bucket1">>, <<"Key2">>), testutil:check_forobject(Book1, TestObject), lists:foreach( fun(_X) -> V = testutil:get_compressiblevalue(), Indexes = testutil:get_randomindexes_generator(8), SW = os:timestamp(), - ObjL1 = testutil:generate_objects(10000, - binary_uuid, - [], - V, - Indexes), + ObjL1 = + testutil:generate_objects(10000, binary_uuid, [], V, Indexes), testutil:riakload(Book1, ObjL1), io:format( "Put of 10000 objects with 8 index entries " @@ -681,15 +683,17 @@ query_count(_Config) -> {true, undefined}}, {async, Mia2KFolder2} = leveled_bookie:book_returnfolder(Book2, Query2), - Mia2000Count2 = lists:foldl(fun({Term, _Key}, Acc) -> - case re:run(Term, RegMia) of - nomatch -> - Acc; - _ -> - Acc + 1 - end end, - 0, - Mia2KFolder2()), + Mia2000Count2 = + lists:foldl( + fun({Term, _Key}, Acc) -> + case re:run(Term, RegMia) of + nomatch -> + Acc; + _ -> + Acc + 1 + end end, + 0, + Mia2KFolder2()), ok = case Mia2000Count2 of Mia2000Count1 when Mia2000Count1 > 0 -> io:format("Mia2000 counts match at ~w~n", @@ -731,20 +735,22 @@ query_count(_Config) -> Spc9Del = lists:map(fun({add, IdxF, IdxT}) -> {remove, IdxF, IdxT} end, Spc9), ok = testutil:book_riakput(Book2, Obj9, Spc9Del), - lists:foreach(fun({IdxF, IdxT, X}) -> - Q = {index_query, - BucketBin, - {fun testutil:foldkeysfun/3, []}, - {IdxF, IdxT, IdxT}, - ?KEY_ONLY}, - R = leveled_bookie:book_returnfolder(Book2, Q), - {async, Fldr} = R, - case length(Fldr()) of - Y -> - Y = X - 1 - end - end, - R9), + lists:foreach( + fun({IdxF, IdxT, X}) -> + Q = {index_query, + BucketBin, + {fun testutil:foldkeysfun/3, []}, + {IdxF, IdxT, IdxT}, + ?KEY_ONLY}, + R = leveled_bookie:book_returnfolder(Book2, Q), + {async, Fldr} = R, + case length(Fldr()) of + Y -> + Y = X - 1 + end + end, + R9 + ), ok = leveled_bookie:book_close(Book2), {ok, Book3} = leveled_bookie:book_start( @@ -800,13 +806,13 @@ query_count(_Config) -> ObjList10A = testutil:generate_objects( - 5000, binary_uuid, [], V9, Indexes9, "BucketA"), + 5000, binary_uuid, [], V9, Indexes9, <<"BucketA">>), ObjList10B = testutil:generate_objects( - 5000, binary_uuid, [], V9, Indexes9, "BucketB"), + 5000, binary_uuid, [], V9, Indexes9, <<"BucketB">>), ObjList10C = testutil:generate_objects( - 5000, binary_uuid, [], V9, Indexes9, "BucketC"), + 5000, binary_uuid, [], V9, Indexes9, <<"BucketC">>), testutil:riakload(Book4, ObjList10A), testutil:riakload(Book4, ObjList10B), testutil:riakload(Book4, ObjList10C), @@ -819,10 +825,9 @@ query_count(_Config) -> ok = leveled_bookie:book_close(Book4), - {ok, Book5} = leveled_bookie:book_start(RootPath, - 2000, - 50000000, - testutil:sync_strategy()), + {ok, Book5} = + leveled_bookie:book_start( + RootPath, 2000, 50000000, testutil:sync_strategy()), {async, BLF3} = leveled_bookie:book_returnfolder(Book5, BucketListQuery), SW_QC = os:timestamp(), BucketSet3 = BLF3(), @@ -866,33 +871,25 @@ multibucket_fold(_Config) -> testutil:sync_strategy()), ObjectGen = testutil:get_compressiblevalue_andinteger(), IndexGen = fun() -> [] end, - ObjL1 = testutil:generate_objects(13000, - uuid, - [], - ObjectGen, - IndexGen, - {<<"Type1">>, <<"Bucket1">>}), + ObjL1 = + testutil:generate_objects( + 13000, uuid, [], ObjectGen, IndexGen, {<<"Type1">>, <<"Bucket1">>} + ), testutil:riakload(Bookie1, ObjL1), - ObjL2 = testutil:generate_objects(17000, - uuid, - [], - ObjectGen, - IndexGen, - <<"Bucket2">>), + ObjL2 = + testutil:generate_objects( + 17000, uuid, [], ObjectGen, IndexGen, <<"Bucket2">> + ), testutil:riakload(Bookie1, ObjL2), - ObjL3 = testutil:generate_objects(7000, - uuid, - [], - ObjectGen, - IndexGen, - <<"Bucket3">>), + ObjL3 = + testutil:generate_objects( + 7000, uuid, [], ObjectGen, IndexGen, <<"Bucket3">> + ), testutil:riakload(Bookie1, ObjL3), - ObjL4 = testutil:generate_objects(23000, - uuid, - [], - ObjectGen, - IndexGen, - {<<"Type2">>, <<"Bucket4">>}), + ObjL4 = + testutil:generate_objects( + 23000, uuid, [], ObjectGen, IndexGen, {<<"Type2">>, <<"Bucket4">>} + ), testutil:riakload(Bookie1, ObjL4), FF = fun(B, K, _PO, Acc) -> @@ -901,30 +898,30 @@ multibucket_fold(_Config) -> FoldAccT = {FF, []}, {async, R1} = - leveled_bookie:book_headfold(Bookie1, - ?RIAK_TAG, - {bucket_list, - [{<<"Type1">>, <<"Bucket1">>}, - {<<"Type2">>, <<"Bucket4">>}]}, - FoldAccT, - false, - true, - false), + leveled_bookie:book_headfold( + Bookie1, + ?RIAK_TAG, + {bucket_list, + [{<<"Type1">>, <<"Bucket1">>}, {<<"Type2">>, <<"Bucket4">>}]}, + FoldAccT, + false, + true, + false + ), O1 = length(R1()), io:format("Result R1 of length ~w~n", [O1]), {async, R2} = - leveled_bookie:book_headfold(Bookie1, - ?RIAK_TAG, - {bucket_list, - [<<"Bucket2">>, - <<"Bucket3">>]}, - {fun(_B, _K, _PO, Acc) -> - Acc +1 - end, - 0}, - false, true, false), + leveled_bookie:book_headfold( + Bookie1, + ?RIAK_TAG, + {bucket_list, [<<"Bucket2">>, <<"Bucket3">>]}, + {fun(_B, _K, _PO, Acc) -> Acc +1 end, 0}, + false, + true, + false + ), O2 = R2(), io:format("Result R2 of ~w~n", [O2]), @@ -933,10 +930,8 @@ multibucket_fold(_Config) -> FoldBucketsFun = fun(B, Acc) -> [B|Acc] end, {async, Folder} = - leveled_bookie:book_bucketlist(Bookie1, - ?RIAK_TAG, - {FoldBucketsFun, []}, - all), + leveled_bookie:book_bucketlist( + Bookie1, ?RIAK_TAG, {FoldBucketsFun, []}, all), BucketList = lists:reverse(Folder()), ExpectedBucketList = [{<<"Type1">>, <<"Bucket1">>}, {<<"Type2">>, <<"Bucket4">>}, @@ -949,54 +944,53 @@ multibucket_fold(_Config) -> rotating_objects(_Config) -> RootPath = testutil:reset_filestructure(), - ok = testutil:rotating_object_check(RootPath, "Bucket1", 10), - ok = testutil:rotating_object_check(RootPath, "Bucket2", 200), - ok = testutil:rotating_object_check(RootPath, "Bucket3", 800), - ok = testutil:rotating_object_check(RootPath, "Bucket4", 1600), - ok = testutil:rotating_object_check(RootPath, "Bucket5", 3200), - ok = testutil:rotating_object_check(RootPath, "Bucket6", 9600), + ok = testutil:rotating_object_check(RootPath, <<"Bucket1">>, 10), + ok = testutil:rotating_object_check(RootPath, <<"Bucket2">>, 200), + ok = testutil:rotating_object_check(RootPath, <<"Bucket3">>, 800), + ok = testutil:rotating_object_check(RootPath, <<"Bucket4">>, 1600), + ok = testutil:rotating_object_check(RootPath, <<"Bucket5">>, 3200), + ok = testutil:rotating_object_check(RootPath, <<"Bucket6">>, 9600), testutil:reset_filestructure(). foldobjects_bybucket_range(_Config) -> RootPath = testutil:reset_filestructure(), - {ok, Bookie1} = leveled_bookie:book_start(RootPath, - 2000, - 50000000, - testutil:sync_strategy()), + {ok, Bookie1} = + leveled_bookie:book_start( + RootPath, 2000, 50000000, testutil:sync_strategy()), ObjectGen = testutil:get_compressiblevalue_andinteger(), IndexGen = fun() -> [] end, - ObjL1 = testutil:generate_objects(1300, - {fixed_binary, 1}, - [], - ObjectGen, - IndexGen, - <<"Bucket1">>), + ObjL1 = + testutil:generate_objects( + 1300, {fixed_binary, 1}, [], ObjectGen, IndexGen, <<"Bucket1">>), testutil:riakload(Bookie1, ObjL1), - FoldKeysFun = fun(_B, K,_V, Acc) -> - [ K |Acc] - end, + FoldKeysFun = fun(_B, K,_V, Acc) -> [ K |Acc] end, StartKey = testutil:fixed_bin_key(123), EndKey = testutil:fixed_bin_key(779), - {async, Folder} = leveled_bookie:book_objectfold(Bookie1, - ?RIAK_TAG, - <<"Bucket1">>, - {StartKey, EndKey}, {FoldKeysFun, []}, - true - ), + {async, Folder} = + leveled_bookie:book_objectfold( + Bookie1, + ?RIAK_TAG, + <<"Bucket1">>, + {StartKey, EndKey}, + {FoldKeysFun, []}, + true + ), ResLen = length(Folder()), io:format("Length of Result of folder ~w~n", [ResLen]), true = 657 == ResLen, - {async, AllFolder} = leveled_bookie:book_objectfold(Bookie1, - ?RIAK_TAG, - <<"Bucket1">>, - all, - {FoldKeysFun, []}, - true - ), + {async, AllFolder} = + leveled_bookie:book_objectfold( + Bookie1, + ?RIAK_TAG, + <<"Bucket1">>, + all, + {FoldKeysFun, []}, + true + ), AllResLen = length(AllFolder()), io:format("Length of Result of all keys folder ~w~n", [AllResLen]), diff --git a/test/end_to_end/perf_SUITE.erl b/test/end_to_end/perf_SUITE.erl index c4d40f3..59d746f 100644 --- a/test/end_to_end/perf_SUITE.erl +++ b/test/end_to_end/perf_SUITE.erl @@ -101,7 +101,7 @@ riak_load_tester(Bucket, KeyCount, ObjSize, ProfileList, PM, LC) -> IndexGenFun = fun(ListID) -> fun() -> - RandInt = leveled_rand:uniform(IndexCount - 1), + RandInt = rand:uniform(IndexCount - 1), IntIndex = ["integer", integer_to_list(ListID), "_int"], BinIndex = ["binary", integer_to_list(ListID), "_bin"], [{add, iolist_to_binary(IntIndex), RandInt}, @@ -434,35 +434,35 @@ rotation_withnocheck(Book, B, NumberOfObjects, ObjSize, IdxCnt) -> Book, B, NumberOfObjects, - base64:encode(leveled_rand:rand_bytes(ObjSize)), + base64:encode(crypto:strong_rand_bytes(ObjSize)), IdxCnt ), rotation_with_prefetch( Book, B, NumberOfObjects, - base64:encode(leveled_rand:rand_bytes(ObjSize)), + base64:encode(crypto:strong_rand_bytes(ObjSize)), IdxCnt ), rotation_with_prefetch( Book, B, NumberOfObjects, - base64:encode(leveled_rand:rand_bytes(ObjSize)), + base64:encode(crypto:strong_rand_bytes(ObjSize)), IdxCnt ), rotation_with_prefetch( Book, B, NumberOfObjects, - base64:encode(leveled_rand:rand_bytes(ObjSize)), + base64:encode(crypto:strong_rand_bytes(ObjSize)), IdxCnt ), rotation_with_prefetch( Book, B, NumberOfObjects, - base64:encode(leveled_rand:rand_bytes(ObjSize)), + base64:encode(crypto:strong_rand_bytes(ObjSize)), IdxCnt ), ok. @@ -471,7 +471,7 @@ generate_chunk(CountPerList, ObjSize, IndexGenFun, Bucket, Chunk) -> testutil:generate_objects( CountPerList, {fixed_binary, (Chunk - 1) * CountPerList + 1}, [], - base64:encode(leveled_rand:rand_bytes(ObjSize)), + base64:encode(crypto:strong_rand_bytes(ObjSize)), IndexGenFun(Chunk), Bucket ). @@ -480,7 +480,7 @@ load_chunk(Bookie, CountPerList, ObjSize, IndexGenFun, Bucket, Chunk) -> ct:log(?INFO, "Generating and loading ObjList ~w", [Chunk]), time_load_chunk( Bookie, - {Bucket, base64:encode(leveled_rand:rand_bytes(ObjSize)), IndexGenFun(Chunk)}, + {Bucket, base64:encode(crypto:strong_rand_bytes(ObjSize)), IndexGenFun(Chunk)}, (Chunk - 1) * CountPerList + 1, Chunk * CountPerList, 0, @@ -577,9 +577,9 @@ random_fetches(FetchType, Bookie, Bucket, ObjCount, Fetches) -> case I rem 5 of 1 -> testutil:fixed_bin_key( - Twenty + leveled_rand:uniform(ObjCount - Twenty)); + Twenty + rand:uniform(ObjCount - Twenty)); _ -> - testutil:fixed_bin_key(leveled_rand:uniform(Twenty)) + testutil:fixed_bin_key(rand:uniform(Twenty)) end end, {TC, ok} = @@ -616,18 +616,18 @@ random_fetches(FetchType, Bookie, Bucket, ObjCount, Fetches) -> random_queries(Bookie, Bucket, IDs, IdxCnt, MaxRange, IndexesReturned) -> QueryFun = fun() -> - ID = leveled_rand:uniform(IDs), + ID = rand:uniform(IDs), BinIndex = iolist_to_binary(["binary", integer_to_list(ID), "_bin"]), Twenty = IdxCnt div 5, - RI = leveled_rand:uniform(MaxRange), + RI = rand:uniform(MaxRange), [Start, End] = case RI of RI when RI < (MaxRange div 5) -> - R0 = leveled_rand:uniform(IdxCnt - (Twenty + RI)), + R0 = rand:uniform(IdxCnt - (Twenty + RI)), [R0 + Twenty, R0 + Twenty + RI]; _ -> - R0 = leveled_rand:uniform(Twenty - RI), + R0 = rand:uniform(Twenty - RI), [R0, R0 + RI] end, FoldKeysFun = fun(_B, _K, Cnt) -> Cnt + 1 end, diff --git a/test/end_to_end/riak_SUITE.erl b/test/end_to_end/riak_SUITE.erl index b2c84d1..e5288c7 100644 --- a/test/end_to_end/riak_SUITE.erl +++ b/test/end_to_end/riak_SUITE.erl @@ -58,7 +58,7 @@ basic_riak_tester(Bucket, KeyCount) -> IndexGenFun = fun(ListID) -> fun() -> - RandInt = leveled_rand:uniform(IndexCount), + RandInt = rand:uniform(IndexCount), ID = integer_to_list(ListID), [{add, list_to_binary("integer" ++ ID ++ "_int"), @@ -75,7 +75,7 @@ basic_riak_tester(Bucket, KeyCount) -> testutil:generate_objects( CountPerList, {fixed_binary, 1}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), IndexGenFun(1), Bucket ), @@ -83,7 +83,7 @@ basic_riak_tester(Bucket, KeyCount) -> testutil:generate_objects( CountPerList, {fixed_binary, CountPerList + 1}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), IndexGenFun(2), Bucket ), @@ -92,7 +92,7 @@ basic_riak_tester(Bucket, KeyCount) -> testutil:generate_objects( CountPerList, {fixed_binary, 2 * CountPerList + 1}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), IndexGenFun(3), Bucket ), @@ -101,7 +101,7 @@ basic_riak_tester(Bucket, KeyCount) -> testutil:generate_objects( CountPerList, {fixed_binary, 3 * CountPerList + 1}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), IndexGenFun(4), Bucket ), @@ -110,7 +110,7 @@ basic_riak_tester(Bucket, KeyCount) -> testutil:generate_objects( CountPerList, {fixed_binary, 4 * CountPerList + 1}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), IndexGenFun(5), Bucket ), @@ -276,7 +276,7 @@ summarisable_sstindex(_Config) -> ObjListToSort = lists:map( fun(I) -> - {leveled_rand:uniform(KeyCount * 10), + {rand:uniform(KeyCount * 10), testutil:set_object( Bucket, KeyGen(I), integer_to_binary(I), IndexGen, [])} end, @@ -344,7 +344,7 @@ summarisable_sstindex(_Config) -> true = 200 == length(KeyRangeCheckFun(StartKey, EndKey)) end, lists:map( - fun(_I) -> leveled_rand:uniform(KeyCount - 200) end, + fun(_I) -> rand:uniform(KeyCount - 200) end, lists:seq(1, 100))), IdxObjKeyCount = 50000, @@ -367,7 +367,7 @@ summarisable_sstindex(_Config) -> IdxObjListToSort = lists:map( fun(I) -> - {leveled_rand:uniform(KeyCount * 10), + {rand:uniform(KeyCount * 10), testutil:set_object( Bucket, KeyGen(I), @@ -419,7 +419,7 @@ summarisable_sstindex(_Config) -> end, lists:map( fun(_I) -> - leveled_rand:uniform(IdxObjKeyCount - 20) + rand:uniform(IdxObjKeyCount - 20) end, lists:seq(1, 100))), lists:foreach( @@ -430,7 +430,7 @@ summarisable_sstindex(_Config) -> end, lists:map( fun(_I) -> - leveled_rand:uniform(IdxObjKeyCount - 10) + rand:uniform(IdxObjKeyCount - 10) end, lists:seq(1, 100))), @@ -451,7 +451,7 @@ summarisable_sstindex(_Config) -> true = 200 == length(KeyRangeCheckFun(StartKey, EndKey)) end, lists:map( - fun(_I) -> leveled_rand:uniform(KeyCount - 200) end, + fun(_I) -> rand:uniform(KeyCount - 200) end, lists:seq(1, 100))), ok = leveled_bookie:book_destroy(Bookie1). @@ -475,7 +475,7 @@ fetchclocks_modifiedbetween(_Config) -> testutil:generate_objects( 100000, {fixed_binary, 1}, [], - leveled_rand:rand_bytes(32), + crypto:strong_rand_bytes(32), fun() -> [] end, <<"BaselineB">> ), @@ -485,7 +485,7 @@ fetchclocks_modifiedbetween(_Config) -> testutil:generate_objects( 20000, {fixed_binary, 1}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), fun() -> [] end, <<"B0">> ), @@ -498,7 +498,7 @@ fetchclocks_modifiedbetween(_Config) -> testutil:generate_objects( 15000, {fixed_binary, 20001}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), fun() -> [] end, <<"B0">> ), @@ -511,7 +511,7 @@ fetchclocks_modifiedbetween(_Config) -> testutil:generate_objects( 35000, {fixed_binary, 35001}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), fun() -> [] end, <<"B0">> ), @@ -524,7 +524,7 @@ fetchclocks_modifiedbetween(_Config) -> testutil:generate_objects( 30000, {fixed_binary, 70001}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), fun() -> [] end, <<"B0">> ), @@ -537,7 +537,7 @@ fetchclocks_modifiedbetween(_Config) -> testutil:generate_objects( 8000, {fixed_binary, 1}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), fun() -> [] end, <<"B1">> ), @@ -550,7 +550,7 @@ fetchclocks_modifiedbetween(_Config) -> testutil:generate_objects( 7000, {fixed_binary, 1}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), fun() -> [] end, <<"B2">> ), @@ -815,7 +815,7 @@ fetchclocks_modifiedbetween(_Config) -> testutil:generate_objects( 200000, {fixed_binary, 1}, [], - leveled_rand:rand_bytes(32), + crypto:strong_rand_bytes(32), fun() -> [] end, <<"B1.9">> ), @@ -1637,7 +1637,7 @@ bigobject_memorycheck(_Config) -> ObjPutFun = fun(I) -> Key = base64:encode(<>), - Value = leveled_rand:rand_bytes(1024 * 1024), + Value = crypto:strong_rand_bytes(1024 * 1024), % a big object each time! {Obj, Spc} = testutil:set_object(Bucket, Key, Value, IndexGen, []), testutil:book_riakput(Bookie, Obj, Spc) diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index 82c0235..5816d11 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -231,12 +231,14 @@ sync_strategy() -> none. book_riakput(Pid, RiakObject, IndexSpecs) -> - leveled_bookie:book_put(Pid, - RiakObject#r_object.bucket, - RiakObject#r_object.key, - to_binary(v1, RiakObject), - IndexSpecs, - ?RIAK_TAG). + leveled_bookie:book_put( + Pid, + RiakObject#r_object.bucket, + RiakObject#r_object.key, + to_binary(v1, RiakObject), + IndexSpecs, + ?RIAK_TAG + ). book_tempriakput(Pid, RiakObject, IndexSpecs, TTL) -> leveled_bookie:book_tempput( @@ -246,7 +248,8 @@ book_tempriakput(Pid, RiakObject, IndexSpecs, TTL) -> to_binary(v1, RiakObject), IndexSpecs, ?RIAK_TAG, - TTL). + TTL + ). book_riakdelete(Pid, Bucket, Key, IndexSpecs) -> leveled_bookie:book_put(Pid, Bucket, Key, delete, IndexSpecs, ?RIAK_TAG). @@ -383,9 +386,8 @@ wait_for_compaction(Bookie) -> check_bucket_stats(Bookie, Bucket) -> FoldSW1 = os:timestamp(), io:format("Checking bucket size~n"), - {async, Folder1} = leveled_bookie:book_returnfolder(Bookie, - {riakbucket_stats, - Bucket}), + {async, Folder1} = + leveled_bookie:book_returnfolder(Bookie, {riakbucket_stats, Bucket}), {B1Size, B1Count} = Folder1(), io:format("Bucket fold completed in ~w microseconds~n", [timer:now_diff(os:timestamp(), FoldSW1)]), @@ -399,28 +401,32 @@ check_forlist(Bookie, ChkList) -> check_forlist(Bookie, ChkList, Log) -> SW = os:timestamp(), - lists:foreach(fun({_RN, Obj, _Spc}) -> - if - Log == true -> - io:format("Fetching Key ~s~n", [Obj#r_object.key]); - true -> - ok - end, - R = book_riakget(Bookie, - Obj#r_object.bucket, - Obj#r_object.key), - true = case R of - {ok, Val} -> - to_binary(v1, Obj) == Val; - not_found -> - io:format("Object not found for key ~s~n", - [Obj#r_object.key]), - error - end - end, - ChkList), - io:format("Fetch check took ~w microseconds checking list of length ~w~n", - [timer:now_diff(os:timestamp(), SW), length(ChkList)]). + lists:foreach( + fun({_RN, Obj, _Spc}) -> + if + Log == true -> + io:format("Fetching Key ~s~n", [Obj#r_object.key]); + true -> + ok + end, + R = book_riakget(Bookie, + Obj#r_object.bucket, + Obj#r_object.key), + true = + case R of + {ok, Val} -> + to_binary(v1, Obj) == Val; + not_found -> + io:format("Object not found for key ~s~n", + [Obj#r_object.key]), + error + end + end, + ChkList), + io:format( + "Fetch check took ~w microseconds checking list of length ~w~n", + [timer:now_diff(os:timestamp(), SW), length(ChkList)] + ). checkhead_forlist(Bookie, ChkList) -> SW = os:timestamp(), @@ -470,11 +476,14 @@ check_formissingobject(Bookie, Bucket, Key) -> generate_testobject() -> - {B1, K1, V1, Spec1, MD} = {"Bucket1", - "Key1", - "Value1", - [], - [{"MDK1", "MDV1"}]}, + {B1, K1, V1, Spec1, MD} = + { + <<"Bucket1">>, + <<"Key1">>, + <<"Value1">>, + [], + [{<<"MDK1">>, <<"MDV1">>}] + }, generate_testobject(B1, K1, V1, Spec1, MD). generate_testobject(B, K, V, Spec, MD) -> @@ -493,7 +502,7 @@ generate_compressibleobjects(Count, KeyNumber) -> get_compressiblevalue_andinteger() -> - {leveled_rand:uniform(1000), get_compressiblevalue()}. + {rand:uniform(1000), get_compressiblevalue()}. get_compressiblevalue() -> S1 = "111111111111111", @@ -510,7 +519,7 @@ get_compressiblevalue() -> iolist_to_binary( lists:foldl( fun(_X, Acc) -> - {_, Str} = lists:keyfind(leveled_rand:uniform(8), 1, Selector), + {_, Str} = lists:keyfind(rand:uniform(8), 1, Selector), [Str|Acc] end, [""], L @@ -518,28 +527,39 @@ get_compressiblevalue() -> ). generate_smallobjects(Count, KeyNumber) -> - generate_objects(Count, KeyNumber, [], leveled_rand:rand_bytes(512)). + generate_objects(Count, KeyNumber, [], crypto:strong_rand_bytes(512)). generate_objects(Count, KeyNumber) -> - generate_objects(Count, KeyNumber, [], leveled_rand:rand_bytes(4096)). + generate_objects(Count, KeyNumber, [], crypto:strong_rand_bytes(4096)). generate_objects(Count, KeyNumber, ObjL, Value) -> generate_objects(Count, KeyNumber, ObjL, Value, fun() -> [] end). generate_objects(Count, KeyNumber, ObjL, Value, IndexGen) -> - generate_objects(Count, KeyNumber, ObjL, Value, IndexGen, "Bucket"). + generate_objects(Count, KeyNumber, ObjL, Value, IndexGen, <<"Bucket">>). generate_objects(0, _KeyNumber, ObjL, _Value, _IndexGen, _Bucket) -> lists:reverse(ObjL); -generate_objects(Count, binary_uuid, ObjL, Value, IndexGen, Bucket) -> - {Obj1, Spec1} = set_object(list_to_binary(Bucket), - list_to_binary(leveled_util:generate_uuid()), - Value, - IndexGen), +generate_objects( + Count, binary_uuid, ObjL, Value, IndexGen, Bucket) + when is_list(Bucket) -> + generate_objects( + Count, binary_uuid, ObjL, Value, IndexGen, list_to_binary(Bucket) + ); +generate_objects( + Count, binary_uuid, ObjL, Value, IndexGen, Bucket) + when is_binary(Bucket) -> + {Obj1, Spec1} = + set_object( + Bucket, + list_to_binary(leveled_util:generate_uuid()), + Value, + IndexGen + ), generate_objects(Count - 1, binary_uuid, - [{leveled_rand:uniform(), Obj1, Spec1}|ObjL], + [{rand:uniform(), Obj1, Spec1}|ObjL], Value, IndexGen, Bucket); @@ -550,19 +570,29 @@ generate_objects(Count, uuid, ObjL, Value, IndexGen, Bucket) -> IndexGen), generate_objects(Count - 1, uuid, - [{leveled_rand:uniform(), Obj1, Spec1}|ObjL], + [{rand:uniform(), Obj1, Spec1}|ObjL], Value, IndexGen, Bucket); -generate_objects(Count, {binary, KeyNumber}, ObjL, Value, IndexGen, Bucket) -> +generate_objects( + Count, {binary, KeyNumber}, ObjL, Value, IndexGen, Bucket) + when is_list(Bucket) -> + generate_objects( + Count, {binary, KeyNumber}, ObjL, Value, IndexGen, list_to_binary(Bucket) + ); +generate_objects( + Count, {binary, KeyNumber}, ObjL, Value, IndexGen, Bucket) + when is_binary(Bucket) -> {Obj1, Spec1} = - set_object(list_to_binary(Bucket), - list_to_binary(numbered_key(KeyNumber)), - Value, - IndexGen), + set_object( + Bucket, + list_to_binary(numbered_key(KeyNumber)), + Value, + IndexGen + ), generate_objects(Count - 1, {binary, KeyNumber + 1}, - [{leveled_rand:uniform(), Obj1, Spec1}|ObjL], + [{rand:uniform(), Obj1, Spec1}|ObjL], Value, IndexGen, Bucket); @@ -574,7 +604,7 @@ generate_objects(Count, {fixed_binary, KeyNumber}, ObjL, Value, IndexGen, Bucket IndexGen), generate_objects(Count - 1, {fixed_binary, KeyNumber + 1}, - [{leveled_rand:uniform(), Obj1, Spec1}|ObjL], + [{rand:uniform(), Obj1, Spec1}|ObjL], Value, IndexGen, Bucket); @@ -585,7 +615,7 @@ generate_objects(Count, KeyNumber, ObjL, Value, IndexGen, Bucket) -> IndexGen), generate_objects(Count - 1, KeyNumber + 1, - [{leveled_rand:uniform(), Obj1, Spec1}|ObjL], + [{rand:uniform(), Obj1, Spec1}|ObjL], Value, IndexGen, Bucket). @@ -652,7 +682,7 @@ update_some_objects(Bookie, ObjList, SampleSize) -> [C] = Obj#r_object.contents, MD = C#r_content.metadata, MD0 = dict:store(?MD_LASTMOD, os:timestamp(), MD), - C0 = C#r_content{value = leveled_rand:rand_bytes(512), + C0 = C#r_content{value = crypto:strong_rand_bytes(512), metadata = MD0}, UpdObj = Obj#r_object{vclock = VC0, contents = [C0]}, {R, UpdObj, Spec} @@ -679,11 +709,11 @@ delete_some_objects(Bookie, ObjList, SampleSize) -> generate_vclock() -> lists:map(fun(X) -> - {_, Actor} = lists:keyfind(leveled_rand:uniform(10), + {_, Actor} = lists:keyfind(rand:uniform(10), 1, actor_list()), {Actor, X} end, - lists:seq(1, leveled_rand:uniform(8))). + lists:seq(1, rand:uniform(8))). update_vclock(VC) -> [{Actor, X}|Rest] = VC, @@ -785,14 +815,14 @@ name_list() -> get_randomname() -> NameList = name_list(), - N = leveled_rand:uniform(16), + N = rand:uniform(16), {N, Name} = lists:keyfind(N, 1, NameList), Name. get_randomdate() -> LowTime = 60000000000, HighTime = 70000000000, - RandPoint = LowTime + leveled_rand:uniform(HighTime - LowTime), + RandPoint = LowTime + rand:uniform(HighTime - LowTime), Date = calendar:gregorian_seconds_to_datetime(RandPoint), {{Year, Month, Day}, {Hour, Minute, Second}} = Date, lists:flatten(io_lib:format("~4..0w~2..0w~2..0w~2..0w~2..0w~2..0w", diff --git a/test/end_to_end/tictac_SUITE.erl b/test/end_to_end/tictac_SUITE.erl index 277a7a3..69b4f8d 100644 --- a/test/end_to_end/tictac_SUITE.erl +++ b/test/end_to_end/tictac_SUITE.erl @@ -41,11 +41,14 @@ many_put_compare(_Config) -> {max_pencillercachesize, 16000}, {sync_strategy, riak_sync}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), - {B1, K1, V1, S1, MD} = {"Bucket", - "Key1.1.4567.4321", - "Value1", - [], - [{"MDK1", "MDV1"}]}, + {B1, K1, V1, S1, MD} = + { + <<"Bucket">>, + <<"Key1.1.4567.4321">>, + <<"Value1">>, + [], + [{<<"MDK1">>, <<"MDV1">>}] + }, {TestObject, TestSpec} = testutil:generate_testobject(B1, K1, V1, S1, MD), ok = testutil:book_riakput(Bookie1, TestObject, TestSpec), testutil:check_forobject(Bookie1, TestObject), @@ -63,12 +66,15 @@ many_put_compare(_Config) -> GenList = [2, 20002, 40002, 60002, 80002, 100002, 120002, 140002, 160002, 180002], - CLs = testutil:load_objects(20000, - GenList, - Bookie2, - TestObject, - fun testutil:generate_smallobjects/2, - 20000), + CLs = + testutil:load_objects( + 20000, + GenList, + Bookie2, + TestObject, + fun testutil:generate_smallobjects/2, + 20000 + ), % Start a new store, and load the same objects (except fot the original % test object) into this store @@ -84,7 +90,7 @@ many_put_compare(_Config) -> % state between stores is consistent TicTacQ = {tictactree_obj, - {o_rkv, "Bucket", null, null, true}, + {o_rkv, <<"Bucket">>, null, null, true}, TreeSize, fun(_B, _K) -> accumulate end}, {async, TreeAFolder} = leveled_bookie:book_returnfolder(Bookie2, TicTacQ), @@ -113,10 +119,13 @@ many_put_compare(_Config) -> true = length(AltList) > 10000, % check there are a significant number of differences from empty - WrongPartitionTicTacQ = {tictactree_obj, - {o_rkv, "Bucket", null, null, false}, - TreeSize, - fun(_B, _K) -> pass end}, + WrongPartitionTicTacQ = + { + tictactree_obj, + {o_rkv, <<"Bucket">>, null, null, false}, + TreeSize, + fun(_B, _K) -> pass end + }, {async, TreeAFolder_WP} = leveled_bookie:book_returnfolder(Bookie2, WrongPartitionTicTacQ), TreeAWP = TreeAFolder_WP(), @@ -151,7 +160,7 @@ many_put_compare(_Config) -> {async, TreeAObjFolder0} = leveled_bookie:book_headfold(Bookie2, o_rkv, - {range, "Bucket", all}, + {range, <<"Bucket">>, all}, FoldAccT, false, true, @@ -170,7 +179,7 @@ many_put_compare(_Config) -> leveled_bookie:book_headfold( Bookie2, ?RIAK_TAG, - {range, "Bucket", all}, + {range, <<"Bucket">>, all}, {FoldObjectsFun, InitAccTree}, true, true, @@ -188,7 +197,7 @@ many_put_compare(_Config) -> leveled_bookie:book_headfold( Bookie2, ?RIAK_TAG, - {range, "Bucket", all}, + {range, <<"Bucket">>, all}, {FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize, false)}, true, true, @@ -218,29 +227,38 @@ many_put_compare(_Config) -> end, {async, TreeAAltObjFolder0} = - leveled_bookie:book_headfold(Bookie2, - ?RIAK_TAG, - {range, "Bucket", all}, - {AltFoldObjectsFun, - InitAccTree}, - false, true, false), + leveled_bookie:book_headfold( + Bookie2, + ?RIAK_TAG, + {range, <<"Bucket">>, all}, + {AltFoldObjectsFun, InitAccTree}, + false, + true, + false + ), SWB2Obj = os:timestamp(), TreeAAltObj = TreeAAltObjFolder0(), - io:format("Build tictac tree via object fold with no "++ - "presence check and 200K objects and alt hash in ~w~n", - [timer:now_diff(os:timestamp(), SWB2Obj)]), + io:format( + "Build tictac tree via object fold with no " + "presence check and 200K objects and alt hash in ~w~n", + [timer:now_diff(os:timestamp(), SWB2Obj)] + ), {async, TreeBAltObjFolder0} = - leveled_bookie:book_headfold(Bookie3, - ?RIAK_TAG, - {range, "Bucket", all}, - {AltFoldObjectsFun, - InitAccTree}, - false, true, false), + leveled_bookie:book_headfold( + Bookie3, + ?RIAK_TAG, + {range, <<"Bucket">>, all}, + {AltFoldObjectsFun, InitAccTree}, + false, + true, + false + ), SWB3Obj = os:timestamp(), TreeBAltObj = TreeBAltObjFolder0(), - io:format("Build tictac tree via object fold with no "++ - "presence check and 200K objects and alt hash in ~w~n", - [timer:now_diff(os:timestamp(), SWB3Obj)]), + io:format( + "Build tictac tree via object fold with no " + "presence check and 200K objects and alt hash in ~w~n", + [timer:now_diff(os:timestamp(), SWB3Obj)]), DL_ExportFold = length(leveled_tictac:find_dirtyleaves(TreeBAltObj, TreeAAltObj)), io:format("Found dirty leaves with exportable comparison of ~w~n", @@ -261,7 +279,7 @@ many_put_compare(_Config) -> end end end, - SegQuery = {keylist, o_rkv, "Bucket", {FoldKeysFun(SegList0), []}}, + SegQuery = {keylist, o_rkv, <<"Bucket">>, {FoldKeysFun(SegList0), []}}, {async, SegKeyFinder} = leveled_bookie:book_returnfolder(Bookie2, SegQuery), SWSKL0 = os:timestamp(), @@ -273,7 +291,7 @@ many_put_compare(_Config) -> true = length(SegKeyList) >= 1, true = length(SegKeyList) < 10, - true = lists:member("Key1.1.4567.4321", SegKeyList), + true = lists:member(<<"Key1.1.4567.4321">>, SegKeyList), % Now remove the object which represents the difference between these % stores and confirm that the tictac trees will now match @@ -630,20 +648,23 @@ tuplebuckets_headonly(_Config) -> SW1 = os:timestamp(), {async, HeadRunner1} = - leveled_bookie:book_headfold(Bookie1, - ?HEAD_TAG, - {bucket_list, BucketList}, - {FoldHeadFun, []}, - false, false, - false), + leveled_bookie:book_headfold( + Bookie1, + ?HEAD_TAG, + {bucket_list, BucketList}, + {FoldHeadFun, []}, + false, false, + false + ), ReturnedObjSpecL1 = lists:reverse(HeadRunner1()), [FirstItem|_Rest] = ReturnedObjSpecL1, LastItem = lists:last(ReturnedObjSpecL1), - io:format("Returned ~w objects with first ~w and last ~w in ~w ms~n", - [length(ReturnedObjSpecL1), - FirstItem, LastItem, - timer:now_diff(os:timestamp(), SW1)/1000]), + io:format( + "Returned ~w objects with first ~w and last ~w in ~w ms~n", + [length(ReturnedObjSpecL1), + FirstItem, LastItem, + timer:now_diff(os:timestamp(), SW1)/1000]), true = ReturnedObjSpecL1 == lists:sort(ObjectSpecL), @@ -654,12 +675,14 @@ tuplebuckets_headonly(_Config) -> SW2 = os:timestamp(), {async, HeadRunner2} = - leveled_bookie:book_headfold(Bookie1, - ?HEAD_TAG, - {bucket_list, BucketList}, - {FoldHeadFun, []}, - false, false, - SegList), + leveled_bookie:book_headfold( + Bookie1, + ?HEAD_TAG, + {bucket_list, BucketList}, + {FoldHeadFun, []}, + false, false, + SegList + ), ReturnedObjSpecL2 = lists:reverse(HeadRunner2()), io:format("Returned ~w objects using seglist in ~w ms~n", @@ -674,7 +697,6 @@ tuplebuckets_headonly(_Config) -> leveled_bookie:book_destroy(Bookie1). - basic_headonly(_Config) -> ObjectCount = 200000, RemoveCount = 100, @@ -694,11 +716,14 @@ basic_headonly_test(ObjectCount, RemoveCount, HeadOnly) -> {head_only, HeadOnly}, {max_journalsize, 500000}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), - {B1, K1, V1, S1, MD} = {"Bucket", - "Key1.1.4567.4321", - "Value1", - [], - [{"MDK1", "MDV1"}]}, + {B1, K1, V1, S1, MD} = + { + <<"Bucket">>, + <<"Key1.1.4567.4321">>, + <<"Value1">>, + [], + [{<<"MDK1">>, <<"MDV1">>}] + }, {TestObject, TestSpec} = testutil:generate_testobject(B1, K1, V1, S1, MD), {unsupported_message, put} = testutil:book_riakput(Bookie1, TestObject, TestSpec), @@ -818,23 +843,21 @@ basic_headonly_test(ObjectCount, RemoveCount, HeadOnly) -> false = is_process_alive(AltSnapshot); no_lookup -> {unsupported_message, head} = - leveled_bookie:book_head(Bookie1, - SegmentID0, - {Bucket0, Key0}, - h), + leveled_bookie:book_head( + Bookie1, SegmentID0, {Bucket0, Key0}, h), {unsupported_message, head} = - leveled_bookie:book_headonly(Bookie1, - SegmentID0, - Bucket0, - Key0), + leveled_bookie:book_headonly( + Bookie1, SegmentID0, Bucket0, Key0), io:format("Closing actual store ~w~n", [Bookie1]), ok = leveled_bookie:book_close(Bookie1) end, {ok, FinalJournals} = file:list_dir(JFP), - io:format("Trim has reduced journal count from " ++ - "~w to ~w and ~w after restart~n", - [length(FNs), length(FinalFNs), length(FinalJournals)]), + io:format( + "Trim has reduced journal count from " + "~w to ~w and ~w after restart~n", + [length(FNs), length(FinalFNs), length(FinalJournals)] + ), {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), @@ -849,16 +872,12 @@ basic_headonly_test(ObjectCount, RemoveCount, HeadOnly) -> % If we allow HEAD_TAG to be suubject to a lookup, then test this % here {ok, Hash0} = - leveled_bookie:book_head(Bookie2, - SegmentID0, - {Bucket0, Key0}, - h); + leveled_bookie:book_head( + Bookie2, SegmentID0, {Bucket0, Key0}, h); no_lookup -> {unsupported_message, head} = - leveled_bookie:book_head(Bookie2, - SegmentID0, - {Bucket0, Key0}, - h) + leveled_bookie:book_head( + Bookie2, SegmentID0, {Bucket0, Key0}, h) end, RemoveSpecL0 = lists:sublist(ObjectSpecL, RemoveCount), @@ -873,12 +892,9 @@ basic_headonly_test(ObjectCount, RemoveCount, HeadOnly) -> true = AccC3 == (ObjectCount - RemoveCount), false = AccH3 == AccH2, - ok = leveled_bookie:book_close(Bookie2). - - load_objectspecs([], _SliceSize, _Bookie) -> ok; load_objectspecs(ObjectSpecL, SliceSize, Bookie)