From 1a3d3daa89bda17f2d0056ec2a9c1a8c5ed4245e Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 21 Sep 2018 12:04:32 +0100 Subject: [PATCH 1/2] Add regex support to $key index Regex to be applied to key only --- src/leveled_bookie.erl | 33 +++++++++++++++++++++++---- src/leveled_runner.erl | 41 ++++++++++++++++++++++------------ test/end_to_end/riak_SUITE.erl | 33 ++++++++++++++++++++++----- 3 files changed, 84 insertions(+), 23 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 177f857..b0b8a96 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -76,6 +76,7 @@ book_keylist/3, book_keylist/4, book_keylist/5, + book_keylist/6, book_objectfold/4, book_objectfold/5, book_objectfold/6, @@ -677,7 +678,8 @@ book_keylist(Pid, Tag, Bucket, FoldAccT) -> %% is `StartKey', the first key in the range and `EndKey' the last, %% (inclusive.) Or the atom `all', which will return all keys in the %% `Bucket'. --spec book_keylist(pid(), Tag, Bucket, KeyRange, FoldAccT) -> {async, Runner} when +-spec book_keylist(pid(), Tag, Bucket, KeyRange, FoldAccT) -> + {async, Runner} when Tag :: leveled_codec:tag(), FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Acc) -> Acc), @@ -689,9 +691,30 @@ book_keylist(Pid, Tag, Bucket, FoldAccT) -> Key :: term(), Runner :: fun(() -> Acc). book_keylist(Pid, Tag, Bucket, KeyRange, FoldAccT) -> - RunnerType = {keylist, Tag, Bucket, KeyRange, FoldAccT}, + RunnerType = {keylist, Tag, Bucket, KeyRange, FoldAccT, undefined}, book_returnfolder(Pid, RunnerType). +%% @doc as for book_keylist/5 with additional constraint that a compile regular +%% expression is passed to be applied against any key that is in the range. +%% This is always applied to the Key and only the Key, not to any SubKey. +-spec book_keylist(pid(), Tag, Bucket, KeyRange, FoldAccT, TermRegex) -> + {async, Runner} when + Tag :: leveled_codec:tag(), + FoldAccT :: {FoldFun, Acc}, + FoldFun :: fun((Bucket, Key, Acc) -> Acc), + Acc :: term(), + Bucket :: term(), + KeyRange :: {StartKey, EndKey} | all, + StartKey :: Key, + EndKey :: Key, + Key :: term(), + TermRegex :: re:mp(), + Runner :: fun(() -> Acc). +book_keylist(Pid, Tag, Bucket, KeyRange, FoldAccT, TermRegex) -> + RunnerType = {keylist, Tag, Bucket, KeyRange, FoldAccT, TermRegex}, + book_returnfolder(Pid, RunnerType). + + %% @doc fold over all the objects/values in the store in key %% order. `Tag' is the tagged type of object. `FoldAccT' is a 2-tuple, %% the first element being a 4-arity fun, that is called once for each @@ -1490,9 +1513,11 @@ get_runner(State, {keylist, Tag, FoldAccT}) -> get_runner(State, {keylist, Tag, Bucket, FoldAccT}) -> SnapFun = return_snapfun(State, ledger, no_lookup, true, true), leveled_runner:bucketkey_query(SnapFun, Tag, Bucket, FoldAccT); -get_runner(State, {keylist, Tag, Bucket, KeyRange, FoldAccT}) -> +get_runner(State, {keylist, Tag, Bucket, KeyRange, FoldAccT, TermRegex}) -> SnapFun = return_snapfun(State, ledger, no_lookup, true, true), - leveled_runner:bucketkey_query(SnapFun, Tag, Bucket, KeyRange, FoldAccT); + leveled_runner:bucketkey_query(SnapFun, + Tag, Bucket, KeyRange, + FoldAccT, TermRegex); %% Set of runners for object or metadata folds get_runner(State, {foldheads_allkeys, diff --git a/src/leveled_runner.erl b/src/leveled_runner.erl index aa7885c..90e27d5 100644 --- a/src/leveled_runner.erl +++ b/src/leveled_runner.erl @@ -27,7 +27,7 @@ bucket_list/5, index_query/3, bucketkey_query/4, - bucketkey_query/5, + bucketkey_query/6, hashlist_query/3, tictactree/5, foldheads_allkeys/5, @@ -47,6 +47,7 @@ leveled_codec:ledger_key()|null}. -type fun_and_acc() :: {fun(), any()}. +-type term_regex() :: re:mp()|undefined. %%%============================================================================ %%% External functions @@ -132,15 +133,16 @@ index_query(SnapFun, {StartKey, EndKey, TermHandling}, FoldAccT) -> {async, Runner}. -spec bucketkey_query(fun(), leveled_codec:tag(), any(), - key_range(), fun_and_acc()) -> {async, fun()}. + key_range(), fun_and_acc(), term_regex()) -> {async, fun()}. %% @doc %% Fold over all keys in `KeyRange' under tag (restricted to a given bucket) bucketkey_query(SnapFun, Tag, Bucket, {StartKey, EndKey}, - {FoldKeysFun, InitAcc}) -> + {FoldKeysFun, InitAcc}, + TermRegex) -> SK = leveled_codec:to_ledgerkey(Bucket, StartKey, Tag), EK = leveled_codec:to_ledgerkey(Bucket, EndKey, Tag), - AccFun = accumulate_keys(FoldKeysFun), + AccFun = accumulate_keys(FoldKeysFun, TermRegex), Runner = fun() -> {ok, LedgerSnapshot, _JournalSnapshot} = SnapFun(), @@ -159,7 +161,7 @@ bucketkey_query(SnapFun, Tag, Bucket, %% @doc %% Fold over all keys under tag (potentially restricted to a given bucket) bucketkey_query(SnapFun, Tag, Bucket, FunAcc) -> - bucketkey_query(SnapFun, Tag, Bucket, {null, null}, FunAcc). + bucketkey_query(SnapFun, Tag, Bucket, {null, null}, FunAcc, undefined). -spec hashlist_query(fun(), leveled_codec:tag(), boolean()) -> {async, fun()}. %% @doc @@ -660,17 +662,28 @@ check_presence(Key, Value, InkerClone) -> false end. -accumulate_keys(FoldKeysFun) -> +accumulate_keys(FoldKeysFun, TermRegex) -> Now = leveled_util:integer_now(), - AccFun = fun(Key, Value, Acc) -> - case leveled_codec:is_active(Key, Value, Now) of - true -> - {B, K} = leveled_codec:from_ledgerkey(Key), + AccFun = + fun(Key, Value, Acc) -> + case leveled_codec:is_active(Key, Value, Now) of + true -> + {B, K} = leveled_codec:from_ledgerkey(Key), + case TermRegex of + undefined -> FoldKeysFun(B, K, Acc); - false -> - Acc - end - end, + Re -> + case re:run(K, Re) of + nomatch -> + Acc; + _ -> + FoldKeysFun(B, K, Acc) + end + end; + false -> + Acc + end + end, AccFun. add_keys(ObjKey, _IdxValue) -> diff --git a/test/end_to_end/riak_SUITE.erl b/test/end_to_end/riak_SUITE.erl index 8e688cd..71b0b76 100644 --- a/test/end_to_end/riak_SUITE.erl +++ b/test/end_to_end/riak_SUITE.erl @@ -525,15 +525,38 @@ dollar_key_index(_Config) -> EndKey = testutil:fixed_bin_key(779), - {async, Folder} = leveled_bookie:book_keylist(Bookie1, - ?RIAK_TAG, - <<"Bucket1">>, - {StartKey, EndKey}, {FoldKeysFun, []} - ), + {async, Folder} = + leveled_bookie:book_keylist(Bookie1, + ?RIAK_TAG, + <<"Bucket1">>, + {StartKey, EndKey}, + {FoldKeysFun, []} + ), ResLen = length(Folder()), io:format("Length of Result of folder ~w~n", [ResLen]), true = 657 == ResLen, + {ok, REMatch} = re:compile("K.y"), + {ok, REMiss} = re:compile("key"), + + {async, FolderREMatch} = + leveled_bookie:book_keylist(Bookie1, + ?RIAK_TAG, + <<"Bucket1">>, + {StartKey, EndKey}, + {FoldKeysFun, []}, + REMatch), + {async, FolderREMiss} = + leveled_bookie:book_keylist(Bookie1, + ?RIAK_TAG, + <<"Bucket1">>, + {StartKey, EndKey}, + {FoldKeysFun, []}, + REMiss), + + true = 657 == length(FolderREMatch()), + true = 0 == length(FolderREMiss()), + ok = leveled_bookie:book_close(Bookie1), testutil:reset_filestructure(). From ba93ea149f54dee1295bb1926d95b7a733af1662 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 21 Sep 2018 13:27:10 +0100 Subject: [PATCH 2/2] Account for slower tests in older Erlang versions Tests run slower in OTP16 - so do less work in this version. --- rebar.config | 3 ++- src/leveled_cdb.erl | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/rebar.config b/rebar.config index 04f914b..7a46b31 100644 --- a/rebar.config +++ b/rebar.config @@ -1,8 +1,9 @@ {erl_opts, [warnings_as_errors, {platform_define, "^2[0-1]{1}", fsm_deprecated}, {platform_define, "^1[7-8]{1}", old_rand}, + {platform_define, "^R", no_sync}, {platform_define, "^R", old_rand}, - {platform_define, "^R", no_sync}]}. + {platform_define, "^R", slow_test}]}. {xref_checks, [undefined_function_calls,undefined_functions]}. diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 4d60c6a..5c56433 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -61,6 +61,12 @@ {gen_fsm, send_all_state_event, 2}]}). -endif. +-ifdef(slow_test). +-define(KEYCOUNT, 2048). +-else. +-define(KEYCOUNT, 16384). +-endif. + -export([init/1, handle_sync_event/4, handle_event/3, @@ -2235,10 +2241,10 @@ generate_sequentialkeys(Count, KVList) -> generate_sequentialkeys(Count - 1, KVList ++ [KV]). get_keys_byposition_manykeys_test_() -> - {timeout, 120, fun get_keys_byposition_manykeys_test_to/0}. + {timeout, 600, fun get_keys_byposition_manykeys_test_to/0}. get_keys_byposition_manykeys_test_to() -> - KeyCount = 16384, + KeyCount = ?KEYCOUNT, {ok, P1} = cdb_open_writer("../test/poskeymany.pnd", #cdb_options{binary_mode=false}), KVList = generate_sequentialkeys(KeyCount, []),