Add FoldKeysFun
Add the capability to pass FoldKeysFun into the index_query to allow for compatability with riak backend requirements.
This commit is contained in:
martinsumner
parent
e3004faa6e
commit
ac223ced68
4 changed files with 83 additions and 87 deletions
|
|
@ -337,11 +337,13 @@ handle_call({return_folder, FolderType}, _From, State) ->
|
|||
State};
|
||||
{index_query,
|
||||
Bucket,
|
||||
{FoldKeysFun, Acc},
|
||||
{IdxField, StartValue, EndValue},
|
||||
{ReturnTerms, TermRegex}} ->
|
||||
{reply,
|
||||
index_query(State,
|
||||
Bucket,
|
||||
{FoldKeysFun, Acc},
|
||||
{IdxField, StartValue, EndValue},
|
||||
{ReturnTerms, TermRegex}),
|
||||
State};
|
||||
|
|
@ -430,6 +432,7 @@ bucket_stats(State, Bucket, Tag) ->
|
|||
|
||||
index_query(State,
|
||||
Bucket,
|
||||
{FoldKeysFun, InitAcc},
|
||||
{IdxField, StartValue, EndValue},
|
||||
{ReturnTerms, TermRegex}) ->
|
||||
{ok,
|
||||
|
|
@ -445,16 +448,16 @@ index_query(State,
|
|||
IdxField, EndValue),
|
||||
AddFun = case ReturnTerms of
|
||||
true ->
|
||||
fun add_terms/3;
|
||||
fun add_terms/2;
|
||||
_ ->
|
||||
fun add_keys/3
|
||||
fun add_keys/2
|
||||
end,
|
||||
AccFun = accumulate_index(TermRegex, AddFun),
|
||||
AccFun = accumulate_index(TermRegex, AddFun, FoldKeysFun),
|
||||
Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot,
|
||||
StartKey,
|
||||
EndKey,
|
||||
AccFun,
|
||||
[]),
|
||||
InitAcc),
|
||||
ok = leveled_penciller:pcl_close(LedgerSnapshot),
|
||||
Acc
|
||||
end,
|
||||
|
|
@ -724,23 +727,23 @@ accumulate_keys() ->
|
|||
end,
|
||||
AccFun.
|
||||
|
||||
add_keys(ObjKey, _IdxValue, Acc) ->
|
||||
Acc ++ [ObjKey].
|
||||
add_keys(ObjKey, _IdxValue) ->
|
||||
ObjKey.
|
||||
|
||||
add_terms(ObjKey, IdxValue, Acc) ->
|
||||
Acc ++ [{IdxValue, ObjKey}].
|
||||
add_terms(ObjKey, IdxValue) ->
|
||||
{IdxValue, ObjKey}.
|
||||
|
||||
accumulate_index(TermRe, AddFun) ->
|
||||
accumulate_index(TermRe, AddFun, FoldKeysFun) ->
|
||||
Now = leveled_codec:integer_now(),
|
||||
case TermRe of
|
||||
undefined ->
|
||||
fun(Key, Value, Acc) ->
|
||||
case leveled_codec:is_active(Key, Value, Now) of
|
||||
true ->
|
||||
{_Bucket,
|
||||
{Bucket,
|
||||
ObjKey,
|
||||
IdxValue} = leveled_codec:from_ledgerkey(Key),
|
||||
AddFun(ObjKey, IdxValue, Acc);
|
||||
FoldKeysFun(Bucket, AddFun(ObjKey, IdxValue), Acc);
|
||||
false ->
|
||||
Acc
|
||||
end end;
|
||||
|
|
@ -748,14 +751,16 @@ accumulate_index(TermRe, AddFun) ->
|
|||
fun(Key, Value, Acc) ->
|
||||
case leveled_codec:is_active(Key, Value, Now) of
|
||||
true ->
|
||||
{_Bucket,
|
||||
{Bucket,
|
||||
ObjKey,
|
||||
IdxValue} = leveled_codec:from_ledgerkey(Key),
|
||||
case re:run(IdxValue, TermRe) of
|
||||
nomatch ->
|
||||
Acc;
|
||||
_ ->
|
||||
AddFun(ObjKey, IdxValue, Acc)
|
||||
FoldKeysFun(Bucket,
|
||||
AddFun(ObjKey, IdxValue),
|
||||
Acc)
|
||||
end;
|
||||
false ->
|
||||
Acc
|
||||
|
|
@ -1031,10 +1036,12 @@ ttl_test() ->
|
|||
{bucket_stats, "Bucket"}),
|
||||
{_Size, Count} = BucketFolder(),
|
||||
?assertMatch(100, Count),
|
||||
FoldKeysFun = fun(_B, Item, FKFAcc) -> FKFAcc ++ [Item] end,
|
||||
{async,
|
||||
IndexFolder} = book_returnfolder(Bookie1,
|
||||
{index_query,
|
||||
"Bucket",
|
||||
{FoldKeysFun, []},
|
||||
{"idx1_bin", "f8", "f9"},
|
||||
{false, undefined}}),
|
||||
KeyList = IndexFolder(),
|
||||
|
|
@ -1045,6 +1052,7 @@ ttl_test() ->
|
|||
IndexFolderTR} = book_returnfolder(Bookie1,
|
||||
{index_query,
|
||||
"Bucket",
|
||||
{FoldKeysFun, []},
|
||||
{"idx1_bin", "f8", "f9"},
|
||||
{true, Regex}}),
|
||||
TermKeyList = IndexFolderTR(),
|
||||
|
|
@ -1057,6 +1065,7 @@ ttl_test() ->
|
|||
IndexFolderTR2} = book_returnfolder(Bookie2,
|
||||
{index_query,
|
||||
"Bucket",
|
||||
{FoldKeysFun, []},
|
||||
{"idx1_bin", "f7", "f9"},
|
||||
{false, Regex}}),
|
||||
KeyList2 = IndexFolderTR2(),
|
||||
|
|
|
|||
|
|
@ -187,25 +187,21 @@ query_count(_Config) ->
|
|||
ok
|
||||
end,
|
||||
{ok, RegMia} = re:compile("[0-9]+Mia"),
|
||||
Query1 = {index_query,
|
||||
"Bucket",
|
||||
{fun testutil:foldkeysfun/3, []},
|
||||
{"idx2_bin", "2000", "2000~"},
|
||||
{false, RegMia}},
|
||||
{async,
|
||||
Mia2KFolder1} = leveled_bookie:book_returnfolder(Book2,
|
||||
{index_query,
|
||||
"Bucket",
|
||||
{"idx2_bin",
|
||||
"2000",
|
||||
"2000~"},
|
||||
{false,
|
||||
RegMia}}),
|
||||
Mia2KFolder1} = leveled_bookie:book_returnfolder(Book2, Query1),
|
||||
Mia2000Count1 = length(Mia2KFolder1()),
|
||||
Query2 = {index_query,
|
||||
"Bucket",
|
||||
{fun testutil:foldkeysfun/3, []},
|
||||
{"idx2_bin", "2000", "2001"},
|
||||
{true, undefined}},
|
||||
{async,
|
||||
Mia2KFolder2} = leveled_bookie:book_returnfolder(Book2,
|
||||
{index_query,
|
||||
"Bucket",
|
||||
{"idx2_bin",
|
||||
"2000",
|
||||
"2001"},
|
||||
{true,
|
||||
undefined}}),
|
||||
Mia2KFolder2} = leveled_bookie:book_returnfolder(Book2, Query2),
|
||||
Mia2000Count2 = lists:foldl(fun({Term, _Key}, Acc) ->
|
||||
case re:run(Term, RegMia) of
|
||||
nomatch ->
|
||||
|
|
@ -222,15 +218,13 @@ query_count(_Config) ->
|
|||
ok
|
||||
end,
|
||||
{ok, RxMia2K} = re:compile("^2000[0-9]+Mia"),
|
||||
Query3 = {index_query,
|
||||
"Bucket",
|
||||
{fun testutil:foldkeysfun/3, []},
|
||||
{"idx2_bin", "1980", "2100"},
|
||||
{false, RxMia2K}},
|
||||
{async,
|
||||
Mia2KFolder3} = leveled_bookie:book_returnfolder(Book2,
|
||||
{index_query,
|
||||
"Bucket",
|
||||
{"idx2_bin",
|
||||
"1980",
|
||||
"2100"},
|
||||
{false,
|
||||
RxMia2K}}),
|
||||
Mia2KFolder3} = leveled_bookie:book_returnfolder(Book2, Query3),
|
||||
Mia2000Count1 = length(Mia2KFolder3()),
|
||||
|
||||
V9 = testutil:get_compressiblevalue(),
|
||||
|
|
@ -238,13 +232,12 @@ query_count(_Config) ->
|
|||
[{_RN, Obj9, Spc9}] = testutil:generate_objects(1, uuid, [], V9, Indexes9),
|
||||
ok = testutil:book_riakput(Book2, Obj9, Spc9),
|
||||
R9 = lists:map(fun({add, IdxF, IdxT}) ->
|
||||
R = leveled_bookie:book_returnfolder(Book2,
|
||||
{index_query,
|
||||
"Bucket",
|
||||
{IdxF,
|
||||
IdxT,
|
||||
IdxT},
|
||||
?KEY_ONLY}),
|
||||
Q = {index_query,
|
||||
"Bucket",
|
||||
{fun testutil:foldkeysfun/3, []},
|
||||
{IdxF, IdxT, IdxT},
|
||||
?KEY_ONLY},
|
||||
R = leveled_bookie:book_returnfolder(Book2, Q),
|
||||
{async, Fldr} = R,
|
||||
case length(Fldr()) of
|
||||
X when X > 0 ->
|
||||
|
|
@ -256,13 +249,12 @@ query_count(_Config) ->
|
|||
Spc9),
|
||||
ok = testutil:book_riakput(Book2, Obj9, Spc9Del),
|
||||
lists:foreach(fun({IdxF, IdxT, X}) ->
|
||||
R = leveled_bookie:book_returnfolder(Book2,
|
||||
{index_query,
|
||||
"Bucket",
|
||||
{IdxF,
|
||||
IdxT,
|
||||
IdxT},
|
||||
?KEY_ONLY}),
|
||||
Q = {index_query,
|
||||
"Bucket",
|
||||
{fun testutil:foldkeysfun/3, []},
|
||||
{IdxF, IdxT, IdxT},
|
||||
?KEY_ONLY},
|
||||
R = leveled_bookie:book_returnfolder(Book2, Q),
|
||||
{async, Fldr} = R,
|
||||
case length(Fldr()) of
|
||||
Y ->
|
||||
|
|
@ -273,13 +265,12 @@ query_count(_Config) ->
|
|||
ok = leveled_bookie:book_close(Book2),
|
||||
{ok, Book3} = leveled_bookie:book_start(RootPath, 2000, 50000000),
|
||||
lists:foreach(fun({IdxF, IdxT, X}) ->
|
||||
R = leveled_bookie:book_returnfolder(Book3,
|
||||
{index_query,
|
||||
"Bucket",
|
||||
{IdxF,
|
||||
IdxT,
|
||||
IdxT},
|
||||
?KEY_ONLY}),
|
||||
Q = {index_query,
|
||||
"Bucket",
|
||||
{fun testutil:foldkeysfun/3, []},
|
||||
{IdxF, IdxT, IdxT},
|
||||
?KEY_ONLY},
|
||||
R = leveled_bookie:book_returnfolder(Book3, Q),
|
||||
{async, Fldr} = R,
|
||||
case length(Fldr()) of
|
||||
Y ->
|
||||
|
|
@ -291,13 +282,12 @@ query_count(_Config) ->
|
|||
ok = leveled_bookie:book_close(Book3),
|
||||
{ok, Book4} = leveled_bookie:book_start(RootPath, 2000, 50000000),
|
||||
lists:foreach(fun({IdxF, IdxT, X}) ->
|
||||
R = leveled_bookie:book_returnfolder(Book4,
|
||||
{index_query,
|
||||
"Bucket",
|
||||
{IdxF,
|
||||
IdxT,
|
||||
IdxT},
|
||||
?KEY_ONLY}),
|
||||
Q = {index_query,
|
||||
"Bucket",
|
||||
{fun testutil:foldkeysfun/3, []},
|
||||
{IdxF, IdxT, IdxT},
|
||||
?KEY_ONLY},
|
||||
R = leveled_bookie:book_returnfolder(Book4, Q),
|
||||
{async, Fldr} = R,
|
||||
case length(Fldr()) of
|
||||
X ->
|
||||
|
|
@ -316,13 +306,12 @@ count_termsonindex(Bucket, IdxField, Book, QType) ->
|
|||
SW = os:timestamp(),
|
||||
ST = integer_to_list(X),
|
||||
ET = ST ++ "~",
|
||||
R = leveled_bookie:book_returnfolder(Book,
|
||||
{index_query,
|
||||
Bucket,
|
||||
{IdxField,
|
||||
ST,
|
||||
ET},
|
||||
QType}),
|
||||
Q = {index_query,
|
||||
Bucket,
|
||||
{fun testutil:foldkeysfun/3, []},
|
||||
{IdxField, ST, ET},
|
||||
QType},
|
||||
R = leveled_bookie:book_returnfolder(Book, Q),
|
||||
{async, Folder} = R,
|
||||
Items = length(Folder()),
|
||||
io:format("2i query from term ~s on index ~s took " ++
|
||||
|
|
|
|||
|
|
@ -70,21 +70,15 @@ recovr_strategy(_Config) ->
|
|||
true = V == V4
|
||||
end,
|
||||
lists:nthtail(6400, AllSpcL)),
|
||||
{async, TFolder} = leveled_bookie:book_returnfolder(Book1,
|
||||
{index_query,
|
||||
"Bucket6",
|
||||
{"idx1_bin",
|
||||
"#", "~"},
|
||||
{true,
|
||||
undefined}}),
|
||||
Q = fun(RT) -> {index_query,
|
||||
"Bucket6",
|
||||
{fun testutil:foldkeysfun/3, []},
|
||||
{"idx1_bin", "#", "~"},
|
||||
{RT, undefined}}
|
||||
end,
|
||||
{async, TFolder} = leveled_bookie:book_returnfolder(Book1, Q(true)),
|
||||
KeyTermList = TFolder(),
|
||||
{async, KFolder} = leveled_bookie:book_returnfolder(Book1,
|
||||
{index_query,
|
||||
"Bucket6",
|
||||
{"idx1_bin",
|
||||
"#", "~"},
|
||||
{false,
|
||||
undefined}}),
|
||||
{async, KFolder} = leveled_bookie:book_returnfolder(Book1, Q(false)),
|
||||
KeyList = lists:usort(KFolder()),
|
||||
io:format("KeyList ~w KeyTermList ~w~n",
|
||||
[length(KeyList), length(KeyTermList)]),
|
||||
|
|
|
|||
|
|
@ -40,7 +40,8 @@
|
|||
restore_topending/2,
|
||||
find_journals/1,
|
||||
riak_hash/1,
|
||||
wait_for_compaction/1]).
|
||||
wait_for_compaction/1,
|
||||
foldkeysfun/3]).
|
||||
|
||||
-define(RETURN_TERMS, {true, undefined}).
|
||||
-define(SLOWOFFER_DELAY, 5).
|
||||
|
|
@ -328,6 +329,8 @@ get_randomdate() ->
|
|||
[Year, Month, Day, Hour, Minute, Second])).
|
||||
|
||||
|
||||
foldkeysfun(_Bucket, Item, Acc) -> Acc ++ [Item].
|
||||
|
||||
check_indexed_objects(Book, B, KSpecL, V) ->
|
||||
% Check all objects match, return what should be the results of an all
|
||||
% index query
|
||||
|
|
@ -343,6 +346,7 @@ check_indexed_objects(Book, B, KSpecL, V) ->
|
|||
R = leveled_bookie:book_returnfolder(Book,
|
||||
{index_query,
|
||||
B,
|
||||
{fun foldkeysfun/3, []},
|
||||
{"idx1_bin",
|
||||
"0",
|
||||
"~"},
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue