umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Add FoldKeysFun

Browse source 
Add the capability to pass FoldKeysFun into the index_query to allow for
compatability with riak backend requirements.
This commit is contained in:
martinsumner 2016-11-18 11:53:14 +00:00
parent e3004faa6e
commit ac223ced68
4 changed files with 83 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

35
src/leveled_bookie.erl
View file

@ -337,11 +337,13 @@ handle_call({return_folder, FolderType}, _From, State) ->
State}; State};
{index_query, {index_query,
Bucket, Bucket,
{FoldKeysFun, Acc},
{IdxField, StartValue, EndValue}, {IdxField, StartValue, EndValue},
{ReturnTerms, TermRegex}} -> {ReturnTerms, TermRegex}} ->
{reply, {reply,
index_query(State, index_query(State,
Bucket, Bucket,
{FoldKeysFun, Acc},
{IdxField, StartValue, EndValue}, {IdxField, StartValue, EndValue},
{ReturnTerms, TermRegex}), {ReturnTerms, TermRegex}),
State}; State};
@ -430,6 +432,7 @@ bucket_stats(State, Bucket, Tag) ->
index_query(State, index_query(State,
Bucket, Bucket,
{FoldKeysFun, InitAcc},
{IdxField, StartValue, EndValue}, {IdxField, StartValue, EndValue},
{ReturnTerms, TermRegex}) -> {ReturnTerms, TermRegex}) ->
{ok, {ok,
@ -445,16 +448,16 @@ index_query(State,
IdxField, EndValue), IdxField, EndValue),
AddFun = case ReturnTerms of AddFun = case ReturnTerms of
true -> true ->
fun add_terms/3; fun add_terms/2;
_ -> _ ->
fun add_keys/3 fun add_keys/2
end, end,
AccFun = accumulate_index(TermRegex, AddFun), AccFun = accumulate_index(TermRegex, AddFun, FoldKeysFun),
Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot,
StartKey, StartKey,
EndKey, EndKey,
AccFun, AccFun,
[]), InitAcc),
ok = leveled_penciller:pcl_close(LedgerSnapshot), ok = leveled_penciller:pcl_close(LedgerSnapshot),
Acc Acc
end, end,
@ -724,23 +727,23 @@ accumulate_keys() ->
end, end,
AccFun. AccFun.
add_keys(ObjKey, _IdxValue, Acc) -> add_keys(ObjKey, _IdxValue) ->
Acc ++ [ObjKey]. ObjKey.
add_terms(ObjKey, IdxValue, Acc) -> add_terms(ObjKey, IdxValue) ->
Acc ++ [{IdxValue, ObjKey}]. {IdxValue, ObjKey}.
accumulate_index(TermRe, AddFun) -> accumulate_index(TermRe, AddFun, FoldKeysFun) ->
Now = leveled_codec:integer_now(), Now = leveled_codec:integer_now(),
case TermRe of case TermRe of
undefined -> undefined ->
fun(Key, Value, Acc) -> fun(Key, Value, Acc) ->
case leveled_codec:is_active(Key, Value, Now) of case leveled_codec:is_active(Key, Value, Now) of
true -> true ->
{_Bucket, {Bucket,
ObjKey, ObjKey,
IdxValue} = leveled_codec:from_ledgerkey(Key), IdxValue} = leveled_codec:from_ledgerkey(Key),
AddFun(ObjKey, IdxValue, Acc); FoldKeysFun(Bucket, AddFun(ObjKey, IdxValue), Acc);
false -> false ->
Acc Acc
end end; end end;
@ -748,14 +751,16 @@ accumulate_index(TermRe, AddFun) ->
fun(Key, Value, Acc) -> fun(Key, Value, Acc) ->
case leveled_codec:is_active(Key, Value, Now) of case leveled_codec:is_active(Key, Value, Now) of
true -> true ->
{_Bucket, {Bucket,
ObjKey, ObjKey,
IdxValue} = leveled_codec:from_ledgerkey(Key), IdxValue} = leveled_codec:from_ledgerkey(Key),
case re:run(IdxValue, TermRe) of case re:run(IdxValue, TermRe) of
nomatch -> nomatch ->
Acc; Acc;
_ -> _ ->
AddFun(ObjKey, IdxValue, Acc) FoldKeysFun(Bucket,
AddFun(ObjKey, IdxValue),
Acc)
end; end;
false -> false ->
Acc Acc
@ -1031,10 +1036,12 @@ ttl_test() ->
{bucket_stats, "Bucket"}), {bucket_stats, "Bucket"}),
{_Size, Count} = BucketFolder(), {_Size, Count} = BucketFolder(),
?assertMatch(100, Count), ?assertMatch(100, Count),
FoldKeysFun = fun(_B, Item, FKFAcc) -> FKFAcc ++ [Item] end,
{async, {async,
IndexFolder} = book_returnfolder(Bookie1, IndexFolder} = book_returnfolder(Bookie1,
{index_query, {index_query,
"Bucket", "Bucket",
{FoldKeysFun, []},
{"idx1_bin", "f8", "f9"}, {"idx1_bin", "f8", "f9"},
{false, undefined}}), {false, undefined}}),
KeyList = IndexFolder(), KeyList = IndexFolder(),
@ -1045,6 +1052,7 @@ ttl_test() ->
IndexFolderTR} = book_returnfolder(Bookie1, IndexFolderTR} = book_returnfolder(Bookie1,
{index_query, {index_query,
"Bucket", "Bucket",
{FoldKeysFun, []},
{"idx1_bin", "f8", "f9"}, {"idx1_bin", "f8", "f9"},
{true, Regex}}), {true, Regex}}),
TermKeyList = IndexFolderTR(), TermKeyList = IndexFolderTR(),
@ -1057,6 +1065,7 @@ ttl_test() ->
IndexFolderTR2} = book_returnfolder(Bookie2, IndexFolderTR2} = book_returnfolder(Bookie2,
{index_query, {index_query,
"Bucket", "Bucket",
{FoldKeysFun, []},
{"idx1_bin", "f7", "f9"}, {"idx1_bin", "f7", "f9"},
{false, Regex}}), {false, Regex}}),
KeyList2 = IndexFolderTR2(), KeyList2 = IndexFolderTR2(),

107
test/end_to_end/iterator_SUITE.erl
View file

@ -187,25 +187,21 @@ query_count(_Config) ->
ok ok
end, end,
{ok, RegMia} = re:compile("[0-9]+Mia"), {ok, RegMia} = re:compile("[0-9]+Mia"),
Query1 = {index_query,
"Bucket",
{fun testutil:foldkeysfun/3, []},
{"idx2_bin", "2000", "2000~"},
{false, RegMia}},
{async, {async,
Mia2KFolder1} = leveled_bookie:book_returnfolder(Book2, Mia2KFolder1} = leveled_bookie:book_returnfolder(Book2, Query1),
{index_query,
"Bucket",
{"idx2_bin",
"2000",
"2000~"},
{false,
RegMia}}),
Mia2000Count1 = length(Mia2KFolder1()), Mia2000Count1 = length(Mia2KFolder1()),
Query2 = {index_query,
"Bucket",
{fun testutil:foldkeysfun/3, []},
{"idx2_bin", "2000", "2001"},
{true, undefined}},
{async, {async,
Mia2KFolder2} = leveled_bookie:book_returnfolder(Book2, Mia2KFolder2} = leveled_bookie:book_returnfolder(Book2, Query2),
{index_query,
"Bucket",
{"idx2_bin",
"2000",
"2001"},
{true,
undefined}}),
Mia2000Count2 = lists:foldl(fun({Term, _Key}, Acc) -> Mia2000Count2 = lists:foldl(fun({Term, _Key}, Acc) ->
case re:run(Term, RegMia) of case re:run(Term, RegMia) of
nomatch -> nomatch ->
@ -222,15 +218,13 @@ query_count(_Config) ->
ok ok
end, end,
{ok, RxMia2K} = re:compile("^2000[0-9]+Mia"), {ok, RxMia2K} = re:compile("^2000[0-9]+Mia"),
Query3 = {index_query,
"Bucket",
{fun testutil:foldkeysfun/3, []},
{"idx2_bin", "1980", "2100"},
{false, RxMia2K}},
{async, {async,
Mia2KFolder3} = leveled_bookie:book_returnfolder(Book2, Mia2KFolder3} = leveled_bookie:book_returnfolder(Book2, Query3),
{index_query,
"Bucket",
{"idx2_bin",
"1980",
"2100"},
{false,
RxMia2K}}),
Mia2000Count1 = length(Mia2KFolder3()), Mia2000Count1 = length(Mia2KFolder3()),
V9 = testutil:get_compressiblevalue(), V9 = testutil:get_compressiblevalue(),
@ -238,13 +232,12 @@ query_count(_Config) ->
[{_RN, Obj9, Spc9}] = testutil:generate_objects(1, uuid, [], V9, Indexes9), [{_RN, Obj9, Spc9}] = testutil:generate_objects(1, uuid, [], V9, Indexes9),
ok = testutil:book_riakput(Book2, Obj9, Spc9), ok = testutil:book_riakput(Book2, Obj9, Spc9),
R9 = lists:map(fun({add, IdxF, IdxT}) -> R9 = lists:map(fun({add, IdxF, IdxT}) ->
R = leveled_bookie:book_returnfolder(Book2, Q = {index_query,
{index_query, "Bucket",
"Bucket", {fun testutil:foldkeysfun/3, []},
{IdxF, {IdxF, IdxT, IdxT},
IdxT, ?KEY_ONLY},
IdxT}, R = leveled_bookie:book_returnfolder(Book2, Q),
?KEY_ONLY}),
{async, Fldr} = R, {async, Fldr} = R,
case length(Fldr()) of case length(Fldr()) of
X when X > 0 -> X when X > 0 ->
@ -256,13 +249,12 @@ query_count(_Config) ->
Spc9), Spc9),
ok = testutil:book_riakput(Book2, Obj9, Spc9Del), ok = testutil:book_riakput(Book2, Obj9, Spc9Del),
lists:foreach(fun({IdxF, IdxT, X}) -> lists:foreach(fun({IdxF, IdxT, X}) ->
R = leveled_bookie:book_returnfolder(Book2, Q = {index_query,
{index_query, "Bucket",
"Bucket", {fun testutil:foldkeysfun/3, []},
{IdxF, {IdxF, IdxT, IdxT},
IdxT, ?KEY_ONLY},
IdxT}, R = leveled_bookie:book_returnfolder(Book2, Q),
?KEY_ONLY}),
{async, Fldr} = R, {async, Fldr} = R,
case length(Fldr()) of case length(Fldr()) of
Y -> Y ->
@ -273,13 +265,12 @@ query_count(_Config) ->
ok = leveled_bookie:book_close(Book2), ok = leveled_bookie:book_close(Book2),
{ok, Book3} = leveled_bookie:book_start(RootPath, 2000, 50000000), {ok, Book3} = leveled_bookie:book_start(RootPath, 2000, 50000000),
lists:foreach(fun({IdxF, IdxT, X}) -> lists:foreach(fun({IdxF, IdxT, X}) ->
R = leveled_bookie:book_returnfolder(Book3, Q = {index_query,
{index_query, "Bucket",
"Bucket", {fun testutil:foldkeysfun/3, []},
{IdxF, {IdxF, IdxT, IdxT},
IdxT, ?KEY_ONLY},
IdxT}, R = leveled_bookie:book_returnfolder(Book3, Q),
?KEY_ONLY}),
{async, Fldr} = R, {async, Fldr} = R,
case length(Fldr()) of case length(Fldr()) of
Y -> Y ->
@ -291,13 +282,12 @@ query_count(_Config) ->
ok = leveled_bookie:book_close(Book3), ok = leveled_bookie:book_close(Book3),
{ok, Book4} = leveled_bookie:book_start(RootPath, 2000, 50000000), {ok, Book4} = leveled_bookie:book_start(RootPath, 2000, 50000000),
lists:foreach(fun({IdxF, IdxT, X}) -> lists:foreach(fun({IdxF, IdxT, X}) ->
R = leveled_bookie:book_returnfolder(Book4, Q = {index_query,
{index_query, "Bucket",
"Bucket", {fun testutil:foldkeysfun/3, []},
{IdxF, {IdxF, IdxT, IdxT},
IdxT, ?KEY_ONLY},
IdxT}, R = leveled_bookie:book_returnfolder(Book4, Q),
?KEY_ONLY}),
{async, Fldr} = R, {async, Fldr} = R,
case length(Fldr()) of case length(Fldr()) of
X -> X ->
@ -316,13 +306,12 @@ count_termsonindex(Bucket, IdxField, Book, QType) ->
SW = os:timestamp(), SW = os:timestamp(),
ST = integer_to_list(X), ST = integer_to_list(X),
ET = ST ++ "~", ET = ST ++ "~",
R = leveled_bookie:book_returnfolder(Book, Q = {index_query,
{index_query, Bucket,
Bucket, {fun testutil:foldkeysfun/3, []},
{IdxField, {IdxField, ST, ET},
ST, QType},
ET}, R = leveled_bookie:book_returnfolder(Book, Q),
QType}),
{async, Folder} = R, {async, Folder} = R,
Items = length(Folder()), Items = length(Folder()),
io:format("2i query from term ~s on index ~s took " ++ io:format("2i query from term ~s on index ~s took " ++

22
test/end_to_end/recovery_SUITE.erl
View file

@ -70,21 +70,15 @@ recovr_strategy(_Config) ->
true = V == V4 true = V == V4
end, end,
lists:nthtail(6400, AllSpcL)), lists:nthtail(6400, AllSpcL)),
{async, TFolder} = leveled_bookie:book_returnfolder(Book1, Q = fun(RT) -> {index_query,
{index_query, "Bucket6",
"Bucket6", {fun testutil:foldkeysfun/3, []},
{"idx1_bin", {"idx1_bin", "#", "~"},
"#", "~"}, {RT, undefined}}
{true, end,
undefined}}), {async, TFolder} = leveled_bookie:book_returnfolder(Book1, Q(true)),
KeyTermList = TFolder(), KeyTermList = TFolder(),
{async, KFolder} = leveled_bookie:book_returnfolder(Book1, {async, KFolder} = leveled_bookie:book_returnfolder(Book1, Q(false)),
{index_query,
"Bucket6",
{"idx1_bin",
"#", "~"},
{false,
undefined}}),
KeyList = lists:usort(KFolder()), KeyList = lists:usort(KFolder()),
io:format("KeyList ~w KeyTermList ~w~n", io:format("KeyList ~w KeyTermList ~w~n",
[length(KeyList), length(KeyTermList)]), [length(KeyList), length(KeyTermList)]),

6
test/end_to_end/testutil.erl
View file

@ -40,7 +40,8 @@
restore_topending/2, restore_topending/2,
find_journals/1, find_journals/1,
riak_hash/1, riak_hash/1,
wait_for_compaction/1]). wait_for_compaction/1,
foldkeysfun/3]).
-define(RETURN_TERMS, {true, undefined}). -define(RETURN_TERMS, {true, undefined}).
-define(SLOWOFFER_DELAY, 5). -define(SLOWOFFER_DELAY, 5).
@ -328,6 +329,8 @@ get_randomdate() ->
[Year, Month, Day, Hour, Minute, Second])). [Year, Month, Day, Hour, Minute, Second])).
foldkeysfun(_Bucket, Item, Acc) -> Acc ++ [Item].
check_indexed_objects(Book, B, KSpecL, V) -> check_indexed_objects(Book, B, KSpecL, V) ->
% Check all objects match, return what should be the results of an all % Check all objects match, return what should be the results of an all
% index query % index query
@ -343,6 +346,7 @@ check_indexed_objects(Book, B, KSpecL, V) ->
R = leveled_bookie:book_returnfolder(Book, R = leveled_bookie:book_returnfolder(Book,
{index_query, {index_query,
B, B,
{fun foldkeysfun/3, []},
{"idx1_bin", {"idx1_bin",
"0", "0",
"~"}, "~"},