From ac223ced680f62f97abd33091e14cae83276fba6 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 18 Nov 2016 11:53:14 +0000 Subject: [PATCH] Add FoldKeysFun Add the capability to pass FoldKeysFun into the index_query to allow for compatability with riak backend requirements. --- src/leveled_bookie.erl | 35 ++++++---- test/end_to_end/iterator_SUITE.erl | 107 +++++++++++++---------------- test/end_to_end/recovery_SUITE.erl | 22 +++--- test/end_to_end/testutil.erl | 6 +- 4 files changed, 83 insertions(+), 87 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 2ebc012..d1cc1b9 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -337,11 +337,13 @@ handle_call({return_folder, FolderType}, _From, State) -> State}; {index_query, Bucket, + {FoldKeysFun, Acc}, {IdxField, StartValue, EndValue}, {ReturnTerms, TermRegex}} -> {reply, index_query(State, Bucket, + {FoldKeysFun, Acc}, {IdxField, StartValue, EndValue}, {ReturnTerms, TermRegex}), State}; @@ -430,6 +432,7 @@ bucket_stats(State, Bucket, Tag) -> index_query(State, Bucket, + {FoldKeysFun, InitAcc}, {IdxField, StartValue, EndValue}, {ReturnTerms, TermRegex}) -> {ok, @@ -445,16 +448,16 @@ index_query(State, IdxField, EndValue), AddFun = case ReturnTerms of true -> - fun add_terms/3; + fun add_terms/2; _ -> - fun add_keys/3 + fun add_keys/2 end, - AccFun = accumulate_index(TermRegex, AddFun), + AccFun = accumulate_index(TermRegex, AddFun, FoldKeysFun), Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, StartKey, EndKey, AccFun, - []), + InitAcc), ok = leveled_penciller:pcl_close(LedgerSnapshot), Acc end, @@ -724,23 +727,23 @@ accumulate_keys() -> end, AccFun. -add_keys(ObjKey, _IdxValue, Acc) -> - Acc ++ [ObjKey]. +add_keys(ObjKey, _IdxValue) -> + ObjKey. -add_terms(ObjKey, IdxValue, Acc) -> - Acc ++ [{IdxValue, ObjKey}]. +add_terms(ObjKey, IdxValue) -> + {IdxValue, ObjKey}. -accumulate_index(TermRe, AddFun) -> +accumulate_index(TermRe, AddFun, FoldKeysFun) -> Now = leveled_codec:integer_now(), case TermRe of undefined -> fun(Key, Value, Acc) -> case leveled_codec:is_active(Key, Value, Now) of true -> - {_Bucket, + {Bucket, ObjKey, IdxValue} = leveled_codec:from_ledgerkey(Key), - AddFun(ObjKey, IdxValue, Acc); + FoldKeysFun(Bucket, AddFun(ObjKey, IdxValue), Acc); false -> Acc end end; @@ -748,14 +751,16 @@ accumulate_index(TermRe, AddFun) -> fun(Key, Value, Acc) -> case leveled_codec:is_active(Key, Value, Now) of true -> - {_Bucket, + {Bucket, ObjKey, IdxValue} = leveled_codec:from_ledgerkey(Key), case re:run(IdxValue, TermRe) of nomatch -> Acc; _ -> - AddFun(ObjKey, IdxValue, Acc) + FoldKeysFun(Bucket, + AddFun(ObjKey, IdxValue), + Acc) end; false -> Acc @@ -1031,10 +1036,12 @@ ttl_test() -> {bucket_stats, "Bucket"}), {_Size, Count} = BucketFolder(), ?assertMatch(100, Count), + FoldKeysFun = fun(_B, Item, FKFAcc) -> FKFAcc ++ [Item] end, {async, IndexFolder} = book_returnfolder(Bookie1, {index_query, "Bucket", + {FoldKeysFun, []}, {"idx1_bin", "f8", "f9"}, {false, undefined}}), KeyList = IndexFolder(), @@ -1045,6 +1052,7 @@ ttl_test() -> IndexFolderTR} = book_returnfolder(Bookie1, {index_query, "Bucket", + {FoldKeysFun, []}, {"idx1_bin", "f8", "f9"}, {true, Regex}}), TermKeyList = IndexFolderTR(), @@ -1057,6 +1065,7 @@ ttl_test() -> IndexFolderTR2} = book_returnfolder(Bookie2, {index_query, "Bucket", + {FoldKeysFun, []}, {"idx1_bin", "f7", "f9"}, {false, Regex}}), KeyList2 = IndexFolderTR2(), diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index 538f37a..d1bddc0 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -187,25 +187,21 @@ query_count(_Config) -> ok end, {ok, RegMia} = re:compile("[0-9]+Mia"), + Query1 = {index_query, + "Bucket", + {fun testutil:foldkeysfun/3, []}, + {"idx2_bin", "2000", "2000~"}, + {false, RegMia}}, {async, - Mia2KFolder1} = leveled_bookie:book_returnfolder(Book2, - {index_query, - "Bucket", - {"idx2_bin", - "2000", - "2000~"}, - {false, - RegMia}}), + Mia2KFolder1} = leveled_bookie:book_returnfolder(Book2, Query1), Mia2000Count1 = length(Mia2KFolder1()), + Query2 = {index_query, + "Bucket", + {fun testutil:foldkeysfun/3, []}, + {"idx2_bin", "2000", "2001"}, + {true, undefined}}, {async, - Mia2KFolder2} = leveled_bookie:book_returnfolder(Book2, - {index_query, - "Bucket", - {"idx2_bin", - "2000", - "2001"}, - {true, - undefined}}), + Mia2KFolder2} = leveled_bookie:book_returnfolder(Book2, Query2), Mia2000Count2 = lists:foldl(fun({Term, _Key}, Acc) -> case re:run(Term, RegMia) of nomatch -> @@ -222,15 +218,13 @@ query_count(_Config) -> ok end, {ok, RxMia2K} = re:compile("^2000[0-9]+Mia"), + Query3 = {index_query, + "Bucket", + {fun testutil:foldkeysfun/3, []}, + {"idx2_bin", "1980", "2100"}, + {false, RxMia2K}}, {async, - Mia2KFolder3} = leveled_bookie:book_returnfolder(Book2, - {index_query, - "Bucket", - {"idx2_bin", - "1980", - "2100"}, - {false, - RxMia2K}}), + Mia2KFolder3} = leveled_bookie:book_returnfolder(Book2, Query3), Mia2000Count1 = length(Mia2KFolder3()), V9 = testutil:get_compressiblevalue(), @@ -238,13 +232,12 @@ query_count(_Config) -> [{_RN, Obj9, Spc9}] = testutil:generate_objects(1, uuid, [], V9, Indexes9), ok = testutil:book_riakput(Book2, Obj9, Spc9), R9 = lists:map(fun({add, IdxF, IdxT}) -> - R = leveled_bookie:book_returnfolder(Book2, - {index_query, - "Bucket", - {IdxF, - IdxT, - IdxT}, - ?KEY_ONLY}), + Q = {index_query, + "Bucket", + {fun testutil:foldkeysfun/3, []}, + {IdxF, IdxT, IdxT}, + ?KEY_ONLY}, + R = leveled_bookie:book_returnfolder(Book2, Q), {async, Fldr} = R, case length(Fldr()) of X when X > 0 -> @@ -256,13 +249,12 @@ query_count(_Config) -> Spc9), ok = testutil:book_riakput(Book2, Obj9, Spc9Del), lists:foreach(fun({IdxF, IdxT, X}) -> - R = leveled_bookie:book_returnfolder(Book2, - {index_query, - "Bucket", - {IdxF, - IdxT, - IdxT}, - ?KEY_ONLY}), + Q = {index_query, + "Bucket", + {fun testutil:foldkeysfun/3, []}, + {IdxF, IdxT, IdxT}, + ?KEY_ONLY}, + R = leveled_bookie:book_returnfolder(Book2, Q), {async, Fldr} = R, case length(Fldr()) of Y -> @@ -273,13 +265,12 @@ query_count(_Config) -> ok = leveled_bookie:book_close(Book2), {ok, Book3} = leveled_bookie:book_start(RootPath, 2000, 50000000), lists:foreach(fun({IdxF, IdxT, X}) -> - R = leveled_bookie:book_returnfolder(Book3, - {index_query, - "Bucket", - {IdxF, - IdxT, - IdxT}, - ?KEY_ONLY}), + Q = {index_query, + "Bucket", + {fun testutil:foldkeysfun/3, []}, + {IdxF, IdxT, IdxT}, + ?KEY_ONLY}, + R = leveled_bookie:book_returnfolder(Book3, Q), {async, Fldr} = R, case length(Fldr()) of Y -> @@ -291,13 +282,12 @@ query_count(_Config) -> ok = leveled_bookie:book_close(Book3), {ok, Book4} = leveled_bookie:book_start(RootPath, 2000, 50000000), lists:foreach(fun({IdxF, IdxT, X}) -> - R = leveled_bookie:book_returnfolder(Book4, - {index_query, - "Bucket", - {IdxF, - IdxT, - IdxT}, - ?KEY_ONLY}), + Q = {index_query, + "Bucket", + {fun testutil:foldkeysfun/3, []}, + {IdxF, IdxT, IdxT}, + ?KEY_ONLY}, + R = leveled_bookie:book_returnfolder(Book4, Q), {async, Fldr} = R, case length(Fldr()) of X -> @@ -316,13 +306,12 @@ count_termsonindex(Bucket, IdxField, Book, QType) -> SW = os:timestamp(), ST = integer_to_list(X), ET = ST ++ "~", - R = leveled_bookie:book_returnfolder(Book, - {index_query, - Bucket, - {IdxField, - ST, - ET}, - QType}), + Q = {index_query, + Bucket, + {fun testutil:foldkeysfun/3, []}, + {IdxField, ST, ET}, + QType}, + R = leveled_bookie:book_returnfolder(Book, Q), {async, Folder} = R, Items = length(Folder()), io:format("2i query from term ~s on index ~s took " ++ diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index d522efc..3d51cd9 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -70,21 +70,15 @@ recovr_strategy(_Config) -> true = V == V4 end, lists:nthtail(6400, AllSpcL)), - {async, TFolder} = leveled_bookie:book_returnfolder(Book1, - {index_query, - "Bucket6", - {"idx1_bin", - "#", "~"}, - {true, - undefined}}), + Q = fun(RT) -> {index_query, + "Bucket6", + {fun testutil:foldkeysfun/3, []}, + {"idx1_bin", "#", "~"}, + {RT, undefined}} + end, + {async, TFolder} = leveled_bookie:book_returnfolder(Book1, Q(true)), KeyTermList = TFolder(), - {async, KFolder} = leveled_bookie:book_returnfolder(Book1, - {index_query, - "Bucket6", - {"idx1_bin", - "#", "~"}, - {false, - undefined}}), + {async, KFolder} = leveled_bookie:book_returnfolder(Book1, Q(false)), KeyList = lists:usort(KFolder()), io:format("KeyList ~w KeyTermList ~w~n", [length(KeyList), length(KeyTermList)]), diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index e08de45..4ac529d 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -40,7 +40,8 @@ restore_topending/2, find_journals/1, riak_hash/1, - wait_for_compaction/1]). + wait_for_compaction/1, + foldkeysfun/3]). -define(RETURN_TERMS, {true, undefined}). -define(SLOWOFFER_DELAY, 5). @@ -328,6 +329,8 @@ get_randomdate() -> [Year, Month, Day, Hour, Minute, Second])). +foldkeysfun(_Bucket, Item, Acc) -> Acc ++ [Item]. + check_indexed_objects(Book, B, KSpecL, V) -> % Check all objects match, return what should be the results of an all % index query @@ -343,6 +346,7 @@ check_indexed_objects(Book, B, KSpecL, V) -> R = leveled_bookie:book_returnfolder(Book, {index_query, B, + {fun foldkeysfun/3, []}, {"idx1_bin", "0", "~"},