umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Add bybucket to fold_heads

Browse source 
give fold_heads equivalent functionality to fold_objects - both can now
be done for allkeys and bybucket
This commit is contained in:
martinsumner 2017-04-07 14:56:28 +00:00
parent 9375baf636
commit 6d42abbc1a
1 changed files with 81 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

82
src/leveled_bookie.erl
View file

@ -536,6 +536,10 @@ handle_call({return_folder, FolderType}, _From, State) ->
{reply,
foldheads_allkeys(State, Tag, FoldHeadsFun),
State};
{foldheads_bybucket, Tag, Bucket, FoldHeadsFun} ->
{reply,
foldheads_bybucket(State, Tag, Bucket, FoldHeadsFun),
State};
{foldobjects_allkeys, Tag, FoldObjectsFun} ->
{reply,
foldobjects_allkeys(State, Tag, FoldObjectsFun),
@ -830,6 +834,11 @@ foldobjects_bybucket(State, Tag, Bucket, FoldObjectsFun) ->
EndKey = leveled_codec:to_ledgerkey(Bucket, null, Tag),
foldobjects(State, Tag, StartKey, EndKey, FoldObjectsFun, false).
foldheads_bybucket(State, Tag, Bucket, FoldHeadsFun) ->
StartKey = leveled_codec:to_ledgerkey(Bucket, null, Tag),
EndKey = leveled_codec:to_ledgerkey(Bucket, null, Tag),
foldobjects(State, Tag, StartKey, EndKey, FoldHeadsFun, true).
foldobjects_byindex(State, Tag, Bucket,
Field, FromTerm, ToTerm, FoldObjectsFun) ->
StartKey =
@ -1497,7 +1506,7 @@ foldobjects_vs_hashtree_test() ->
?STD_TAG,
false}),
KeyHashList1 = lists:usort(HTFolder1()),
io:format("First item ~w~n", [lists:nth(1, KeyHashList1)]),
FoldObjectsFun = fun(B, K, V, Acc) ->
[{B, K, erlang:phash2(term_to_binary(V))}|Acc] end,
{async, HTFolder2} =
@ -1542,6 +1551,77 @@ foldobjects_vs_hashtree_test() ->
reset_filestructure().
foldobjects_vs_foldheads_bybucket_test() ->
RootPath = reset_filestructure(),
{ok, Bookie1} = book_start([{root_path, RootPath},
{max_journalsize, 1000000},
{cache_size, 500}]),
ObjL1 = generate_multiple_objects(400, 1),
ObjL2 = generate_multiple_objects(400, 1),
% Put in all the objects with a TTL in the future
Future = leveled_codec:integer_now() + 300,
lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1,
"BucketA", K, V, S,
?STD_TAG,
Future) end,
ObjL1),
lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1,
"BucketB", K, V, S,
?STD_TAG,
Future) end,
ObjL2),
FoldObjectsFun = fun(B, K, V, Acc) ->
[{B, K, erlang:phash2(term_to_binary(V))}|Acc] end,
{async, HTFolder1A} =
book_returnfolder(Bookie1,
{foldobjects_bybucket,
?STD_TAG,
"BucketA",
FoldObjectsFun}),
KeyHashList1A = HTFolder1A(),
{async, HTFolder1B} =
book_returnfolder(Bookie1,
{foldobjects_bybucket,
?STD_TAG,
"BucketB",
FoldObjectsFun}),
KeyHashList1B = HTFolder1B(),
?assertMatch(false,
lists:usort(KeyHashList1A) == lists:usort(KeyHashList1B)),
FoldHeadsFun =
fun(B, K, ProxyV, Acc) ->
{proxy_object,
_MDBin,
_Size,
{FetchFun, Clone, JK}} = ProxyV,
V = FetchFun(Clone, JK),
[{B, K, erlang:phash2(term_to_binary(V))}|Acc]
end,
{async, HTFolder2A} =
book_returnfolder(Bookie1,
{foldheads_bybucket,
?STD_TAG,
"BucketA",
FoldHeadsFun}),
KeyHashList2A = HTFolder2A(),
{async, HTFolder2B} =
book_returnfolder(Bookie1,
{foldheads_bybucket,
?STD_TAG,
"BucketB",
FoldHeadsFun}),
KeyHashList2B = HTFolder2B(),
?assertMatch(true,
lists:usort(KeyHashList1A) == lists:usort(KeyHashList2A)),
?assertMatch(true,
lists:usort(KeyHashList1B) == lists:usort(KeyHashList2B)),
ok = book_close(Bookie1),
reset_filestructure().
scan_table_test() ->
K1 = leveled_codec:to_ledgerkey(<<"B1">>,