Add tests using fold_heads
Comparing the inbuilt tictac_tree fold, to using "proper" abstraction and achieving the same thing through fold_heads. The fold_heads method is slower (a lot more manipulation required in the fold) - expect it to require > 2 x CPU. However, this does give the flexibility to change the hash algorithm. This would allow for a fold over a database of AAE trees (where the hash has been pre-computed using sha) to be compared with a fold over a database of leveled backends. Also can vary whether the fold_heads checks for presence of the object in the Inker. So normally we can get the speed advantage of not checking the Journal for presence, but periodically we can.
This commit is contained in:
Martin Sumner
parent
dd20132892
commit
53ddc8950b
2 changed files with 207 additions and 121 deletions
|
|
@ -102,6 +102,7 @@
|
|||
put_timing :: tuple() | undefined,
|
||||
get_timing :: tuple() | undefined}).
|
||||
|
||||
-type book_state() :: #state{}.
|
||||
|
||||
%%%============================================================================
|
||||
%%% API
|
||||
|
|
@ -595,13 +596,21 @@ handle_call({return_folder, FolderType}, _From, State) ->
|
|||
TreeSize,
|
||||
PartitionFilter),
|
||||
State};
|
||||
{foldheads_allkeys, Tag, FoldHeadsFun} ->
|
||||
{foldheads_allkeys, Tag, FoldHeadsFun,
|
||||
CheckPresence, SnapPreFold} ->
|
||||
{reply,
|
||||
foldheads_allkeys(State, Tag, FoldHeadsFun),
|
||||
foldheads_allkeys(State, Tag,
|
||||
FoldHeadsFun,
|
||||
CheckPresence,
|
||||
SnapPreFold),
|
||||
State};
|
||||
{foldheads_bybucket, Tag, Bucket, FoldHeadsFun} ->
|
||||
{foldheads_bybucket, Tag, Bucket, FoldHeadsFun,
|
||||
CheckPresence, SnapPreFold} ->
|
||||
{reply,
|
||||
foldheads_bybucket(State, Tag, Bucket, FoldHeadsFun),
|
||||
foldheads_bybucket(State, Tag, Bucket,
|
||||
FoldHeadsFun,
|
||||
CheckPresence,
|
||||
SnapPreFold),
|
||||
State};
|
||||
{foldobjects_allkeys, Tag, FoldObjectsFun} ->
|
||||
{reply,
|
||||
|
|
@ -982,22 +991,26 @@ tictactree(State, Tag, Bucket, Query, JournalCheck, TreeSize, Filter) ->
|
|||
foldobjects_allkeys(State, Tag, FoldObjectsFun) ->
|
||||
StartKey = leveled_codec:to_ledgerkey(null, null, Tag),
|
||||
EndKey = leveled_codec:to_ledgerkey(null, null, Tag),
|
||||
foldobjects(State, Tag, StartKey, EndKey, FoldObjectsFun, false).
|
||||
foldobjects(State, Tag, StartKey, EndKey, FoldObjectsFun,
|
||||
false, true).
|
||||
|
||||
foldheads_allkeys(State, Tag, FoldHeadsFun) ->
|
||||
foldheads_allkeys(State, Tag, FoldHeadsFun, CheckPresence, SnapPreFold) ->
|
||||
StartKey = leveled_codec:to_ledgerkey(null, null, Tag),
|
||||
EndKey = leveled_codec:to_ledgerkey(null, null, Tag),
|
||||
foldobjects(State, Tag, StartKey, EndKey, FoldHeadsFun, true).
|
||||
foldobjects(State, Tag, StartKey, EndKey, FoldHeadsFun,
|
||||
{true, CheckPresence}, SnapPreFold).
|
||||
|
||||
foldobjects_bybucket(State, Tag, Bucket, FoldObjectsFun) ->
|
||||
StartKey = leveled_codec:to_ledgerkey(Bucket, null, Tag),
|
||||
EndKey = leveled_codec:to_ledgerkey(Bucket, null, Tag),
|
||||
foldobjects(State, Tag, StartKey, EndKey, FoldObjectsFun, false).
|
||||
foldobjects(State, Tag, StartKey, EndKey, FoldObjectsFun,
|
||||
false, true).
|
||||
|
||||
foldheads_bybucket(State, Tag, Bucket, FoldHeadsFun) ->
|
||||
foldheads_bybucket(State, Tag, Bucket, FoldHeadsFun, CheckPresence, SnapPreFold) ->
|
||||
StartKey = leveled_codec:to_ledgerkey(Bucket, null, Tag),
|
||||
EndKey = leveled_codec:to_ledgerkey(Bucket, null, Tag),
|
||||
foldobjects(State, Tag, StartKey, EndKey, FoldHeadsFun, true).
|
||||
foldobjects(State, Tag, StartKey, EndKey, FoldHeadsFun,
|
||||
{true, CheckPresence}, SnapPreFold).
|
||||
|
||||
foldobjects_byindex(State, Tag, Bucket,
|
||||
Field, FromTerm, ToTerm, FoldObjectsFun) ->
|
||||
|
|
@ -1005,34 +1018,53 @@ foldobjects_byindex(State, Tag, Bucket,
|
|||
leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, Field, FromTerm),
|
||||
EndKey =
|
||||
leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, Field, ToTerm),
|
||||
foldobjects(State, Tag, StartKey, EndKey, FoldObjectsFun, false).
|
||||
foldobjects(State, Tag, StartKey, EndKey, FoldObjectsFun,
|
||||
false, true).
|
||||
|
||||
|
||||
foldobjects(_State, Tag, StartKey, EndKey, FoldObjectsFun, DeferredFetch) ->
|
||||
{FoldFun, InitAcc} = case is_tuple(FoldObjectsFun) of
|
||||
true ->
|
||||
FoldObjectsFun;
|
||||
false ->
|
||||
{FoldObjectsFun, []}
|
||||
end,
|
||||
% For fold_objects the snapshot has been moved inside of the Folder
|
||||
% function.
|
||||
%
|
||||
% fold_objects and fold_heads are called by the riak_kv_sweeper in Riak,
|
||||
% and the sweeper prompts the fold before checking to see if the fold is
|
||||
% ready to be run. This may lead to the fold being called on an old
|
||||
% snapshot.
|
||||
Self = self(),
|
||||
Folder =
|
||||
fun() ->
|
||||
{ok,
|
||||
LedgerSnapshot,
|
||||
JournalSnapshot} = book_snapshotstore(Self, Self, 5400),
|
||||
-spec foldobjects(book_state(), atom(), tuple(), tuple(), fun(),
|
||||
false|{true, boolean()}, boolean()) ->
|
||||
{async, fun()}.
|
||||
%% @doc
|
||||
%% The object folder should be passed DeferredFetch and SnapPreFold.
|
||||
%% DeferredFetch can either be false (which will return to the fold function
|
||||
%% the full object), or {true, CheckPresence} - in which case a proxy object
|
||||
%% will be created that if understood by the fold function will allow the fold
|
||||
%% function to work on the head of the object, and defer fetching the body in
|
||||
%% case such a fetch is unecessary.
|
||||
%% SnapPreFold determines if the snapshot of the database is done prior to
|
||||
%% returning the Folder function (SnapPreFold=true) or when the Folder function
|
||||
%% is called as Folder() {SnapPreFold=false}
|
||||
foldobjects(State, Tag, StartKey, EndKey, FoldObjectsFun,
|
||||
DeferredFetch, SnapPreFold) ->
|
||||
{FoldFun, InitAcc} =
|
||||
case is_tuple(FoldObjectsFun) of
|
||||
true ->
|
||||
% FoldObjectsFun is already a tuple with a Fold function and an
|
||||
% initial accumulator
|
||||
FoldObjectsFun;
|
||||
false ->
|
||||
% no initial accumulatr passed, and so should be just a list
|
||||
{FoldObjectsFun, []}
|
||||
end,
|
||||
SnapFun =
|
||||
case SnapPreFold of
|
||||
true ->
|
||||
{ok, LS, JS} = snapshot_store(State, store, undefined),
|
||||
fun() -> {ok, LS, JS} end;
|
||||
false ->
|
||||
Self = self(),
|
||||
% Timeout will be ignored, as will Requestor
|
||||
%
|
||||
% This uses the external snapshot - as the snpshot will need
|
||||
% This uses the external snapshot - as the snapshot will need
|
||||
% to have consistent state between Bookie and Penciller when
|
||||
% it is made.
|
||||
fun() -> book_snapshotstore(Self, Self, 5400) end
|
||||
end,
|
||||
|
||||
Folder =
|
||||
fun() ->
|
||||
{ok, LedgerSnapshot, JournalSnapshot} = SnapFun(),
|
||||
|
||||
AccFun = accumulate_objects(FoldFun,
|
||||
JournalSnapshot,
|
||||
Tag,
|
||||
|
|
@ -1292,30 +1324,25 @@ accumulate_objects(FoldObjectsFun, InkerClone, Tag, DeferredFetch) ->
|
|||
end,
|
||||
JK = {leveled_codec:to_ledgerkey(B, K, Tag), SQN},
|
||||
case DeferredFetch of
|
||||
true ->
|
||||
{true, true} ->
|
||||
InJournal =
|
||||
leveled_inker:ink_keycheck(InkerClone,
|
||||
LK,
|
||||
SQN),
|
||||
case InJournal of
|
||||
probably ->
|
||||
Size = leveled_codec:get_size(LK, V),
|
||||
MDBin =
|
||||
leveled_codec:build_metadata_object(LK,
|
||||
MD),
|
||||
Value = {proxy_object,
|
||||
MDBin,
|
||||
Size,
|
||||
{fun fetch_value/2,
|
||||
InkerClone,
|
||||
JK}},
|
||||
FoldObjectsFun(B,
|
||||
K,
|
||||
term_to_binary(Value),
|
||||
Acc);
|
||||
ProxyObj = make_proxy_object(LK, JK,
|
||||
MD, V,
|
||||
InkerClone),
|
||||
FoldObjectsFun(B, K,ProxyObj, Acc);
|
||||
missing ->
|
||||
Acc
|
||||
end;
|
||||
{true, false} ->
|
||||
ProxyObj = make_proxy_object(LK, JK,
|
||||
MD, V,
|
||||
InkerClone),
|
||||
FoldObjectsFun(B, K,ProxyObj, Acc);
|
||||
false ->
|
||||
R = fetch_value(InkerClone, JK),
|
||||
case R of
|
||||
|
|
@ -1332,6 +1359,13 @@ accumulate_objects(FoldObjectsFun, InkerClone, Tag, DeferredFetch) ->
|
|||
end,
|
||||
AccFun.
|
||||
|
||||
make_proxy_object(LK, JK, MD, V, InkerClone) ->
|
||||
Size = leveled_codec:get_size(LK, V),
|
||||
MDBin = leveled_codec:build_metadata_object(LK, MD),
|
||||
term_to_binary({proxy_object,
|
||||
MDBin,
|
||||
Size,
|
||||
{fun fetch_value/2, InkerClone, JK}}).
|
||||
|
||||
check_presence(Key, Value, InkerClone) ->
|
||||
{LedgerKey, SQN} = leveled_codec:strip_to_keyseqonly({Key, Value}),
|
||||
|
|
|
|||
|
|
@ -113,31 +113,83 @@ many_put_compare(_Config) ->
|
|||
|
||||
% Now run the same query by putting the tree-building responsibility onto
|
||||
% the fold_objects_fun
|
||||
HashFun =
|
||||
fun(_Key, Value) ->
|
||||
{proxy_object, HeadBin, _Size, _FetchFun} = binary_to_term(Value),
|
||||
<<?MAGIC:8/integer, ?V1_VERS:8/integer, VclockLen:32/integer,
|
||||
Rest/binary>> = HeadBin,
|
||||
<<VclockBin:VclockLen/binary, _NotNeeded/binary>> = Rest,
|
||||
erlang:phash2(lists:sort(binary_to_term(VclockBin)))
|
||||
|
||||
ApplyHash =
|
||||
fun(HashFun) ->
|
||||
fun(_Key, Value) ->
|
||||
{proxy_object, HeadBin, _Size, _FetchFun} = binary_to_term(Value),
|
||||
<<?MAGIC:8/integer, ?V1_VERS:8/integer, VclockLen:32/integer,
|
||||
Rest/binary>> = HeadBin,
|
||||
<<VclockBin:VclockLen/binary, _NotNeeded/binary>> = Rest,
|
||||
HashFun(lists:sort(binary_to_term(VclockBin)))
|
||||
end
|
||||
end,
|
||||
FoldObjectsFun =
|
||||
fun(_Bucket, Key, Value, Acc) ->
|
||||
leveled_tictac:add_kv(Acc, Key, Value, HashFun)
|
||||
leveled_tictac:add_kv(Acc, Key, Value, ApplyHash(fun erlang:phash2/1))
|
||||
end,
|
||||
FoldQ = {foldheads_bybucket,
|
||||
|
||||
FoldQ0 = {foldheads_bybucket,
|
||||
o_rkv,
|
||||
"Bucket",
|
||||
{FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)}},
|
||||
{async, TreeAObjFolder} = leveled_bookie:book_returnfolder(Bookie2, FoldQ),
|
||||
{FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)},
|
||||
false, true},
|
||||
{async, TreeAObjFolder0} =
|
||||
leveled_bookie:book_returnfolder(Bookie2, FoldQ0),
|
||||
SWB0Obj = os:timestamp(),
|
||||
TreeAObj = TreeAObjFolder(),
|
||||
io:format("Build tictac tree via object foldwith 200K objects in ~w~n",
|
||||
TreeAObj0 = TreeAObjFolder0(),
|
||||
io:format("Build tictac tree via object fold with no "++
|
||||
"presence check and 200K objects in ~w~n",
|
||||
[timer:now_diff(os:timestamp(), SWB0Obj)]),
|
||||
SegList0Obj = leveled_tictac:find_dirtyleaves(TreeA, TreeAObj),
|
||||
io:format("Fold object compared with tictac fold has ~w diffs~n",
|
||||
[length(SegList0Obj)]),
|
||||
true = length(SegList0Obj) == 0,
|
||||
true = length(leveled_tictac:find_dirtyleaves(TreeA, TreeAObj0)) == 0,
|
||||
|
||||
FoldQ1 = {foldheads_bybucket,
|
||||
o_rkv,
|
||||
"Bucket",
|
||||
{FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)},
|
||||
true, true},
|
||||
{async, TreeAObjFolder1} =
|
||||
leveled_bookie:book_returnfolder(Bookie2, FoldQ1),
|
||||
SWB1Obj = os:timestamp(),
|
||||
TreeAObj1 = TreeAObjFolder1(),
|
||||
io:format("Build tictac tree via object fold with "++
|
||||
"presence check and 200K objects in ~w~n",
|
||||
[timer:now_diff(os:timestamp(), SWB1Obj)]),
|
||||
true = length(leveled_tictac:find_dirtyleaves(TreeA, TreeAObj1)) == 0,
|
||||
|
||||
% AAE trees within riak are based on a sha of the vector clock. So to
|
||||
% compare with an AAE tree we need to compare outputs when we're hashing
|
||||
% a hash
|
||||
AltHashFun =
|
||||
fun(Term) ->
|
||||
erlang:phash2(crypto:hash(sha, term_to_binary(Term)))
|
||||
end,
|
||||
AltFoldObjectsFun =
|
||||
fun(_Bucket, Key, Value, Acc) ->
|
||||
leveled_tictac:add_kv(Acc, Key, Value, ApplyHash(AltHashFun))
|
||||
end,
|
||||
AltFoldQ0 = {foldheads_bybucket,
|
||||
o_rkv,
|
||||
"Bucket",
|
||||
{AltFoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)},
|
||||
false, true},
|
||||
{async, TreeAAltObjFolder0} =
|
||||
leveled_bookie:book_returnfolder(Bookie2, AltFoldQ0),
|
||||
SWB2Obj = os:timestamp(),
|
||||
TreeAAltObj = TreeAAltObjFolder0(),
|
||||
io:format("Build tictac tree via object fold with no "++
|
||||
"presence check and 200K objects and alt hash in ~w~n",
|
||||
[timer:now_diff(os:timestamp(), SWB2Obj)]),
|
||||
{async, TreeBAltObjFolder0} =
|
||||
leveled_bookie:book_returnfolder(Bookie3, AltFoldQ0),
|
||||
SWB3Obj = os:timestamp(),
|
||||
TreeBAltObj = TreeBAltObjFolder0(),
|
||||
io:format("Build tictac tree via object fold with no "++
|
||||
"presence check and 200K objects and alt hash in ~w~n",
|
||||
[timer:now_diff(os:timestamp(), SWB3Obj)]),
|
||||
true =
|
||||
length(leveled_tictac:find_dirtyleaves(TreeBAltObj, TreeAAltObj)) == 1,
|
||||
|
||||
|
||||
%% Finding differing keys
|
||||
FoldKeysFun =
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue