umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Add exportable option to tictac

Browse source 
Idea being that sometimes you may wish to compare a tictac tree between leveled and something that doesn't understand erlang:phash or term_to_binary.  So allow the magic_hash to be used instead - and perhaps an extract function that does base64 encoding or something similar.
This commit is contained in:
Martin Sumner 2017-09-26 22:49:40 +01:00
parent 69ed945e58
commit 389694b11b
4 changed files with 117 additions and 81 deletions
Download patch file Download diff file Expand all files Collapse all files

16
src/leveled_bookie.erl
View file

@ -934,14 +934,10 @@ tictactree(State, Tag, Bucket, Query, JournalCheck, TreeSize, Filter) ->
fun() -> fun() ->
% The start key and end key will vary depending on whether the % The start key and end key will vary depending on whether the
% fold is to fold over an index or a key range % fold is to fold over an index or a key range
{StartKey, EndKey, HashFun} = {StartKey, EndKey, ExtractFun} =
case Tag of case Tag of
?IDX_TAG -> ?IDX_TAG ->
{IdxField, StartIdx, EndIdx} = Query, {IdxField, StartIdx, EndIdx} = Query,
HashIdxValFun =
fun(_Key, IdxValue) ->
erlang:phash2(IdxValue)
end,
{leveled_codec:to_ledgerkey(Bucket, {leveled_codec:to_ledgerkey(Bucket,
null, null,
?IDX_TAG, ?IDX_TAG,
@ -952,23 +948,21 @@ tictactree(State, Tag, Bucket, Query, JournalCheck, TreeSize, Filter) ->
?IDX_TAG, ?IDX_TAG,
IdxField, IdxField,
EndIdx), EndIdx),
HashIdxValFun}; fun(K, T) -> {K, T} end};
_ -> _ ->
{StartObjKey, EndObjKey} = Query, {StartObjKey, EndObjKey} = Query,
PassHashFun = fun(_Key, Hash) -> Hash end,
{leveled_codec:to_ledgerkey(Bucket, {leveled_codec:to_ledgerkey(Bucket,
StartObjKey, StartObjKey,
Tag), Tag),
leveled_codec:to_ledgerkey(Bucket, leveled_codec:to_ledgerkey(Bucket,
EndObjKey, EndObjKey,
Tag), Tag),
PassHashFun} fun(K, H) -> {K, {is_hash, H}} end}
end, end,
AccFun = accumulate_tree(Filter, AccFun = accumulate_tree(Filter,
JournalCheck, JournalCheck,
JournalSnapshot, JournalSnapshot,
HashFun), ExtractFun),
Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot,
StartKey, StartKey,
EndKey, EndKey,
@ -1263,7 +1257,7 @@ accumulate_tree(FilterFun, JournalCheck, InkerClone, HashFun) ->
fun(B, K, H, Tree) -> fun(B, K, H, Tree) ->
case FilterFun(B, K) of case FilterFun(B, K) of
accumulate -> accumulate ->
leveled_tictac:add_kv(Tree, K, H, HashFun); leveled_tictac:add_kv(Tree, K, H, HashFun, false);
pass -> pass ->
Tree Tree
end end

3
src/leveled_codec.erl
View file

@ -464,7 +464,8 @@ aae_indexspecs(AAE, Bucket, Key, SQN, H, LastMods) ->
{LMD1, TTL} -> {LMD1, TTL} ->
TreeSize = AAE#recent_aae.tree_size, TreeSize = AAE#recent_aae.tree_size,
SegID = SegID =
leveled_tictac:get_segment(Key, TreeSize), leveled_tictac:get_segment(erlang:phash2(Key),
TreeSize),
IdxFldStr = ?NRT_IDX ++ LMD1 ++ "_bin", IdxFldStr = ?NRT_IDX ++ LMD1 ++ "_bin",
IdxTrmStr = IdxTrmStr =
string:right(integer_to_list(SegID), 8, $0) ++ string:right(integer_to_list(SegID), 8, $0) ++

122
src/leveled_tictac.erl
View file

@ -57,14 +57,14 @@
-export([ -export([
new_tree/1, new_tree/1,
new_tree/2, new_tree/2,
add_kv/4, add_kv/5,
find_dirtyleaves/2, find_dirtyleaves/2,
find_dirtysegments/2, find_dirtysegments/2,
fetch_root/1, fetch_root/1,
fetch_leaves/2, fetch_leaves/2,
merge_trees/2, merge_trees/2,
get_segment/2, get_segment/2,
tictac_hash/2, tictac_hash/3,
export_tree/1, export_tree/1,
import_tree/1 import_tree/1
]). ]).
@ -161,13 +161,24 @@ import_tree(ExportedTree) ->
level2 = Lv2}. level2 = Lv2}.
-spec add_kv(tictactree(), tuple(), tuple(), fun()) -> tictactree(). -spec add_kv(tictactree(), tuple(), tuple(), fun()) -> tictactree().
add_kv(TicTacTree, Key, Value, BinExtractFun) ->
add_kv(TicTacTree, Key, Value, BinExtractFun, false).
-spec add_kv(tictactree(), tuple(), tuple(), fun(), boolean()) -> tictactree().
%% @doc %% @doc
%% Add a Key and value to a tictactree using the HashFun to calculate the Hash %% Add a Key and value to a tictactree using the BinExtractFun to extract a
%% based on that key and value %% binary from the Key and value from which to generate the hash. The
add_kv(TicTacTree, Key, Value, HashFun) -> %% BinExtractFun will also need to do any canonicalisation necessary to make
HashV = HashFun(Key, Value), %% the hash consistent (such as whitespace removal, or sorting)
SegChangeHash = tictac_hash(Key, HashV), %%
Segment = get_segment(Key, TicTacTree#tictactree.segment_count), %% For exportable trees the hash function will be based on the CJ Bernstein
%% magic hash. For non-exportable trees erlang:phash2 will be used, and so
%% non-binary Keys and Values can be returned from the BinExtractFun in this
%% case.
add_kv(TicTacTree, Key, Value, BinExtractFun, Exportable) ->
{BinK, BinV} = BinExtractFun(Key, Value),
{SegHash, SegChangeHash} = tictac_hash(BinK, BinV, Exportable),
Segment = get_segment(SegHash, TicTacTree#tictactree.segment_count),
Level2Pos = Level2Pos =
Segment band (TicTacTree#tictactree.width - 1), Segment band (TicTacTree#tictactree.width - 1),
@ -275,21 +286,33 @@ merge_trees(TreeA, TreeB) ->
MergedTree#tictactree{level1 = NewLevel1, level2 = NewLevel2}. MergedTree#tictactree{level1 = NewLevel1, level2 = NewLevel2}.
-spec get_segment(any(), integer()|small|medium|large|xlarge) -> integer(). -spec get_segment(integer(), integer()|small|medium|large|xlarge) -> integer().
%% @doc %% @doc
%% Return the segment ID for a Key. Can pass the tree size or the actual %% Return the segment ID for a Key. Can pass the tree size or the actual
%% segment count derived from the size %% segment count derived from the size
get_segment(Key, SegmentCount) when is_integer(SegmentCount) -> get_segment(Hash, SegmentCount) when is_integer(SegmentCount) ->
erlang:phash2(Key) band (SegmentCount - 1); Hash band (SegmentCount - 1);
get_segment(Key, TreeSize) -> get_segment(Hash, TreeSize) ->
get_segment(Key, element(3, get_size(TreeSize))). get_segment(Hash, element(3, get_size(TreeSize))).
-spec tictac_hash(tuple(), any()) -> integer(). -spec tictac_hash(any(), any(), boolean()) -> integer().
%% @doc %% @doc
%% Hash the key and term %% Hash the key and term, to either something repetable in Erlang, or using
tictac_hash(Key, Term) -> %% the DJ Bernstein hash if it is the tree needs to be compared with one
erlang:phash2({Key, Term}). %% calculated with a non-Erlang store
%%
%% Boolean is Exportable. does the hash need to be repetable by a non-Erlang
%% machine
tictac_hash(BinKey, BinVal, true)
when is_binary(BinKey) and is_binary(BinVal) ->
HashKey = leveled_codec:magic_hash({binary, BinKey}),
HashVal = leveled_codec:magic_hash({binary, BinVal}),
{HashKey, HashKey bxor HashVal};
tictac_hash(BinKey, {is_hash, HashedVal}, false) ->
{erlang:phash2(BinKey), erlang:phash2(BinKey) bxor HashedVal};
tictac_hash(BinKey, BinVal, false) ->
{erlang:phash2(BinKey), erlang:phash2(BinKey) bxor erlang:phash2(BinVal)}.
%%%============================================================================ %%%============================================================================
%%% Internal functions %%% Internal functions
@ -363,13 +386,17 @@ simple_bysize_test() ->
simple_test_withsize(xlarge). simple_test_withsize(xlarge).
simple_test_withsize(Size) -> simple_test_withsize(Size) ->
HashFun = fun(_K, V) -> erlang:phash2(V) end, BinFun = fun(K, V) -> {term_to_binary(K), term_to_binary(V)} end,
K1 = {o, "B1", "K1", null},
K2 = {o, "B1", "K2", null},
K3 = {o, "B1", "K3", null},
Tree0 = new_tree(0, Size), Tree0 = new_tree(0, Size),
Tree1 = add_kv(Tree0, {o, "B1", "K1", null}, {caine, 1}, HashFun), Tree1 = add_kv(Tree0, K1, {caine, 1}, BinFun),
Tree2 = add_kv(Tree1, {o, "B1", "K2", null}, {caine, 2}, HashFun), Tree2 = add_kv(Tree1, K2, {caine, 2}, BinFun),
Tree3 = add_kv(Tree2, {o, "B1", "K3", null}, {caine, 3}, HashFun), Tree3 = add_kv(Tree2, K3, {caine, 3}, BinFun),
Tree3A = add_kv(Tree3, {o, "B1", "K3", null}, {caine, 4}, HashFun), Tree3A = add_kv(Tree3, K3, {caine, 4}, BinFun),
?assertMatch(true, Tree0#tictactree.level1 == Tree0#tictactree.level1), ?assertMatch(true, Tree0#tictactree.level1 == Tree0#tictactree.level1),
?assertMatch(false, Tree0#tictactree.level1 == Tree1#tictactree.level1), ?assertMatch(false, Tree0#tictactree.level1 == Tree1#tictactree.level1),
?assertMatch(false, Tree1#tictactree.level1 == Tree2#tictactree.level1), ?assertMatch(false, Tree1#tictactree.level1 == Tree2#tictactree.level1),
@ -377,23 +404,28 @@ simple_test_withsize(Size) ->
?assertMatch(false, Tree3#tictactree.level1 == Tree3A#tictactree.level1), ?assertMatch(false, Tree3#tictactree.level1 == Tree3A#tictactree.level1),
Tree0X = new_tree(0, Size), Tree0X = new_tree(0, Size),
Tree1X = add_kv(Tree0X, {o, "B1", "K3", null}, {caine, 3}, HashFun), Tree1X = add_kv(Tree0X, K3, {caine, 3}, BinFun),
Tree2X = add_kv(Tree1X, {o, "B1", "K1", null}, {caine, 1}, HashFun), Tree2X = add_kv(Tree1X, K1, {caine, 1}, BinFun),
Tree3X = add_kv(Tree2X, {o, "B1", "K2", null}, {caine, 2}, HashFun), Tree3X = add_kv(Tree2X, K2, {caine, 2}, BinFun),
Tree3XA = add_kv(Tree3X, {o, "B1", "K3", null}, {caine, 4}, HashFun), Tree3XA = add_kv(Tree3X, K3, {caine, 4}, BinFun),
?assertMatch(false, Tree1#tictactree.level1 == Tree1X#tictactree.level1), ?assertMatch(false, Tree1#tictactree.level1 == Tree1X#tictactree.level1),
?assertMatch(false, Tree2#tictactree.level1 == Tree2X#tictactree.level1), ?assertMatch(false, Tree2#tictactree.level1 == Tree2X#tictactree.level1),
?assertMatch(true, Tree3#tictactree.level1 == Tree3X#tictactree.level1), ?assertMatch(true, Tree3#tictactree.level1 == Tree3X#tictactree.level1),
?assertMatch(true, Tree3XA#tictactree.level1 == Tree3XA#tictactree.level1), ?assertMatch(true, Tree3XA#tictactree.level1 == Tree3XA#tictactree.level1),
SC = Tree0#tictactree.segment_count, SC = Tree0#tictactree.segment_count,
GetSegFun =
fun(TK) ->
get_segment(erlang:phash2(term_to_binary(TK)), SC)
end,
DL0 = find_dirtyleaves(Tree1, Tree0), DL0 = find_dirtyleaves(Tree1, Tree0),
?assertMatch(true, lists:member(get_segment({o, "B1", "K1", null}, SC), DL0)), ?assertMatch(true, lists:member(GetSegFun(K1), DL0)),
DL1 = find_dirtyleaves(Tree3, Tree1), DL1 = find_dirtyleaves(Tree3, Tree1),
?assertMatch(true, lists:member(get_segment({o, "B1", "K2", null}, SC), DL1)), ?assertMatch(true, lists:member(GetSegFun(K2), DL1)),
?assertMatch(true, lists:member(get_segment({o, "B1", "K3", null}, SC), DL1)), ?assertMatch(true, lists:member(GetSegFun(K3), DL1)),
?assertMatch(false, lists:member(get_segment({o, "B1", "K1", null}, SC), DL1)), ?assertMatch(false, lists:member(GetSegFun(K1), DL1)),
% Export and import tree to confirm no difference % Export and import tree to confirm no difference
ExpTree3 = export_tree(Tree3), ExpTree3 = export_tree(Tree3),
@ -416,24 +448,24 @@ merge_bysize_xlarge_test2() ->
merge_test_withsize(xlarge). merge_test_withsize(xlarge).
merge_test_withsize(Size) -> merge_test_withsize(Size) ->
HashFun = fun(_K, V) -> erlang:phash2(V) end, BinFun = fun(K, V) -> {term_to_binary(K), term_to_binary(V)} end,
TreeX0 = new_tree(0, Size), TreeX0 = new_tree(0, Size),
TreeX1 = add_kv(TreeX0, {o, "B1", "X1", null}, {caine, 1}, HashFun), TreeX1 = add_kv(TreeX0, {o, "B1", "X1", null}, {caine, 1}, BinFun),
TreeX2 = add_kv(TreeX1, {o, "B1", "X2", null}, {caine, 2}, HashFun), TreeX2 = add_kv(TreeX1, {o, "B1", "X2", null}, {caine, 2}, BinFun),
TreeX3 = add_kv(TreeX2, {o, "B1", "X3", null}, {caine, 3}, HashFun), TreeX3 = add_kv(TreeX2, {o, "B1", "X3", null}, {caine, 3}, BinFun),
TreeX4 = add_kv(TreeX3, {o, "B1", "X3", null}, {caine, 4}, HashFun), TreeX4 = add_kv(TreeX3, {o, "B1", "X3", null}, {caine, 4}, BinFun),
TreeY0 = new_tree(0, Size), TreeY0 = new_tree(0, Size),
TreeY1 = add_kv(TreeY0, {o, "B1", "Y1", null}, {caine, 101}, HashFun), TreeY1 = add_kv(TreeY0, {o, "B1", "Y1", null}, {caine, 101}, BinFun),
TreeY2 = add_kv(TreeY1, {o, "B1", "Y2", null}, {caine, 102}, HashFun), TreeY2 = add_kv(TreeY1, {o, "B1", "Y2", null}, {caine, 102}, BinFun),
TreeY3 = add_kv(TreeY2, {o, "B1", "Y3", null}, {caine, 103}, HashFun), TreeY3 = add_kv(TreeY2, {o, "B1", "Y3", null}, {caine, 103}, BinFun),
TreeY4 = add_kv(TreeY3, {o, "B1", "Y3", null}, {caine, 104}, HashFun), TreeY4 = add_kv(TreeY3, {o, "B1", "Y3", null}, {caine, 104}, BinFun),
TreeZ1 = add_kv(TreeX4, {o, "B1", "Y1", null}, {caine, 101}, HashFun), TreeZ1 = add_kv(TreeX4, {o, "B1", "Y1", null}, {caine, 101}, BinFun),
TreeZ2 = add_kv(TreeZ1, {o, "B1", "Y2", null}, {caine, 102}, HashFun), TreeZ2 = add_kv(TreeZ1, {o, "B1", "Y2", null}, {caine, 102}, BinFun),
TreeZ3 = add_kv(TreeZ2, {o, "B1", "Y3", null}, {caine, 103}, HashFun), TreeZ3 = add_kv(TreeZ2, {o, "B1", "Y3", null}, {caine, 103}, BinFun),
TreeZ4 = add_kv(TreeZ3, {o, "B1", "Y3", null}, {caine, 104}, HashFun), TreeZ4 = add_kv(TreeZ3, {o, "B1", "Y3", null}, {caine, 104}, BinFun),
TreeM0 = merge_trees(TreeX4, TreeY4), TreeM0 = merge_trees(TreeX4, TreeY4),
checktree(TreeM0), checktree(TreeM0),
@ -443,6 +475,10 @@ merge_test_withsize(Size) ->
checktree(TreeM1), checktree(TreeM1),
?assertMatch(false, TreeM1#tictactree.level1 == TreeZ4#tictactree.level1). ?assertMatch(false, TreeM1#tictactree.level1 == TreeZ4#tictactree.level1).
exportable_test() ->
{Int1, Int2} = tictac_hash(<<"key">>, <<"value">>, true),
?assertMatch({true, true}, {is_integer(Int1), is_integer(Int2)}).
-endif. -endif.

57
test/end_to_end/tictac_SUITE.erl
View file

@ -114,19 +114,16 @@ many_put_compare(_Config) ->
% Now run the same query by putting the tree-building responsibility onto % Now run the same query by putting the tree-building responsibility onto
% the fold_objects_fun % the fold_objects_fun
ApplyHash = ExtractClockFun =
fun(HashFun) -> fun(Key, Value) ->
fun(_Key, Value) -> {proxy_object, HeadBin, _Size, _FetchFun} = binary_to_term(Value),
{proxy_object, HeadBin, _Size, _FetchFun} = binary_to_term(Value), <<?MAGIC:8/integer, ?V1_VERS:8/integer, VclockLen:32/integer,
<<?MAGIC:8/integer, ?V1_VERS:8/integer, VclockLen:32/integer, VclockBin:VclockLen/binary, _Rest/binary>> = HeadBin,
Rest/binary>> = HeadBin, {Key, lists:sort(binary_to_term(VclockBin))}
<<VclockBin:VclockLen/binary, _NotNeeded/binary>> = Rest, end,
HashFun(lists:sort(binary_to_term(VclockBin)))
end
end,
FoldObjectsFun = FoldObjectsFun =
fun(_Bucket, Key, Value, Acc) -> fun(_Bucket, Key, Value, Acc) ->
leveled_tictac:add_kv(Acc, Key, Value, ApplyHash(fun erlang:phash2/1)) leveled_tictac:add_kv(Acc, Key, Value, ExtractClockFun, false)
end, end,
FoldQ0 = {foldheads_bybucket, FoldQ0 = {foldheads_bybucket,
@ -157,22 +154,25 @@ many_put_compare(_Config) ->
[timer:now_diff(os:timestamp(), SWB1Obj)]), [timer:now_diff(os:timestamp(), SWB1Obj)]),
true = length(leveled_tictac:find_dirtyleaves(TreeA, TreeAObj1)) == 0, true = length(leveled_tictac:find_dirtyleaves(TreeA, TreeAObj1)) == 0,
% AAE trees within riak are based on a sha of the vector clock. So to % For an exportable comparison, want hash to be based on something not
% compare with an AAE tree we need to compare outputs when we're hashing % coupled to erlang language - so use exportable query
% a hash AltExtractFun =
AltHashFun = fun(K, V) ->
fun(Term) -> {proxy_object, HeadBin, _Size, _FetchFun} = binary_to_term(V),
erlang:phash2(crypto:hash(sha, term_to_binary(Term))) <<?MAGIC:8/integer, ?V1_VERS:8/integer, VclockLen:32/integer,
VclockBin:VclockLen/binary, _Rest/binary>> = HeadBin,
{term_to_binary(K), VclockBin}
end, end,
AltFoldObjectsFun = AltFoldObjectsFun =
fun(_Bucket, Key, Value, Acc) -> fun(_Bucket, Key, Value, Acc) ->
leveled_tictac:add_kv(Acc, Key, Value, ApplyHash(AltHashFun)) leveled_tictac:add_kv(Acc, Key, Value, AltExtractFun, true)
end, end,
AltFoldQ0 = {foldheads_bybucket, AltFoldQ0 = {foldheads_bybucket,
o_rkv, o_rkv,
"Bucket", "Bucket",
{AltFoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)}, {AltFoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)},
false, true}, false,
true},
{async, TreeAAltObjFolder0} = {async, TreeAAltObjFolder0} =
leveled_bookie:book_returnfolder(Bookie2, AltFoldQ0), leveled_bookie:book_returnfolder(Bookie2, AltFoldQ0),
SWB2Obj = os:timestamp(), SWB2Obj = os:timestamp(),
@ -187,15 +187,19 @@ many_put_compare(_Config) ->
io:format("Build tictac tree via object fold with no "++ io:format("Build tictac tree via object fold with no "++
"presence check and 200K objects and alt hash in ~w~n", "presence check and 200K objects and alt hash in ~w~n",
[timer:now_diff(os:timestamp(), SWB3Obj)]), [timer:now_diff(os:timestamp(), SWB3Obj)]),
true = DL_ExportFold =
length(leveled_tictac:find_dirtyleaves(TreeBAltObj, TreeAAltObj)) == 1, length(leveled_tictac:find_dirtyleaves(TreeBAltObj, TreeAAltObj)),
io:format("Found dirty leaves with exportable comparison of ~w~n",
[DL_ExportFold]),
true = DL_ExportFold == 1,
%% Finding differing keys %% Finding differing keys
FoldKeysFun = FoldKeysFun =
fun(SegListToFind) -> fun(SegListToFind) ->
fun(_B, K, Acc) -> fun(_B, K, Acc) ->
Seg = leveled_tictac:get_segment(K, SegmentCount), Seg =
leveled_tictac:get_segment(erlang:phash2(K), SegmentCount),
case lists:member(Seg, SegListToFind) of case lists:member(Seg, SegListToFind) of
true -> true ->
[K|Acc]; [K|Acc];
@ -469,7 +473,8 @@ index_compare(_Config) ->
FoldKeysIndexQFun = FoldKeysIndexQFun =
fun(_Bucket, {Term, Key}, Acc) -> fun(_Bucket, {Term, Key}, Acc) ->
Seg = leveled_tictac:get_segment(Key, SegmentCount), Seg =
leveled_tictac:get_segment(erlang:phash2(Key), SegmentCount),
case lists:member(Seg, DL3_0) of case lists:member(Seg, DL3_0) of
true -> true ->
[{Term, Key}|Acc]; [{Term, Key}|Acc];