From 9f97c82d0dedd2e8c418291c8921362e3d6654dc Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 16 Aug 2017 16:58:38 +0100 Subject: [PATCH 1/4] Add import/export support Also fix for fold_heads unit tests to reflect new booleans required by changes to support there use in MapFolds --- docs/ANTI_ENTROPY.md | 2 +- src/leveled_bookie.erl | 22 ++++++++++++++----- src/leveled_tictac.erl | 49 ++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/docs/ANTI_ENTROPY.md b/docs/ANTI_ENTROPY.md index e28e9c8..2cfb5b0 100644 --- a/docs/ANTI_ENTROPY.md +++ b/docs/ANTI_ENTROPY.md @@ -386,4 +386,4 @@ The AAE hashtree lock situation is complex, but can be summarised as: ### Phase 2 -tbc +For phase 2 the issues of how to efficiently manage AAE queries in leveldb and bitcask will be put to one side, and the focus will be on getting an effective solution up and running with leveled. diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 9fd3b79..7a78178 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -1334,7 +1334,7 @@ accumulate_objects(FoldObjectsFun, InkerClone, Tag, DeferredFetch) -> ProxyObj = make_proxy_object(LK, JK, MD, V, InkerClone), - FoldObjectsFun(B, K,ProxyObj, Acc); + FoldObjectsFun(B, K, ProxyObj, Acc); missing -> Acc end; @@ -1784,7 +1784,11 @@ foldobjects_vs_hashtree_test() -> {async, HTFolder3} = book_returnfolder(Bookie1, - {foldheads_allkeys, ?STD_TAG, FoldHeadsFun}), + {foldheads_allkeys, + ?STD_TAG, + FoldHeadsFun, + true, + true}), KeyHashList3 = HTFolder3(), ?assertMatch(KeyHashList1, lists:usort(KeyHashList3)), @@ -1800,7 +1804,11 @@ foldobjects_vs_hashtree_test() -> {async, HTFolder4} = book_returnfolder(Bookie1, - {foldheads_allkeys, ?STD_TAG, FoldHeadsFun2}), + {foldheads_allkeys, + ?STD_TAG, + FoldHeadsFun2, + false, + false}), KeyHashList4 = HTFolder4(), ?assertMatch(KeyHashList1, lists:usort(KeyHashList4)), @@ -1862,14 +1870,18 @@ foldobjects_vs_foldheads_bybucket_test() -> {foldheads_bybucket, ?STD_TAG, "BucketA", - FoldHeadsFun}), + FoldHeadsFun, + true, + true}), KeyHashList2A = HTFolder2A(), {async, HTFolder2B} = book_returnfolder(Bookie1, {foldheads_bybucket, ?STD_TAG, "BucketB", - FoldHeadsFun}), + FoldHeadsFun, + false, + false}), KeyHashList2B = HTFolder2B(), ?assertMatch(true, lists:usort(KeyHashList1A) == lists:usort(KeyHashList2A)), diff --git a/src/leveled_tictac.erl b/src/leveled_tictac.erl index e7ced1e..452520f 100644 --- a/src/leveled_tictac.erl +++ b/src/leveled_tictac.erl @@ -64,7 +64,8 @@ fetch_leaves/2, merge_trees/2, get_segment/2, - tictac_hash/2 + tictac_hash/2, + export_tree/1 ]). @@ -112,6 +113,45 @@ new_tree(TreeID, Size) -> level1 = Lv1Init, level2 = Lv2Init}. +-spec export_tree(tictactree()) -> list(). +%% @doc +%% Export the tree into a tuple list, with the level1 binary, and then for +%% level2 {branchID, binary()} +export_tree(Tree) -> + L2 = + lists:foldl(fun(X, L2Acc) -> + [{X, array:get(X, Tree#tictactree.level2)}|L2Acc] + end, + [], + lists:seq(0, Tree#tictactree.width - 1)), + [{level1, Tree#tictactree.level1}, {level2, lists:reverse(L2)}]. + +-spec import_tree(list()) -> tictactree(). +%% @doc +%% Reverse the export process +import_tree(ExportedTree) -> + [{level1, L1Bin}, {level2, L2List}] = ExportedTree, + Sizes = [{small, element(2, ?SMALL)}, + {medium, element(2, ?MEDIUM)}, + {large, element(2, ?LARGE)}, + {xlarge, element(2, ?XLARGE)}], + Width = byte_size(L1Bin) div ?HASH_SIZE, + {Size, Width} = lists:keyfind(Width, 2, Sizes), + {BitWidth, Width, SegmentCount} = get_size(Size), + Lv2Init = array:new([{size, Width}]), + Lv2 = lists:foldl(fun({X, L2SegBin}, L2Array) -> + array:set(X, L2SegBin, L2Array) + end, + Lv2Init, + L2List), + #tictactree{treeID = import, + size = Size, + width = Width, + bitwidth = BitWidth, + segment_count = SegmentCount, + level1 = L1Bin, + level2 = Lv2}. + -spec add_kv(tictactree(), tuple(), tuple(), fun()) -> tictactree(). %% @doc %% Add a Key and value to a tictactree using the HashFun to calculate the Hash @@ -345,7 +385,12 @@ simple_test_withsize(Size) -> DL1 = find_dirtyleaves(Tree3, Tree1), ?assertMatch(true, lists:member(get_segment({o, "B1", "K2", null}, SC), DL1)), ?assertMatch(true, lists:member(get_segment({o, "B1", "K3", null}, SC), DL1)), - ?assertMatch(false, lists:member(get_segment({o, "B1", "K1", null}, SC), DL1)). + ?assertMatch(false, lists:member(get_segment({o, "B1", "K1", null}, SC), DL1)), + + % Export and import tree to confirm no difference + ExpTree3 = export_tree(Tree3), + ImpTree3 = import_tree(ExpTree3), + ?assertMatch(DL1, find_dirtyleaves(ImpTree3, Tree1)). merge_bysize_small_test() -> merge_test_withsize(small). From d98534b8f33ca132964de0788f97d6f59edc8369 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 13 Sep 2017 21:56:28 +0100 Subject: [PATCH 2/4] Add struct to help with mochijson2 Try and make this more mochijson2 friendly --- src/leveled_tictac.erl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/leveled_tictac.erl b/src/leveled_tictac.erl index 452520f..94bd5fc 100644 --- a/src/leveled_tictac.erl +++ b/src/leveled_tictac.erl @@ -124,13 +124,19 @@ export_tree(Tree) -> end, [], lists:seq(0, Tree#tictactree.width - 1)), - [{level1, Tree#tictactree.level1}, {level2, lists:reverse(L2)}]. + {struct, + [{<<"level1">>, base64:encode_to_string(Tree#tictactree.level1)}, + {<<"level2">>, {struct, lists:reverse(L2)}} + ]}. -spec import_tree(list()) -> tictactree(). %% @doc %% Reverse the export process import_tree(ExportedTree) -> - [{level1, L1Bin}, {level2, L2List}] = ExportedTree, + {struct, + [{<<"level1">>, L1Base64}, + {<<"level2">>, {struct, L2List}}]} = ExportedTree, + L1Bin = base64:decode(L1Base64), Sizes = [{small, element(2, ?SMALL)}, {medium, element(2, ?MEDIUM)}, {large, element(2, ?LARGE)}, From d8ca0274f6cd48088526f9a6fab7db20fb302186 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 13 Sep 2017 22:02:44 +0100 Subject: [PATCH 3/4] Export the import --- src/leveled_tictac.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/leveled_tictac.erl b/src/leveled_tictac.erl index 94bd5fc..5e9fe0a 100644 --- a/src/leveled_tictac.erl +++ b/src/leveled_tictac.erl @@ -65,7 +65,8 @@ merge_trees/2, get_segment/2, tictac_hash/2, - export_tree/1 + export_tree/1, + import_tree/1 ]). From ed56ef17a1f6bef78428b047e8bb320bd9605f43 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 13 Sep 2017 23:45:48 +0100 Subject: [PATCH 4/4] Make export mochijson friendly --- src/leveled_tictac.erl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/leveled_tictac.erl b/src/leveled_tictac.erl index 5e9fe0a..ad420cc 100644 --- a/src/leveled_tictac.erl +++ b/src/leveled_tictac.erl @@ -121,7 +121,8 @@ new_tree(TreeID, Size) -> export_tree(Tree) -> L2 = lists:foldl(fun(X, L2Acc) -> - [{X, array:get(X, Tree#tictactree.level2)}|L2Acc] + [{integer_to_binary(X), + array:get(X, Tree#tictactree.level2)}|L2Acc] end, [], lists:seq(0, Tree#tictactree.width - 1)), @@ -146,11 +147,11 @@ import_tree(ExportedTree) -> {Size, Width} = lists:keyfind(Width, 2, Sizes), {BitWidth, Width, SegmentCount} = get_size(Size), Lv2Init = array:new([{size, Width}]), - Lv2 = lists:foldl(fun({X, L2SegBin}, L2Array) -> - array:set(X, L2SegBin, L2Array) - end, - Lv2Init, - L2List), + FoldFun = + fun({X, L2SegBin}, L2Array) -> + array:set(binary_to_integer(X), L2SegBin, L2Array) + end, + Lv2 = lists:foldl(FoldFun, Lv2Init, L2List), #tictactree{treeID = import, size = Size, width = Width,