diff --git a/docs/ANTI_ENTROPY.md b/docs/ANTI_ENTROPY.md index e28e9c8..2cfb5b0 100644 --- a/docs/ANTI_ENTROPY.md +++ b/docs/ANTI_ENTROPY.md @@ -386,4 +386,4 @@ The AAE hashtree lock situation is complex, but can be summarised as: ### Phase 2 -tbc +For phase 2 the issues of how to efficiently manage AAE queries in leveldb and bitcask will be put to one side, and the focus will be on getting an effective solution up and running with leveled. diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 9fd3b79..7a78178 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -1334,7 +1334,7 @@ accumulate_objects(FoldObjectsFun, InkerClone, Tag, DeferredFetch) -> ProxyObj = make_proxy_object(LK, JK, MD, V, InkerClone), - FoldObjectsFun(B, K,ProxyObj, Acc); + FoldObjectsFun(B, K, ProxyObj, Acc); missing -> Acc end; @@ -1784,7 +1784,11 @@ foldobjects_vs_hashtree_test() -> {async, HTFolder3} = book_returnfolder(Bookie1, - {foldheads_allkeys, ?STD_TAG, FoldHeadsFun}), + {foldheads_allkeys, + ?STD_TAG, + FoldHeadsFun, + true, + true}), KeyHashList3 = HTFolder3(), ?assertMatch(KeyHashList1, lists:usort(KeyHashList3)), @@ -1800,7 +1804,11 @@ foldobjects_vs_hashtree_test() -> {async, HTFolder4} = book_returnfolder(Bookie1, - {foldheads_allkeys, ?STD_TAG, FoldHeadsFun2}), + {foldheads_allkeys, + ?STD_TAG, + FoldHeadsFun2, + false, + false}), KeyHashList4 = HTFolder4(), ?assertMatch(KeyHashList1, lists:usort(KeyHashList4)), @@ -1862,14 +1870,18 @@ foldobjects_vs_foldheads_bybucket_test() -> {foldheads_bybucket, ?STD_TAG, "BucketA", - FoldHeadsFun}), + FoldHeadsFun, + true, + true}), KeyHashList2A = HTFolder2A(), {async, HTFolder2B} = book_returnfolder(Bookie1, {foldheads_bybucket, ?STD_TAG, "BucketB", - FoldHeadsFun}), + FoldHeadsFun, + false, + false}), KeyHashList2B = HTFolder2B(), ?assertMatch(true, lists:usort(KeyHashList1A) == lists:usort(KeyHashList2A)), diff --git a/src/leveled_tictac.erl b/src/leveled_tictac.erl index e7ced1e..452520f 100644 --- a/src/leveled_tictac.erl +++ b/src/leveled_tictac.erl @@ -64,7 +64,8 @@ fetch_leaves/2, merge_trees/2, get_segment/2, - tictac_hash/2 + tictac_hash/2, + export_tree/1 ]). @@ -112,6 +113,45 @@ new_tree(TreeID, Size) -> level1 = Lv1Init, level2 = Lv2Init}. +-spec export_tree(tictactree()) -> list(). +%% @doc +%% Export the tree into a tuple list, with the level1 binary, and then for +%% level2 {branchID, binary()} +export_tree(Tree) -> + L2 = + lists:foldl(fun(X, L2Acc) -> + [{X, array:get(X, Tree#tictactree.level2)}|L2Acc] + end, + [], + lists:seq(0, Tree#tictactree.width - 1)), + [{level1, Tree#tictactree.level1}, {level2, lists:reverse(L2)}]. + +-spec import_tree(list()) -> tictactree(). +%% @doc +%% Reverse the export process +import_tree(ExportedTree) -> + [{level1, L1Bin}, {level2, L2List}] = ExportedTree, + Sizes = [{small, element(2, ?SMALL)}, + {medium, element(2, ?MEDIUM)}, + {large, element(2, ?LARGE)}, + {xlarge, element(2, ?XLARGE)}], + Width = byte_size(L1Bin) div ?HASH_SIZE, + {Size, Width} = lists:keyfind(Width, 2, Sizes), + {BitWidth, Width, SegmentCount} = get_size(Size), + Lv2Init = array:new([{size, Width}]), + Lv2 = lists:foldl(fun({X, L2SegBin}, L2Array) -> + array:set(X, L2SegBin, L2Array) + end, + Lv2Init, + L2List), + #tictactree{treeID = import, + size = Size, + width = Width, + bitwidth = BitWidth, + segment_count = SegmentCount, + level1 = L1Bin, + level2 = Lv2}. + -spec add_kv(tictactree(), tuple(), tuple(), fun()) -> tictactree(). %% @doc %% Add a Key and value to a tictactree using the HashFun to calculate the Hash @@ -345,7 +385,12 @@ simple_test_withsize(Size) -> DL1 = find_dirtyleaves(Tree3, Tree1), ?assertMatch(true, lists:member(get_segment({o, "B1", "K2", null}, SC), DL1)), ?assertMatch(true, lists:member(get_segment({o, "B1", "K3", null}, SC), DL1)), - ?assertMatch(false, lists:member(get_segment({o, "B1", "K1", null}, SC), DL1)). + ?assertMatch(false, lists:member(get_segment({o, "B1", "K1", null}, SC), DL1)), + + % Export and import tree to confirm no difference + ExpTree3 = export_tree(Tree3), + ImpTree3 = import_tree(ExpTree3), + ?assertMatch(DL1, find_dirtyleaves(ImpTree3, Tree1)). merge_bysize_small_test() -> merge_test_withsize(small).