Add import/export support

Also fix for fold_heads unit tests to reflect new booleans required by changes to support there use in MapFolds

docs/ANTI_ENTROPY.md

@@ -386,4 +386,4 @@ The AAE hashtree lock situation is complex, but can be summarised as:
 
 ### Phase 2
 
-tbc
+For phase 2 the issues of how to efficiently manage AAE queries in leveldb and bitcask will be put to one side, and the focus will be on getting an effective solution up and running with leveled.

src/leveled_bookie.erl

@@ -1334,7 +1334,7 @@ accumulate_objects(FoldObjectsFun, InkerClone, Tag, DeferredFetch) ->
                                     ProxyObj = make_proxy_object(LK, JK,
                                                                   MD, V,
                                                                   InkerClone),
-                                    FoldObjectsFun(B, K,ProxyObj, Acc);
+                                    FoldObjectsFun(B, K, ProxyObj, Acc);
                                 missing ->
                                     Acc
                             end;
@@ -1784,7 +1784,11 @@ foldobjects_vs_hashtree_test() ->
 
     {async, HTFolder3} =
         book_returnfolder(Bookie1,
-                            {foldheads_allkeys, ?STD_TAG, FoldHeadsFun}),
+                            {foldheads_allkeys, 
+                                ?STD_TAG, 
+                                FoldHeadsFun, 
+                                true, 
+                                true}),
     KeyHashList3 = HTFolder3(),
     ?assertMatch(KeyHashList1, lists:usort(KeyHashList3)),
 
@@ -1800,7 +1804,11 @@ foldobjects_vs_hashtree_test() ->
 
     {async, HTFolder4} =
         book_returnfolder(Bookie1,
-                            {foldheads_allkeys, ?STD_TAG, FoldHeadsFun2}),
+                            {foldheads_allkeys, 
+                                ?STD_TAG, 
+                                FoldHeadsFun2,
+                                false, 
+                                false}),
     KeyHashList4 = HTFolder4(),
     ?assertMatch(KeyHashList1, lists:usort(KeyHashList4)),
 
@@ -1862,14 +1870,18 @@ foldobjects_vs_foldheads_bybucket_test() ->
                             {foldheads_bybucket,
                                 ?STD_TAG,
                                 "BucketA",
-                                FoldHeadsFun}),
+                                FoldHeadsFun,
+                                true, 
+                                true}),
     KeyHashList2A = HTFolder2A(),
     {async, HTFolder2B} =
         book_returnfolder(Bookie1,
                             {foldheads_bybucket,
                                 ?STD_TAG,
                                 "BucketB",
-                                FoldHeadsFun}),
+                                FoldHeadsFun,
+                                false, 
+                                false}),
     KeyHashList2B = HTFolder2B(),
     ?assertMatch(true,
                     lists:usort(KeyHashList1A) == lists:usort(KeyHashList2A)),

src/leveled_tictac.erl

@@ -64,7 +64,8 @@
             fetch_leaves/2,
             merge_trees/2,
             get_segment/2,
-            tictac_hash/2
+            tictac_hash/2,
+            export_tree/1
         ]).
 
 
@@ -112,6 +113,45 @@ new_tree(TreeID, Size) ->
                     level1 = Lv1Init,
                     level2 = Lv2Init}.
 
+-spec export_tree(tictactree()) -> list().
+%% @doc
+%% Export the tree into a tuple list, with the level1 binary, and then for 
+%% level2 {branchID, binary()}
+export_tree(Tree) ->
+    L2 = 
+        lists:foldl(fun(X, L2Acc) ->
+                            [{X, array:get(X, Tree#tictactree.level2)}|L2Acc]
+                        end,
+                    [],
+                    lists:seq(0, Tree#tictactree.width - 1)),
+    [{level1, Tree#tictactree.level1}, {level2, lists:reverse(L2)}].
+
+-spec import_tree(list()) -> tictactree().
+%% @doc
+%% Reverse the export process
+import_tree(ExportedTree) ->
+    [{level1, L1Bin}, {level2, L2List}] = ExportedTree,
+    Sizes = [{small, element(2, ?SMALL)}, 
+                {medium, element(2, ?MEDIUM)}, 
+                {large, element(2, ?LARGE)}, 
+                {xlarge, element(2, ?XLARGE)}],
+    Width = byte_size(L1Bin) div ?HASH_SIZE,
+    {Size, Width} = lists:keyfind(Width, 2, Sizes),
+    {BitWidth, Width, SegmentCount} = get_size(Size),
+    Lv2Init = array:new([{size, Width}]),
+    Lv2 = lists:foldl(fun({X, L2SegBin}, L2Array) ->
+                                array:set(X, L2SegBin, L2Array)
+                            end,
+                        Lv2Init,
+                        L2List),
+    #tictactree{treeID = import,
+                    size = Size,
+                    width = Width,
+                    bitwidth = BitWidth,
+                    segment_count = SegmentCount,
+                    level1 = L1Bin,
+                    level2 = Lv2}.
+
 -spec add_kv(tictactree(), tuple(), tuple(), fun()) -> tictactree().
 %% @doc
 %% Add a Key and value to a tictactree using the HashFun to calculate the Hash
@@ -345,7 +385,12 @@ simple_test_withsize(Size) ->
     DL1 = find_dirtyleaves(Tree3, Tree1),
     ?assertMatch(true, lists:member(get_segment({o, "B1", "K2", null}, SC), DL1)),
     ?assertMatch(true, lists:member(get_segment({o, "B1", "K3", null}, SC), DL1)),
-    ?assertMatch(false, lists:member(get_segment({o, "B1", "K1", null}, SC), DL1)).
+    ?assertMatch(false, lists:member(get_segment({o, "B1", "K1", null}, SC), DL1)),
+    
+    % Export and import tree to confirm no difference
+    ExpTree3 = export_tree(Tree3),
+    ImpTree3 = import_tree(ExpTree3),
+    ?assertMatch(DL1, find_dirtyleaves(ImpTree3, Tree1)).
 
 merge_bysize_small_test() ->
     merge_test_withsize(small).