From 9f97c82d0dedd2e8c418291c8921362e3d6654dc Mon Sep 17 00:00:00 2001
From: Martin Sumner <martin.sumner@adaptip.co.uk>
Date: Wed, 16 Aug 2017 16:58:38 +0100
Subject: [PATCH] Add import/export support

Also fix for fold_heads unit tests to reflect new booleans required by changes to support there use in MapFolds
---
 docs/ANTI_ENTROPY.md   |  2 +-
 src/leveled_bookie.erl | 22 ++++++++++++++-----
 src/leveled_tictac.erl | 49 ++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 65 insertions(+), 8 deletions(-)

diff --git a/docs/ANTI_ENTROPY.md b/docs/ANTI_ENTROPY.md
index e28e9c8..2cfb5b0 100644
--- a/docs/ANTI_ENTROPY.md
+++ b/docs/ANTI_ENTROPY.md
@@ -386,4 +386,4 @@ The AAE hashtree lock situation is complex, but can be summarised as:
 
 ### Phase 2
 
-tbc
+For phase 2 the issues of how to efficiently manage AAE queries in leveldb and bitcask will be put to one side, and the focus will be on getting an effective solution up and running with leveled.
diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl
index 9fd3b79..7a78178 100644
--- a/src/leveled_bookie.erl
+++ b/src/leveled_bookie.erl
@@ -1334,7 +1334,7 @@ accumulate_objects(FoldObjectsFun, InkerClone, Tag, DeferredFetch) ->
                                     ProxyObj = make_proxy_object(LK, JK,
                                                                   MD, V,
                                                                   InkerClone),
-                                    FoldObjectsFun(B, K,ProxyObj, Acc);
+                                    FoldObjectsFun(B, K, ProxyObj, Acc);
                                 missing ->
                                     Acc
                             end;
@@ -1784,7 +1784,11 @@ foldobjects_vs_hashtree_test() ->
 
     {async, HTFolder3} =
         book_returnfolder(Bookie1,
-                            {foldheads_allkeys, ?STD_TAG, FoldHeadsFun}),
+                            {foldheads_allkeys, 
+                                ?STD_TAG, 
+                                FoldHeadsFun, 
+                                true, 
+                                true}),
     KeyHashList3 = HTFolder3(),
     ?assertMatch(KeyHashList1, lists:usort(KeyHashList3)),
 
@@ -1800,7 +1804,11 @@ foldobjects_vs_hashtree_test() ->
 
     {async, HTFolder4} =
         book_returnfolder(Bookie1,
-                            {foldheads_allkeys, ?STD_TAG, FoldHeadsFun2}),
+                            {foldheads_allkeys, 
+                                ?STD_TAG, 
+                                FoldHeadsFun2,
+                                false, 
+                                false}),
     KeyHashList4 = HTFolder4(),
     ?assertMatch(KeyHashList1, lists:usort(KeyHashList4)),
 
@@ -1862,14 +1870,18 @@ foldobjects_vs_foldheads_bybucket_test() ->
                             {foldheads_bybucket,
                                 ?STD_TAG,
                                 "BucketA",
-                                FoldHeadsFun}),
+                                FoldHeadsFun,
+                                true, 
+                                true}),
     KeyHashList2A = HTFolder2A(),
     {async, HTFolder2B} =
         book_returnfolder(Bookie1,
                             {foldheads_bybucket,
                                 ?STD_TAG,
                                 "BucketB",
-                                FoldHeadsFun}),
+                                FoldHeadsFun,
+                                false, 
+                                false}),
     KeyHashList2B = HTFolder2B(),
     ?assertMatch(true,
                     lists:usort(KeyHashList1A) == lists:usort(KeyHashList2A)),
diff --git a/src/leveled_tictac.erl b/src/leveled_tictac.erl
index e7ced1e..452520f 100644
--- a/src/leveled_tictac.erl
+++ b/src/leveled_tictac.erl
@@ -64,7 +64,8 @@
             fetch_leaves/2,
             merge_trees/2,
             get_segment/2,
-            tictac_hash/2
+            tictac_hash/2,
+            export_tree/1
         ]).
 
 
@@ -112,6 +113,45 @@ new_tree(TreeID, Size) ->
                     level1 = Lv1Init,
                     level2 = Lv2Init}.
 
+-spec export_tree(tictactree()) -> list().
+%% @doc
+%% Export the tree into a tuple list, with the level1 binary, and then for 
+%% level2 {branchID, binary()}
+export_tree(Tree) ->
+    L2 = 
+        lists:foldl(fun(X, L2Acc) ->
+                            [{X, array:get(X, Tree#tictactree.level2)}|L2Acc]
+                        end,
+                    [],
+                    lists:seq(0, Tree#tictactree.width - 1)),
+    [{level1, Tree#tictactree.level1}, {level2, lists:reverse(L2)}].
+
+-spec import_tree(list()) -> tictactree().
+%% @doc
+%% Reverse the export process
+import_tree(ExportedTree) ->
+    [{level1, L1Bin}, {level2, L2List}] = ExportedTree,
+    Sizes = [{small, element(2, ?SMALL)}, 
+                {medium, element(2, ?MEDIUM)}, 
+                {large, element(2, ?LARGE)}, 
+                {xlarge, element(2, ?XLARGE)}],
+    Width = byte_size(L1Bin) div ?HASH_SIZE,
+    {Size, Width} = lists:keyfind(Width, 2, Sizes),
+    {BitWidth, Width, SegmentCount} = get_size(Size),
+    Lv2Init = array:new([{size, Width}]),
+    Lv2 = lists:foldl(fun({X, L2SegBin}, L2Array) ->
+                                array:set(X, L2SegBin, L2Array)
+                            end,
+                        Lv2Init,
+                        L2List),
+    #tictactree{treeID = import,
+                    size = Size,
+                    width = Width,
+                    bitwidth = BitWidth,
+                    segment_count = SegmentCount,
+                    level1 = L1Bin,
+                    level2 = Lv2}.
+
 -spec add_kv(tictactree(), tuple(), tuple(), fun()) -> tictactree().
 %% @doc
 %% Add a Key and value to a tictactree using the HashFun to calculate the Hash
@@ -345,7 +385,12 @@ simple_test_withsize(Size) ->
     DL1 = find_dirtyleaves(Tree3, Tree1),
     ?assertMatch(true, lists:member(get_segment({o, "B1", "K2", null}, SC), DL1)),
     ?assertMatch(true, lists:member(get_segment({o, "B1", "K3", null}, SC), DL1)),
-    ?assertMatch(false, lists:member(get_segment({o, "B1", "K1", null}, SC), DL1)).
+    ?assertMatch(false, lists:member(get_segment({o, "B1", "K1", null}, SC), DL1)),
+    
+    % Export and import tree to confirm no difference
+    ExpTree3 = export_tree(Tree3),
+    ImpTree3 = import_tree(ExpTree3),
+    ?assertMatch(DL1, find_dirtyleaves(ImpTree3, Tree1)).
 
 merge_bysize_small_test() ->
     merge_test_withsize(small).