Expanded test

ct testing of tictac trees now compares between differently partitioned
stores.

test/end_to_end/tictac_SUITE.erl

@@ -12,7 +12,14 @@ all() -> [
 
 
 many_put_compare(_Config) ->
+    % Test requires multiple different databases, so want to mount them all
+    % on individual file paths
     RootPathA = testutil:reset_filestructure("testA"),
+    RootPathB = testutil:reset_filestructure("testB"),
+    RootPathC = testutil:reset_filestructure("testC"),
+    RootPathD = testutil:reset_filestructure("testD"),
+    
+    % Start the first database, load a test object, close it, start it again
     StartOpts1 = [{root_path, RootPathA},
                     {max_pencillercachesize, 16000},
                     {sync_strategy, riak_sync}],
@@ -32,6 +39,11 @@ many_put_compare(_Config) ->
                     {sync_strategy, testutil:sync_strategy()}],
     {ok, Bookie2} = leveled_bookie:book_start(StartOpts2),
     testutil:check_forobject(Bookie2, TestObject),
+    
+    % Generate 200K objects to be sued within the test, and load them into
+    % the first store (outputting the generated objects as a list of lists)
+    % to be used elsewhere
+    
     GenList = [2, 20002, 40002, 60002, 80002,
                 100002, 120002, 140002, 160002, 180002],
     CLs = testutil:load_objects(20000,
@@ -41,7 +53,9 @@ many_put_compare(_Config) ->
                                 fun testutil:generate_smallobjects/2,
                                 20000),
     
-    RootPathB = testutil:reset_filestructure("testB"),
+    % Start a new store, and load the same objects (except fot the original
+    % test object) into this store
+    
     StartOpts3 = [{root_path, RootPathB},
                     {max_journalsize, 200000000},
                     {max_pencillercachesize, 16000},
@@ -49,6 +63,9 @@ many_put_compare(_Config) ->
     {ok, Bookie3} = leveled_bookie:book_start(StartOpts3),
     lists:foreach(fun(ObjL) -> testutil:riakload(Bookie3, ObjL) end, CLs),
     
+    % Now run a tictac query against both stores to see th extent to which
+    % state between stores is consistent
+    
     TicTacQ = {tictactree_obj,
                 {o_rkv, "Bucket", null, null, false},
                 fun(_B, _K) -> accumulate end},
@@ -63,20 +80,23 @@ many_put_compare(_Config) ->
     io:format("Build tictac tree with 200K objects in ~w~n",
                 [timer:now_diff(os:timestamp(), SWB0)]),
     SWC0 = os:timestamp(),
-    SegList = leveled_tictac:find_dirtyleaves(TreeA, TreeB),
+    SegList0 = leveled_tictac:find_dirtyleaves(TreeA, TreeB),
     io:format("Compare tictac trees with 200K objects  in ~w~n",
                 [timer:now_diff(os:timestamp(), SWC0)]),
     io:format("Tree comparison shows ~w different leaves~n",
-                [length(SegList)]),
+                [length(SegList0)]),
     AltList = leveled_tictac:find_dirtyleaves(TreeA, 
                                                 leveled_tictac:new_tree(0)),
     io:format("Tree comparison shows ~w altered leaves~n",
                 [length(AltList)]),
-    true = length(SegList) == 1, 
+    true = length(SegList0) == 1, 
     % only the test object should be different
     true = length(AltList) > 100000, 
     % check there are a significant number fo differences from empty
     
+    % Now remove the object which represents the difference between these
+    % stores and confirm that the tictac trees will now match
+    
     testutil:book_riakdelete(Bookie2, B1, K1, []),
     {async, TreeAFolder0} = leveled_bookie:book_returnfolder(Bookie2, TicTacQ),
     SWA1 = os:timestamp(),
@@ -84,9 +104,78 @@ many_put_compare(_Config) ->
     io:format("Build tictac tree with 200K objects in ~w~n",
                 [timer:now_diff(os:timestamp(), SWA1)]),
 
-    SegList0 = leveled_tictac:find_dirtyleaves(TreeA0, TreeB),
+    SegList1 = leveled_tictac:find_dirtyleaves(TreeA0, TreeB),
-    true = length(SegList0) == 0,
+    io:format("Tree comparison following delete shows ~w different leaves~n",
+                [length(SegList1)]),
+    true = length(SegList1) == 0,
     % Removed test object so tictac trees should match
+    
+    ok = testutil:book_riakput(Bookie3, TestObject, TestSpec),
+    {async, TreeBFolder0} = leveled_bookie:book_returnfolder(Bookie3, TicTacQ),
+    SWB1 = os:timestamp(),
+    TreeB0 = TreeBFolder0(),
+    io:format("Build tictac tree with 200K objects in ~w~n",
+                [timer:now_diff(os:timestamp(), SWB1)]),
+    SegList2 = leveled_tictac:find_dirtyleaves(TreeA0, TreeB0),
+    true = SegList2 == SegList0,
+    % There is an identical difference now the difference is on Bookie3 not
+    % Bookie 2 (compared to it being in Bookie2 not Bookie3)
 
+    ok = leveled_bookie:book_close(Bookie3),
+    
+    % Replace Bookie 3 with two stores Bookie 4 and Bookie 5 where the ojects
+    % have been randomly split between the stores
+    
+    StartOpts4 = [{root_path, RootPathC},
+                    {max_journalsize, 200000000},
+                    {max_pencillercachesize, 24000},
+                    {sync_strategy, testutil:sync_strategy()}],
+    {ok, Bookie4} = leveled_bookie:book_start(StartOpts4),
+    StartOpts5 = [{root_path, RootPathD},
+                    {max_journalsize, 200000000},
+                    {max_pencillercachesize, 24000},
+                    {sync_strategy, testutil:sync_strategy()}],
+    {ok, Bookie5} = leveled_bookie:book_start(StartOpts5),
+    
+    SplitFun =
+        fun(Obj) ->
+            case erlang:phash2(Obj) rem 2 of
+                0 ->
+                    true;
+                1 ->
+                    false
+            end
+        end,
+    lists:foreach(fun(ObjL) ->
+                        {ObjLA, ObjLB} = lists:partition(SplitFun, ObjL),
+                        testutil:riakload(Bookie4, ObjLA),
+                        testutil:riakload(Bookie5, ObjLB)
+                    end,
+                    CLs),
+    
+    % query both the stores, then merge the trees - the result should be the
+    % same as the result from the tree created aginst the store with both
+    % partitions
+    
+    {async, TreeC0Folder} = leveled_bookie:book_returnfolder(Bookie4, TicTacQ),
+    {async, TreeC1Folder} = leveled_bookie:book_returnfolder(Bookie5, TicTacQ),
+    SWD0 = os:timestamp(),
+    TreeC0 = TreeC0Folder(),
+    io:format("Build tictac tree with 100K objects in ~w~n",
+                [timer:now_diff(os:timestamp(), SWD0)]),
+    SWD1 = os:timestamp(),
+    TreeC1 = TreeC1Folder(),
+    io:format("Build tictac tree with 100K objects in ~w~n",
+                [timer:now_diff(os:timestamp(), SWD1)]),
+    
+    TreeC2 = leveled_tictac:merge_trees(TreeC0, TreeC1),
+    SegList3 = leveled_tictac:find_dirtyleaves(TreeC2, TreeB),
+    io:format("Tree comparison following delete shows ~w different leaves~n",
+                [length(SegList3)]),
+    true = length(SegList3) == 0,
+    
+    
     ok = leveled_bookie:book_close(Bookie2),
-    ok = leveled_bookie:book_close(Bookie3).
+    ok = leveled_bookie:book_close(Bookie4),
+    ok = leveled_bookie:book_close(Bookie5).
+