Expand fold objects support

Fold over bucket and fold over index added

test/end_to_end/iterator_SUITE.erl

@@ -6,17 +6,18 @@
 -define(KEY_ONLY, {false, undefined}).
 
 -export([all/0]).
--export([simple_load_with2i/1,
+-export([small_load_with2i/1,
             query_count/1,
             rotating_objects/1]).
 
 all() -> [
-            simple_load_with2i,
+            small_load_with2i,
             query_count,
-            rotating_objects].
+            rotating_objects
+            ].
 
 
-simple_load_with2i(_Config) ->
+small_load_with2i(_Config) ->
     RootPath = testutil:reset_filestructure(),
     StartOpts1 = [{root_path, RootPath},
                     {max_journalsize, 50000000}],
@@ -37,6 +38,42 @@ simple_load_with2i(_Config) ->
     ChkList1 = lists:sublist(lists:sort(ObjL1), 100),
     testutil:check_forlist(Bookie1, ChkList1),
     testutil:check_forobject(Bookie1, TestObject),
+    
+    %% Delete the objects from the ChkList removing the indexes
+    lists:foreach(fun({_RN, Obj, Spc}) ->
+                        DSpc = lists:map(fun({add, F, T}) -> {remove, F, T}
+                                                                end,
+                                            Spc),
+                        {B, K} = leveled_codec:riakto_keydetails(Obj),
+                        leveled_bookie:book_riakdelete(Bookie1, B, K, DSpc)
+                        end,
+                    ChkList1),
+    %% Get the Buckets Keys and Hashes for the whole bucket
+    FoldObjectsFun = fun(B, K, V, Acc) -> [{B, K, testutil:riak_hash(V)}|Acc]
+                                            end,
+    {async, HTreeF1} = leveled_bookie:book_returnfolder(Bookie1,
+                                                        {foldobjects_allkeys,
+                                                            ?RIAK_TAG,
+                                                            FoldObjectsFun}),
+    KeyHashList1 = HTreeF1(),
+    {async, HTreeF2} = leveled_bookie:book_returnfolder(Bookie1,
+                                                        {foldobjects_bybucket,
+                                                            ?RIAK_TAG,
+                                                            "Bucket",
+                                                            FoldObjectsFun}),
+    KeyHashList2 = HTreeF2(),
+    {async, HTreeF3} = leveled_bookie:book_returnfolder(Bookie1,
+                                                        {foldobjects_byindex,
+                                                            ?RIAK_TAG,
+                                                            "Bucket",
+                                                            {"idx1_bin",
+                                                                "#", "~"},
+                                                            FoldObjectsFun}),
+    KeyHashList3 = HTreeF3(),
+    true = 9901 == length(KeyHashList1), % also includes the test object
+    true = 9900 == length(KeyHashList2),
+    true = 9900 == length(KeyHashList3),
+    
     ok = leveled_bookie:book_close(Bookie1),
     testutil:reset_filestructure().
 

test/end_to_end/recovery_SUITE.erl

@@ -103,7 +103,8 @@ aae_bustedjournal(_Config) ->
     % Will need to remove the file or corrupt the hashtree to get presence to
     % fail
     
-    FoldObjectsFun = fun(B, K, V, Acc) -> [{B, K, riak_hash(V)}|Acc] end,
+    FoldObjectsFun = fun(B, K, V, Acc) -> [{B, K, testutil:riak_hash(V)}|Acc]
+                                            end,
     SW = os:timestamp(),
     {async, HashTreeF3} = leveled_bookie:book_returnfolder(Bookie2,
                                                             {foldobjects_allkeys,
@@ -190,15 +191,6 @@ aae_bustedjournal(_Config) ->
     testutil:reset_filestructure().
 
 
-riak_hash(Obj=#r_object{}) ->
-    Vclock = vclock(Obj),
-    UpdObj = set_vclock(Obj, lists:sort(Vclock)),
-    erlang:phash2(term_to_binary(UpdObj)).
-
-set_vclock(Object=#r_object{}, VClock) -> Object#r_object{vclock=VClock}.
-vclock(#r_object{vclock=VClock}) -> VClock.
-
-
 journal_compaction_bustedjournal(_Config) ->
     % Simply confirms that none of this causes a crash
     RootPath = testutil:reset_filestructure(),

test/end_to_end/testutil.erl

@@ -32,7 +32,8 @@
             corrupt_journal/5,
             restore_file/2,
             restore_topending/2,
-            find_journals/1]).
+            find_journals/1,
+            riak_hash/1]).
 
 -define(RETURN_TERMS, {true, undefined}).
 
@@ -423,4 +424,13 @@ find_journals(RootPath) ->
                                             end,
                                 [],
                                 FNsA_J),
-    CDBFiles.
+    CDBFiles.
+
+
+riak_hash(Obj=#r_object{}) ->
+    Vclock = vclock(Obj),
+    UpdObj = set_vclock(Obj, lists:sort(Vclock)),
+    erlang:phash2(term_to_binary(UpdObj)).
+
+set_vclock(Object=#r_object{}, VClock) -> Object#r_object{vclock=VClock}.
+vclock(#r_object{vclock=VClock}) -> VClock.