Rotating object tests

Recent fixes have been made to problems associated with rapidly changing
objexts especially on re-opening of the bookie.  Test of rotating
objects from both an index query and a fetch perspective added to better
detect such issues in the future.

test/end_to_end/iterator_SUITE.erl

@@ -4,13 +4,16 @@
 -include("include/leveled.hrl").
 
 -define(KEY_ONLY, {false, undefined}).
+-define(RETURN_TERMS, {true, undefined}).
 
 -export([all/0]).
 -export([simple_load_with2i/1,
-            simple_querycount/1]).
+            simple_querycount/1,
+            rotating_objects/1]).
 
 all() -> [simple_load_with2i,
-            simple_querycount].
+            simple_querycount,
+            rotating_objects].
 
 
 simple_load_with2i(_Config) ->
@@ -270,3 +273,103 @@ count_termsonindex(Bucket, IdxField, Book, QType) ->
                         end,
                     0,
                     lists:seq(1901, 2218)).
+
+
+rotating_objects(_Config) ->
+    RootPath = testutil:reset_filestructure(),
+    ok = rotating_object_check(RootPath, "Bucket1", 10),
+    ok = rotating_object_check(RootPath, "Bucket2", 200),
+    ok = rotating_object_check(RootPath, "Bucket3", 800),
+    ok = rotating_object_check(RootPath, "Bucket4", 1600),
+    ok = rotating_object_check(RootPath, "Bucket5", 3200),
+    ok = rotating_object_check(RootPath, "Bucket6", 9600),
+    testutil:reset_filestructure().
+
+
+rotating_object_check(RootPath, Bucket, NumberOfObjects) ->
+    {ok, Book1} = leveled_bookie:book_start(RootPath, 2000, 5000000),
+    {KSpcL1, V1} = put_indexed_objects(Book1, Bucket, NumberOfObjects),
+    ok = check_indexed_objects(Book1, Bucket, KSpcL1, V1),
+    {KSpcL2, V2} = put_altered_indexed_objects(Book1, Bucket, KSpcL1),
+    ok = check_indexed_objects(Book1, Bucket, KSpcL2, V2),
+    {KSpcL3, V3} = put_altered_indexed_objects(Book1, Bucket, KSpcL2),
+    ok = leveled_bookie:book_close(Book1),
+    {ok, Book2} = leveled_bookie:book_start(RootPath, 1000, 5000000),
+    ok = check_indexed_objects(Book2, Bucket, KSpcL3, V3),
+    {KSpcL4, V4} = put_altered_indexed_objects(Book2, Bucket, KSpcL3),
+    ok = check_indexed_objects(Book2, Bucket, KSpcL4, V4),
+    ok = leveled_bookie:book_close(Book2),
+    ok.
+    
+
+
+check_indexed_objects(Book, B, KSpecL, V) ->
+    % Check all objects match, return what should eb the results of an all
+    % index query
+    IdxR = lists:map(fun({K, Spc}) ->
+                            {ok, O} = leveled_bookie:book_riakget(Book, B, K),
+                            V = testutil:get_value(O),
+                            {add,
+                                "idx1_bin",
+                                IdxVal} = lists:keyfind(add, 1, Spc),
+                            {IdxVal, K} end,
+                        KSpecL),
+    % Check the all index query matxhes expectations
+    R = leveled_bookie:book_returnfolder(Book,
+                                            {index_query,
+                                                B,
+                                                {"idx1_bin",
+                                                    "0",
+                                                    "~"},
+                                                ?RETURN_TERMS}),
+    {async, Fldr} = R,
+    QR = lists:sort(Fldr()),
+    ER = lists:sort(IdxR),
+    ok = if
+                ER == QR ->
+                    ok
+            end,
+    ok.
+
+
+put_indexed_objects(Book, Bucket, Count) ->
+    V = testutil:get_compressiblevalue(),
+    IndexGen = testutil:get_randomindexes_generator(1),
+    SW = os:timestamp(),
+    ObjL1 = testutil:generate_objects(Count,
+                                        uuid,
+                                        [],
+                                        V,
+                                        IndexGen,
+                                        Bucket),
+    KSpecL = lists:map(fun({_RN, Obj, Spc}) ->
+                            leveled_bookie:book_riakput(Book,
+                                                        Obj,
+                                                        Spc),
+                            {testutil:get_key(Obj), Spc}
+                            end,
+                        ObjL1),
+    io:format("Put of ~w objects with ~w index entries "
+                    ++
+                    "each completed in ~w microseconds~n",
+                [Count, 1, timer:now_diff(os:timestamp(), SW)]),
+    {KSpecL, V}.
+
+put_altered_indexed_objects(Book, Bucket, KSpecL) ->
+    IndexGen = testutil:get_randomindexes_generator(1),
+    V = testutil:get_compressiblevalue(),
+    RplKSpecL = lists:map(fun({K, Spc}) ->
+                                AddSpc = lists:keyfind(add, 1, Spc),
+                                {O, AltSpc} = testutil:set_object(Bucket,
+                                                                    K,
+                                                                    V,
+                                                                    IndexGen,
+                                                                    [AddSpc]),
+                                ok = leveled_bookie:book_riakput(Book,
+                                                                    O,
+                                                                    AltSpc),
+                                {K, AltSpc} end,
+                            KSpecL),
+    {RplKSpecL, V}.
+    
+

test/end_to_end/testutil.erl

@@ -15,6 +15,10 @@
             generate_smallobjects/2,
             generate_objects/2,
             generate_objects/5,
+            generate_objects/6,
+            set_object/5,
+            get_key/1,
+            get_value/1,
             get_compressiblevalue/0,
             get_randomindexes_generator/1,
             name_list/0,
@@ -152,32 +156,44 @@ generate_objects(Count, KeyNumber) ->
 generate_objects(Count, KeyNumber, ObjL, Value) ->
     generate_objects(Count, KeyNumber, ObjL, Value, fun() -> [] end).
 
-generate_objects(0, _KeyNumber, ObjL, _Value, _IndexGen) ->
+generate_objects(Count, KeyNumber, ObjL, Value, IndexGen) ->
+    generate_objects(Count, KeyNumber, ObjL, Value, IndexGen, "Bucket").
+
+generate_objects(0, _KeyNumber, ObjL, _Value, _IndexGen, _Bucket) ->
     ObjL;
-generate_objects(Count, uuid, ObjL, Value, IndexGen) ->
+generate_objects(Count, uuid, ObjL, Value, IndexGen, Bucket) ->
-    {Obj1, Spec1} = set_object(leveled_codec:generate_uuid(),
+    {Obj1, Spec1} = set_object(Bucket,
+                                leveled_codec:generate_uuid(),
                                 Value,
                                 IndexGen),
     generate_objects(Count - 1,
                         uuid,
                         ObjL ++ [{random:uniform(), Obj1, Spec1}],
                         Value,
-                        IndexGen);
+                        IndexGen,
-generate_objects(Count, KeyNumber, ObjL, Value, IndexGen) ->
+                        Bucket);
-    {Obj1, Spec1} = set_object("Key" ++ integer_to_list(KeyNumber),
+generate_objects(Count, KeyNumber, ObjL, Value, IndexGen, Bucket) ->
+    {Obj1, Spec1} = set_object(Bucket,
+                                "Key" ++ integer_to_list(KeyNumber),
                                 Value,
                                 IndexGen),
     generate_objects(Count - 1,
                         KeyNumber + 1,
                         ObjL ++ [{random:uniform(), Obj1, Spec1}],
                         Value,
-                        IndexGen).
+                        IndexGen,
+                        Bucket).
 
-set_object(Key, Value, IndexGen) ->
+set_object(Bucket, Key, Value, IndexGen) ->
-    Obj = {"Bucket",
+    set_object(Bucket, Key, Value, IndexGen, []).
+
+set_object(Bucket, Key, Value, IndexGen, Indexes2Remove) ->
+    Obj = {Bucket,
             Key,
             Value,
-            IndexGen(),
+            IndexGen() ++ lists:map(fun({add, IdxF, IdxV}) ->
+                                            {remove, IdxF, IdxV} end,
+                                        Indexes2Remove),
             [{"MDK", "MDV" ++ Key},
                 {"MDK2", "MDV" ++ Key}]},
     {B1, K1, V1, Spec1, MD} = Obj,
@@ -185,6 +201,12 @@ set_object(Key, Value, IndexGen) ->
     {#r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]},
         Spec1}.
 
+get_key(Object) ->
+    Object#r_object.key.
+
+get_value(Object) ->
+    [Content] = Object#r_object.contents,
+    Content#r_content.value.
 
 load_objects(ChunkSize, GenList, Bookie, TestObject, Generator) ->
     lists:map(fun(KN) ->