From 0324edd6f694d0a5e984c3354825ff80ad08c793 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 20 Oct 2016 12:16:17 +0100 Subject: [PATCH] Rotating object tests Recent fixes have been made to problems associated with rapidly changing objexts especially on re-opening of the bookie. Test of rotating objects from both an index query and a fetch perspective added to better detect such issues in the future. --- test/end_to_end/iterator_SUITE.erl | 107 ++++++++++++++++++++++++++++- test/end_to_end/testutil.erl | 42 ++++++++--- 2 files changed, 137 insertions(+), 12 deletions(-) diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index 30cc711..8fb7505 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -4,13 +4,16 @@ -include("include/leveled.hrl"). -define(KEY_ONLY, {false, undefined}). +-define(RETURN_TERMS, {true, undefined}). -export([all/0]). -export([simple_load_with2i/1, - simple_querycount/1]). + simple_querycount/1, + rotating_objects/1]). all() -> [simple_load_with2i, - simple_querycount]. + simple_querycount, + rotating_objects]. simple_load_with2i(_Config) -> @@ -270,3 +273,103 @@ count_termsonindex(Bucket, IdxField, Book, QType) -> end, 0, lists:seq(1901, 2218)). + + +rotating_objects(_Config) -> + RootPath = testutil:reset_filestructure(), + ok = rotating_object_check(RootPath, "Bucket1", 10), + ok = rotating_object_check(RootPath, "Bucket2", 200), + ok = rotating_object_check(RootPath, "Bucket3", 800), + ok = rotating_object_check(RootPath, "Bucket4", 1600), + ok = rotating_object_check(RootPath, "Bucket5", 3200), + ok = rotating_object_check(RootPath, "Bucket6", 9600), + testutil:reset_filestructure(). + + +rotating_object_check(RootPath, Bucket, NumberOfObjects) -> + {ok, Book1} = leveled_bookie:book_start(RootPath, 2000, 5000000), + {KSpcL1, V1} = put_indexed_objects(Book1, Bucket, NumberOfObjects), + ok = check_indexed_objects(Book1, Bucket, KSpcL1, V1), + {KSpcL2, V2} = put_altered_indexed_objects(Book1, Bucket, KSpcL1), + ok = check_indexed_objects(Book1, Bucket, KSpcL2, V2), + {KSpcL3, V3} = put_altered_indexed_objects(Book1, Bucket, KSpcL2), + ok = leveled_bookie:book_close(Book1), + {ok, Book2} = leveled_bookie:book_start(RootPath, 1000, 5000000), + ok = check_indexed_objects(Book2, Bucket, KSpcL3, V3), + {KSpcL4, V4} = put_altered_indexed_objects(Book2, Bucket, KSpcL3), + ok = check_indexed_objects(Book2, Bucket, KSpcL4, V4), + ok = leveled_bookie:book_close(Book2), + ok. + + + +check_indexed_objects(Book, B, KSpecL, V) -> + % Check all objects match, return what should eb the results of an all + % index query + IdxR = lists:map(fun({K, Spc}) -> + {ok, O} = leveled_bookie:book_riakget(Book, B, K), + V = testutil:get_value(O), + {add, + "idx1_bin", + IdxVal} = lists:keyfind(add, 1, Spc), + {IdxVal, K} end, + KSpecL), + % Check the all index query matxhes expectations + R = leveled_bookie:book_returnfolder(Book, + {index_query, + B, + {"idx1_bin", + "0", + "~"}, + ?RETURN_TERMS}), + {async, Fldr} = R, + QR = lists:sort(Fldr()), + ER = lists:sort(IdxR), + ok = if + ER == QR -> + ok + end, + ok. + + +put_indexed_objects(Book, Bucket, Count) -> + V = testutil:get_compressiblevalue(), + IndexGen = testutil:get_randomindexes_generator(1), + SW = os:timestamp(), + ObjL1 = testutil:generate_objects(Count, + uuid, + [], + V, + IndexGen, + Bucket), + KSpecL = lists:map(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Book, + Obj, + Spc), + {testutil:get_key(Obj), Spc} + end, + ObjL1), + io:format("Put of ~w objects with ~w index entries " + ++ + "each completed in ~w microseconds~n", + [Count, 1, timer:now_diff(os:timestamp(), SW)]), + {KSpecL, V}. + +put_altered_indexed_objects(Book, Bucket, KSpecL) -> + IndexGen = testutil:get_randomindexes_generator(1), + V = testutil:get_compressiblevalue(), + RplKSpecL = lists:map(fun({K, Spc}) -> + AddSpc = lists:keyfind(add, 1, Spc), + {O, AltSpc} = testutil:set_object(Bucket, + K, + V, + IndexGen, + [AddSpc]), + ok = leveled_bookie:book_riakput(Book, + O, + AltSpc), + {K, AltSpc} end, + KSpecL), + {RplKSpecL, V}. + + diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index c938d6a..e3bf77d 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -15,6 +15,10 @@ generate_smallobjects/2, generate_objects/2, generate_objects/5, + generate_objects/6, + set_object/5, + get_key/1, + get_value/1, get_compressiblevalue/0, get_randomindexes_generator/1, name_list/0, @@ -152,32 +156,44 @@ generate_objects(Count, KeyNumber) -> generate_objects(Count, KeyNumber, ObjL, Value) -> generate_objects(Count, KeyNumber, ObjL, Value, fun() -> [] end). -generate_objects(0, _KeyNumber, ObjL, _Value, _IndexGen) -> +generate_objects(Count, KeyNumber, ObjL, Value, IndexGen) -> + generate_objects(Count, KeyNumber, ObjL, Value, IndexGen, "Bucket"). + +generate_objects(0, _KeyNumber, ObjL, _Value, _IndexGen, _Bucket) -> ObjL; -generate_objects(Count, uuid, ObjL, Value, IndexGen) -> - {Obj1, Spec1} = set_object(leveled_codec:generate_uuid(), +generate_objects(Count, uuid, ObjL, Value, IndexGen, Bucket) -> + {Obj1, Spec1} = set_object(Bucket, + leveled_codec:generate_uuid(), Value, IndexGen), generate_objects(Count - 1, uuid, ObjL ++ [{random:uniform(), Obj1, Spec1}], Value, - IndexGen); -generate_objects(Count, KeyNumber, ObjL, Value, IndexGen) -> - {Obj1, Spec1} = set_object("Key" ++ integer_to_list(KeyNumber), + IndexGen, + Bucket); +generate_objects(Count, KeyNumber, ObjL, Value, IndexGen, Bucket) -> + {Obj1, Spec1} = set_object(Bucket, + "Key" ++ integer_to_list(KeyNumber), Value, IndexGen), generate_objects(Count - 1, KeyNumber + 1, ObjL ++ [{random:uniform(), Obj1, Spec1}], Value, - IndexGen). + IndexGen, + Bucket). -set_object(Key, Value, IndexGen) -> - Obj = {"Bucket", +set_object(Bucket, Key, Value, IndexGen) -> + set_object(Bucket, Key, Value, IndexGen, []). + +set_object(Bucket, Key, Value, IndexGen, Indexes2Remove) -> + Obj = {Bucket, Key, Value, - IndexGen(), + IndexGen() ++ lists:map(fun({add, IdxF, IdxV}) -> + {remove, IdxF, IdxV} end, + Indexes2Remove), [{"MDK", "MDV" ++ Key}, {"MDK2", "MDV" ++ Key}]}, {B1, K1, V1, Spec1, MD} = Obj, @@ -185,6 +201,12 @@ set_object(Key, Value, IndexGen) -> {#r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]}, Spec1}. +get_key(Object) -> + Object#r_object.key. + +get_value(Object) -> + [Content] = Object#r_object.contents, + Content#r_content.value. load_objects(ChunkSize, GenList, Bookie, TestObject, Generator) -> lists:map(fun(KN) ->