Rotating object tests
Recent fixes have been made to problems associated with rapidly changing objexts especially on re-opening of the bookie. Test of rotating objects from both an index query and a fetch perspective added to better detect such issues in the future.
This commit is contained in:
martinsumner
parent
cf66431c8e
commit
0324edd6f6
2 changed files with 137 additions and 12 deletions
|
|
@ -4,13 +4,16 @@
|
|||
-include("include/leveled.hrl").
|
||||
|
||||
-define(KEY_ONLY, {false, undefined}).
|
||||
-define(RETURN_TERMS, {true, undefined}).
|
||||
|
||||
-export([all/0]).
|
||||
-export([simple_load_with2i/1,
|
||||
simple_querycount/1]).
|
||||
simple_querycount/1,
|
||||
rotating_objects/1]).
|
||||
|
||||
all() -> [simple_load_with2i,
|
||||
simple_querycount].
|
||||
simple_querycount,
|
||||
rotating_objects].
|
||||
|
||||
|
||||
simple_load_with2i(_Config) ->
|
||||
|
|
@ -270,3 +273,103 @@ count_termsonindex(Bucket, IdxField, Book, QType) ->
|
|||
end,
|
||||
0,
|
||||
lists:seq(1901, 2218)).
|
||||
|
||||
|
||||
rotating_objects(_Config) ->
|
||||
RootPath = testutil:reset_filestructure(),
|
||||
ok = rotating_object_check(RootPath, "Bucket1", 10),
|
||||
ok = rotating_object_check(RootPath, "Bucket2", 200),
|
||||
ok = rotating_object_check(RootPath, "Bucket3", 800),
|
||||
ok = rotating_object_check(RootPath, "Bucket4", 1600),
|
||||
ok = rotating_object_check(RootPath, "Bucket5", 3200),
|
||||
ok = rotating_object_check(RootPath, "Bucket6", 9600),
|
||||
testutil:reset_filestructure().
|
||||
|
||||
|
||||
rotating_object_check(RootPath, Bucket, NumberOfObjects) ->
|
||||
{ok, Book1} = leveled_bookie:book_start(RootPath, 2000, 5000000),
|
||||
{KSpcL1, V1} = put_indexed_objects(Book1, Bucket, NumberOfObjects),
|
||||
ok = check_indexed_objects(Book1, Bucket, KSpcL1, V1),
|
||||
{KSpcL2, V2} = put_altered_indexed_objects(Book1, Bucket, KSpcL1),
|
||||
ok = check_indexed_objects(Book1, Bucket, KSpcL2, V2),
|
||||
{KSpcL3, V3} = put_altered_indexed_objects(Book1, Bucket, KSpcL2),
|
||||
ok = leveled_bookie:book_close(Book1),
|
||||
{ok, Book2} = leveled_bookie:book_start(RootPath, 1000, 5000000),
|
||||
ok = check_indexed_objects(Book2, Bucket, KSpcL3, V3),
|
||||
{KSpcL4, V4} = put_altered_indexed_objects(Book2, Bucket, KSpcL3),
|
||||
ok = check_indexed_objects(Book2, Bucket, KSpcL4, V4),
|
||||
ok = leveled_bookie:book_close(Book2),
|
||||
ok.
|
||||
|
||||
|
||||
|
||||
check_indexed_objects(Book, B, KSpecL, V) ->
|
||||
% Check all objects match, return what should eb the results of an all
|
||||
% index query
|
||||
IdxR = lists:map(fun({K, Spc}) ->
|
||||
{ok, O} = leveled_bookie:book_riakget(Book, B, K),
|
||||
V = testutil:get_value(O),
|
||||
{add,
|
||||
"idx1_bin",
|
||||
IdxVal} = lists:keyfind(add, 1, Spc),
|
||||
{IdxVal, K} end,
|
||||
KSpecL),
|
||||
% Check the all index query matxhes expectations
|
||||
R = leveled_bookie:book_returnfolder(Book,
|
||||
{index_query,
|
||||
B,
|
||||
{"idx1_bin",
|
||||
"0",
|
||||
"~"},
|
||||
?RETURN_TERMS}),
|
||||
{async, Fldr} = R,
|
||||
QR = lists:sort(Fldr()),
|
||||
ER = lists:sort(IdxR),
|
||||
ok = if
|
||||
ER == QR ->
|
||||
ok
|
||||
end,
|
||||
ok.
|
||||
|
||||
|
||||
put_indexed_objects(Book, Bucket, Count) ->
|
||||
V = testutil:get_compressiblevalue(),
|
||||
IndexGen = testutil:get_randomindexes_generator(1),
|
||||
SW = os:timestamp(),
|
||||
ObjL1 = testutil:generate_objects(Count,
|
||||
uuid,
|
||||
[],
|
||||
V,
|
||||
IndexGen,
|
||||
Bucket),
|
||||
KSpecL = lists:map(fun({_RN, Obj, Spc}) ->
|
||||
leveled_bookie:book_riakput(Book,
|
||||
Obj,
|
||||
Spc),
|
||||
{testutil:get_key(Obj), Spc}
|
||||
end,
|
||||
ObjL1),
|
||||
io:format("Put of ~w objects with ~w index entries "
|
||||
++
|
||||
"each completed in ~w microseconds~n",
|
||||
[Count, 1, timer:now_diff(os:timestamp(), SW)]),
|
||||
{KSpecL, V}.
|
||||
|
||||
put_altered_indexed_objects(Book, Bucket, KSpecL) ->
|
||||
IndexGen = testutil:get_randomindexes_generator(1),
|
||||
V = testutil:get_compressiblevalue(),
|
||||
RplKSpecL = lists:map(fun({K, Spc}) ->
|
||||
AddSpc = lists:keyfind(add, 1, Spc),
|
||||
{O, AltSpc} = testutil:set_object(Bucket,
|
||||
K,
|
||||
V,
|
||||
IndexGen,
|
||||
[AddSpc]),
|
||||
ok = leveled_bookie:book_riakput(Book,
|
||||
O,
|
||||
AltSpc),
|
||||
{K, AltSpc} end,
|
||||
KSpecL),
|
||||
{RplKSpecL, V}.
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,10 @@
|
|||
generate_smallobjects/2,
|
||||
generate_objects/2,
|
||||
generate_objects/5,
|
||||
generate_objects/6,
|
||||
set_object/5,
|
||||
get_key/1,
|
||||
get_value/1,
|
||||
get_compressiblevalue/0,
|
||||
get_randomindexes_generator/1,
|
||||
name_list/0,
|
||||
|
|
@ -152,32 +156,44 @@ generate_objects(Count, KeyNumber) ->
|
|||
generate_objects(Count, KeyNumber, ObjL, Value) ->
|
||||
generate_objects(Count, KeyNumber, ObjL, Value, fun() -> [] end).
|
||||
|
||||
generate_objects(0, _KeyNumber, ObjL, _Value, _IndexGen) ->
|
||||
generate_objects(Count, KeyNumber, ObjL, Value, IndexGen) ->
|
||||
generate_objects(Count, KeyNumber, ObjL, Value, IndexGen, "Bucket").
|
||||
|
||||
generate_objects(0, _KeyNumber, ObjL, _Value, _IndexGen, _Bucket) ->
|
||||
ObjL;
|
||||
generate_objects(Count, uuid, ObjL, Value, IndexGen) ->
|
||||
{Obj1, Spec1} = set_object(leveled_codec:generate_uuid(),
|
||||
generate_objects(Count, uuid, ObjL, Value, IndexGen, Bucket) ->
|
||||
{Obj1, Spec1} = set_object(Bucket,
|
||||
leveled_codec:generate_uuid(),
|
||||
Value,
|
||||
IndexGen),
|
||||
generate_objects(Count - 1,
|
||||
uuid,
|
||||
ObjL ++ [{random:uniform(), Obj1, Spec1}],
|
||||
Value,
|
||||
IndexGen);
|
||||
generate_objects(Count, KeyNumber, ObjL, Value, IndexGen) ->
|
||||
{Obj1, Spec1} = set_object("Key" ++ integer_to_list(KeyNumber),
|
||||
IndexGen,
|
||||
Bucket);
|
||||
generate_objects(Count, KeyNumber, ObjL, Value, IndexGen, Bucket) ->
|
||||
{Obj1, Spec1} = set_object(Bucket,
|
||||
"Key" ++ integer_to_list(KeyNumber),
|
||||
Value,
|
||||
IndexGen),
|
||||
generate_objects(Count - 1,
|
||||
KeyNumber + 1,
|
||||
ObjL ++ [{random:uniform(), Obj1, Spec1}],
|
||||
Value,
|
||||
IndexGen).
|
||||
IndexGen,
|
||||
Bucket).
|
||||
|
||||
set_object(Key, Value, IndexGen) ->
|
||||
Obj = {"Bucket",
|
||||
set_object(Bucket, Key, Value, IndexGen) ->
|
||||
set_object(Bucket, Key, Value, IndexGen, []).
|
||||
|
||||
set_object(Bucket, Key, Value, IndexGen, Indexes2Remove) ->
|
||||
Obj = {Bucket,
|
||||
Key,
|
||||
Value,
|
||||
IndexGen(),
|
||||
IndexGen() ++ lists:map(fun({add, IdxF, IdxV}) ->
|
||||
{remove, IdxF, IdxV} end,
|
||||
Indexes2Remove),
|
||||
[{"MDK", "MDV" ++ Key},
|
||||
{"MDK2", "MDV" ++ Key}]},
|
||||
{B1, K1, V1, Spec1, MD} = Obj,
|
||||
|
|
@ -185,6 +201,12 @@ set_object(Key, Value, IndexGen) ->
|
|||
{#r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]},
|
||||
Spec1}.
|
||||
|
||||
get_key(Object) ->
|
||||
Object#r_object.key.
|
||||
|
||||
get_value(Object) ->
|
||||
[Content] = Object#r_object.contents,
|
||||
Content#r_content.value.
|
||||
|
||||
load_objects(ChunkSize, GenList, Bookie, TestObject, Generator) ->
|
||||
lists:map(fun(KN) ->
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue