Add initial end-to-end common tests
These tests highlighted some logical issues when scanning over databases on startup, so fixes are wrapped in here.
This commit is contained in:
martinsumner
parent
507428bd0b
commit
d903f184fd
6 changed files with 294 additions and 94 deletions
|
|
@ -2,44 +2,112 @@
|
|||
-include_lib("common_test/include/ct.hrl").
|
||||
-include("../include/leveled.hrl").
|
||||
-export([all/0]).
|
||||
-export([simple_put_fetch/1,
|
||||
journal_compaction/1]).
|
||||
-export([simple_put_fetch_head/1,
|
||||
many_put_fetch_head/1,
|
||||
journal_compaction/1,
|
||||
simple_snapshot/1]).
|
||||
|
||||
all() -> [journal_compaction, simple_put_fetch].
|
||||
all() -> [simple_put_fetch_head,
|
||||
many_put_fetch_head,
|
||||
journal_compaction,
|
||||
simple_snapshot].
|
||||
|
||||
simple_put_fetch(_Config) ->
|
||||
simple_put_fetch_head(_Config) ->
|
||||
RootPath = reset_filestructure(),
|
||||
StartOpts1 = #bookie_options{root_path=RootPath},
|
||||
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
|
||||
{TestObject, TestSpec} = generate_testobject(),
|
||||
ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec),
|
||||
{ok, TestObject} = leveled_bookie:book_riakget(Bookie1,
|
||||
TestObject#r_object.bucket,
|
||||
TestObject#r_object.key),
|
||||
check_bookie_forobject(Bookie1, TestObject),
|
||||
ok = leveled_bookie:book_close(Bookie1),
|
||||
StartOpts2 = #bookie_options{root_path=RootPath,
|
||||
max_journalsize=3000000},
|
||||
{ok, Bookie2} = leveled_bookie:book_start(StartOpts2),
|
||||
{ok, TestObject} = leveled_bookie:book_riakget(Bookie2,
|
||||
TestObject#r_object.bucket,
|
||||
TestObject#r_object.key),
|
||||
check_bookie_forobject(Bookie2, TestObject),
|
||||
ObjList1 = generate_multiple_objects(5000, 2),
|
||||
lists:foreach(fun({_RN, Obj, Spc}) ->
|
||||
leveled_bookie:book_riakput(Bookie2, Obj, Spc) end,
|
||||
ObjList1),
|
||||
ChkList1 = lists:sublist(lists:sort(ObjList1), 100),
|
||||
lists:foreach(fun({_RN, Obj, _Spc}) ->
|
||||
R = leveled_bookie:book_riakget(Bookie2,
|
||||
Obj#r_object.bucket,
|
||||
Obj#r_object.key),
|
||||
R = {ok, Obj} end,
|
||||
ChkList1),
|
||||
{ok, TestObject} = leveled_bookie:book_riakget(Bookie2,
|
||||
TestObject#r_object.bucket,
|
||||
TestObject#r_object.key),
|
||||
check_bookie_forlist(Bookie2, ChkList1),
|
||||
check_bookie_forobject(Bookie2, TestObject),
|
||||
ok = leveled_bookie:book_close(Bookie2),
|
||||
reset_filestructure().
|
||||
|
||||
many_put_fetch_head(_Config) ->
|
||||
RootPath = reset_filestructure(),
|
||||
StartOpts1 = #bookie_options{root_path=RootPath},
|
||||
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
|
||||
{TestObject, TestSpec} = generate_testobject(),
|
||||
ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec),
|
||||
check_bookie_forobject(Bookie1, TestObject),
|
||||
ok = leveled_bookie:book_close(Bookie1),
|
||||
StartOpts2 = #bookie_options{root_path=RootPath,
|
||||
max_journalsize=1000000000},
|
||||
{ok, Bookie2} = leveled_bookie:book_start(StartOpts2),
|
||||
check_bookie_forobject(Bookie2, TestObject),
|
||||
GenList = [2, 20002, 40002, 60002, 80002,
|
||||
100002, 120002, 140002, 160002, 180002],
|
||||
CLs = lists:map(fun(KN) ->
|
||||
ObjListA = generate_multiple_smallobjects(20000, KN),
|
||||
StartWatchA = os:timestamp(),
|
||||
lists:foreach(fun({_RN, Obj, Spc}) ->
|
||||
leveled_bookie:book_riakput(Bookie2, Obj, Spc)
|
||||
end,
|
||||
ObjListA),
|
||||
Time = timer:now_diff(os:timestamp(), StartWatchA),
|
||||
io:format("20,000 objects loaded in ~w seconds~n",
|
||||
[Time/1000000]),
|
||||
check_bookie_forobject(Bookie2, TestObject),
|
||||
lists:sublist(ObjListA, 1000) end,
|
||||
GenList),
|
||||
CL1A = lists:nth(1, CLs),
|
||||
ChkListFixed = lists:nth(length(CLs), CLs),
|
||||
check_bookie_forlist(Bookie2, CL1A),
|
||||
ObjList2A = generate_multiple_objects(5000, 2),
|
||||
lists:foreach(fun({_RN, Obj, Spc}) ->
|
||||
leveled_bookie:book_riakput(Bookie2, Obj, Spc) end,
|
||||
ObjList2A),
|
||||
ChkList2A = lists:sublist(lists:sort(ObjList2A), 1000),
|
||||
check_bookie_forlist(Bookie2, ChkList2A),
|
||||
check_bookie_forlist(Bookie2, ChkListFixed),
|
||||
check_bookie_forobject(Bookie2, TestObject),
|
||||
check_bookie_forlist(Bookie2, ChkList2A),
|
||||
check_bookie_forlist(Bookie2, ChkListFixed),
|
||||
check_bookie_forobject(Bookie2, TestObject),
|
||||
ok = leveled_bookie:book_close(Bookie2),
|
||||
{ok, Bookie3} = leveled_bookie:book_start(StartOpts2),
|
||||
check_bookie_forlist(Bookie3, ChkList2A),
|
||||
check_bookie_forobject(Bookie3, TestObject),
|
||||
ok = leveled_bookie:book_close(Bookie3),
|
||||
reset_filestructure().
|
||||
|
||||
|
||||
check_bookie_forlist(Bookie, ChkList) ->
|
||||
lists:foreach(fun({_RN, Obj, _Spc}) ->
|
||||
R = leveled_bookie:book_riakget(Bookie,
|
||||
Obj#r_object.bucket,
|
||||
Obj#r_object.key),
|
||||
io:format("Checking key ~s~n", [Obj#r_object.key]),
|
||||
R = {ok, Obj} end,
|
||||
ChkList).
|
||||
|
||||
check_bookie_forobject(Bookie, TestObject) ->
|
||||
{ok, TestObject} = leveled_bookie:book_riakget(Bookie,
|
||||
TestObject#r_object.bucket,
|
||||
TestObject#r_object.key),
|
||||
{ok, HeadObject} = leveled_bookie:book_riakhead(Bookie,
|
||||
TestObject#r_object.bucket,
|
||||
TestObject#r_object.key),
|
||||
ok = case {HeadObject#r_object.bucket,
|
||||
HeadObject#r_object.key,
|
||||
HeadObject#r_object.vclock} of
|
||||
{B1, K1, VC1} when B1 == TestObject#r_object.bucket,
|
||||
K1 == TestObject#r_object.key,
|
||||
VC1 == TestObject#r_object.vclock ->
|
||||
ok
|
||||
end.
|
||||
|
||||
journal_compaction(_Config) ->
|
||||
RootPath = reset_filestructure(),
|
||||
StartOpts1 = #bookie_options{root_path=RootPath,
|
||||
|
|
@ -93,6 +161,37 @@ journal_compaction(_Config) ->
|
|||
reset_filestructure().
|
||||
|
||||
|
||||
simple_snapshot(_Config) ->
|
||||
RootPath = reset_filestructure(),
|
||||
StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=3000000},
|
||||
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
|
||||
{TestObject, TestSpec} = generate_testobject(),
|
||||
ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec),
|
||||
ObjList1 = generate_multiple_objects(5000, 2),
|
||||
lists:foreach(fun({_RN, Obj, Spc}) ->
|
||||
leveled_bookie:book_riakput(Bookie1, Obj, Spc) end,
|
||||
ObjList1),
|
||||
SnapOpts = #bookie_options{snapshot_bookie=Bookie1},
|
||||
{ok, SnapBookie} = leveled_bookie:book_start(SnapOpts),
|
||||
ChkList1 = lists:sublist(lists:sort(ObjList1), 100),
|
||||
lists:foreach(fun({_RN, Obj, _Spc}) ->
|
||||
R = leveled_bookie:book_riakget(Bookie1,
|
||||
Obj#r_object.bucket,
|
||||
Obj#r_object.key),
|
||||
R = {ok, Obj} end,
|
||||
ChkList1),
|
||||
lists:foreach(fun({_RN, Obj, _Spc}) ->
|
||||
R = leveled_bookie:book_riakget(SnapBookie,
|
||||
Obj#r_object.bucket,
|
||||
Obj#r_object.key),
|
||||
io:format("Finding key ~s~n", [Obj#r_object.key]),
|
||||
R = {ok, Obj} end,
|
||||
ChkList1),
|
||||
ok = leveled_bookie:book_close(SnapBookie),
|
||||
ok = leveled_bookie:book_close(Bookie1),
|
||||
reset_filestructure().
|
||||
|
||||
|
||||
reset_filestructure() ->
|
||||
RootPath = "test",
|
||||
filelib:ensure_dir(RootPath ++ "/journal/"),
|
||||
|
|
@ -111,6 +210,9 @@ generate_testobject() ->
|
|||
{#r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]},
|
||||
Spec1}.
|
||||
|
||||
generate_multiple_smallobjects(Count, KeyNumber) ->
|
||||
generate_multiple_objects(Count, KeyNumber, [], crypto:rand_bytes(512)).
|
||||
|
||||
generate_multiple_objects(Count, KeyNumber) ->
|
||||
generate_multiple_objects(Count, KeyNumber, [], crypto:rand_bytes(4096)).
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue