umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Clean API of Riak-specific Methods

Browse source 
Clena the API of Riak specific methods, and also resolve timing issue in
simple_server unit test.  Previously this would end up with missing data
(and a lower sequence number after start) because of the penciller_clerk
timeout being relatively large in the context of this test.  Now the
timeout has bene reduced the L0 slot is cleared by the time of the
close.  To make sure an extra sleep has been added as a precaution to
avoid any intermittent issues.
This commit is contained in:
martinsumner 2016-11-07 10:11:57 +00:00
parent 2d590c3077
commit 4583460328
6 changed files with 124 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

60
src/leveled_bookie.erl
View file

@ -134,10 +134,6 @@
code_change/3, code_change/3,
book_start/1, book_start/1,
book_start/3, book_start/3,
book_riakput/3,
book_riakdelete/4,
book_riakget/3,
book_riakhead/3,
book_put/5, book_put/5,
book_put/6, book_put/6,
book_tempput/7, book_tempput/7,
@ -184,9 +180,6 @@ book_start(RootPath, LedgerCacheSize, JournalSize) ->
book_start(Opts) -> book_start(Opts) ->
gen_server:start(?MODULE, [Opts], []). gen_server:start(?MODULE, [Opts], []).
book_riakput(Pid, RiakObject, IndexSpecs) ->
{Bucket, Key} = leveled_codec:riakto_keydetails(RiakObject),
book_put(Pid, Bucket, Key, RiakObject, IndexSpecs, ?RIAK_TAG).
book_tempput(Pid, Bucket, Key, Object, IndexSpecs, Tag, TTL) when is_integer(TTL) -> book_tempput(Pid, Bucket, Key, Object, IndexSpecs, Tag, TTL) when is_integer(TTL) ->
book_put(Pid, Bucket, Key, Object, IndexSpecs, Tag, TTL). book_put(Pid, Bucket, Key, Object, IndexSpecs, Tag, TTL).
@ -197,21 +190,12 @@ book_put(Pid, Bucket, Key, Object, IndexSpecs) ->
book_put(Pid, Bucket, Key, Object, IndexSpecs, Tag) -> book_put(Pid, Bucket, Key, Object, IndexSpecs, Tag) ->
book_put(Pid, Bucket, Key, Object, IndexSpecs, Tag, infinity). book_put(Pid, Bucket, Key, Object, IndexSpecs, Tag, infinity).
book_riakdelete(Pid, Bucket, Key, IndexSpecs) ->
book_put(Pid, Bucket, Key, delete, IndexSpecs, ?RIAK_TAG).
book_delete(Pid, Bucket, Key, IndexSpecs) -> book_delete(Pid, Bucket, Key, IndexSpecs) ->
book_put(Pid, Bucket, Key, delete, IndexSpecs, ?STD_TAG). book_put(Pid, Bucket, Key, delete, IndexSpecs, ?STD_TAG).
book_riakget(Pid, Bucket, Key) ->
book_get(Pid, Bucket, Key, ?RIAK_TAG).
book_get(Pid, Bucket, Key) -> book_get(Pid, Bucket, Key) ->
book_get(Pid, Bucket, Key, ?STD_TAG). book_get(Pid, Bucket, Key, ?STD_TAG).
book_riakhead(Pid, Bucket, Key) ->
book_head(Pid, Bucket, Key, ?RIAK_TAG).
book_head(Pid, Bucket, Key) -> book_head(Pid, Bucket, Key) ->
book_head(Pid, Bucket, Key, ?STD_TAG). book_head(Pid, Bucket, Key, ?STD_TAG).
@ -933,12 +917,12 @@ single_key_test() ->
{"MDK1", "MDV1"}}, {"MDK1", "MDV1"}},
Content = #r_content{metadata=MD, value=V1}, Content = #r_content{metadata=MD, value=V1},
Object = #r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]}, Object = #r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]},
ok = book_riakput(Bookie1, Object, Spec1), ok = book_put(Bookie1, B1, K1, Object, Spec1, ?RIAK_TAG),
{ok, F1} = book_riakget(Bookie1, B1, K1), {ok, F1} = book_get(Bookie1, B1, K1, ?RIAK_TAG),
?assertMatch(F1, Object), ?assertMatch(F1, Object),
ok = book_close(Bookie1), ok = book_close(Bookie1),
{ok, Bookie2} = book_start([{root_path, RootPath}]), {ok, Bookie2} = book_start([{root_path, RootPath}]),
{ok, F2} = book_riakget(Bookie2, B1, K1), {ok, F2} = book_get(Bookie2, B1, K1, ?RIAK_TAG),
?assertMatch(F2, Object), ?assertMatch(F2, Object),
ok = book_close(Bookie2), ok = book_close(Bookie2),
reset_filestructure(). reset_filestructure().
@ -960,41 +944,53 @@ multi_key_test() ->
{"MDK2", "MDV2"}}, {"MDK2", "MDV2"}},
C2 = #r_content{metadata=MD2, value=V2}, C2 = #r_content{metadata=MD2, value=V2},
Obj2 = #r_object{bucket=B2, key=K2, contents=[C2], vclock=[{'a',1}]}, Obj2 = #r_object{bucket=B2, key=K2, contents=[C2], vclock=[{'a',1}]},
ok = book_riakput(Bookie1, Obj1, Spec1), ok = book_put(Bookie1, B1, K1, Obj1, Spec1, ?RIAK_TAG),
ObjL1 = generate_multiple_robjects(100, 3), ObjL1 = generate_multiple_robjects(100, 3),
SW1 = os:timestamp(), SW1 = os:timestamp(),
lists:foreach(fun({O, S}) -> ok = book_riakput(Bookie1, O, S) end, ObjL1), lists:foreach(fun({O, S}) ->
{B, K} = leveled_codec:riakto_keydetails(O),
ok = book_put(Bookie1, B, K, O, S, ?RIAK_TAG)
end,
ObjL1),
io:format("PUT of 100 objects completed in ~w microseconds~n", io:format("PUT of 100 objects completed in ~w microseconds~n",
[timer:now_diff(os:timestamp(),SW1)]), [timer:now_diff(os:timestamp(),SW1)]),
ok = book_riakput(Bookie1, Obj2, Spec2), ok = book_put(Bookie1, B2, K2, Obj2, Spec2, ?RIAK_TAG),
{ok, F1A} = book_riakget(Bookie1, B1, K1), {ok, F1A} = book_get(Bookie1, B1, K1, ?RIAK_TAG),
?assertMatch(F1A, Obj1), ?assertMatch(F1A, Obj1),
{ok, F2A} = book_riakget(Bookie1, B2, K2), {ok, F2A} = book_get(Bookie1, B2, K2, ?RIAK_TAG),
?assertMatch(F2A, Obj2), ?assertMatch(F2A, Obj2),
ObjL2 = generate_multiple_robjects(100, 103), ObjL2 = generate_multiple_robjects(100, 103),
SW2 = os:timestamp(), SW2 = os:timestamp(),
lists:foreach(fun({O, S}) -> ok = book_riakput(Bookie1, O, S) end, ObjL2), lists:foreach(fun({O, S}) ->
{B, K} = leveled_codec:riakto_keydetails(O),
ok = book_put(Bookie1, B, K, O, S, ?RIAK_TAG)
end,
ObjL2),
io:format("PUT of 100 objects completed in ~w microseconds~n", io:format("PUT of 100 objects completed in ~w microseconds~n",
[timer:now_diff(os:timestamp(),SW2)]), [timer:now_diff(os:timestamp(),SW2)]),
{ok, F1B} = book_riakget(Bookie1, B1, K1), {ok, F1B} = book_get(Bookie1, B1, K1, ?RIAK_TAG),
?assertMatch(F1B, Obj1), ?assertMatch(F1B, Obj1),
{ok, F2B} = book_riakget(Bookie1, B2, K2), {ok, F2B} = book_get(Bookie1, B2, K2, ?RIAK_TAG),
?assertMatch(F2B, Obj2), ?assertMatch(F2B, Obj2),
ok = book_close(Bookie1), ok = book_close(Bookie1),
% Now reopen the file, and confirm that a fetch is still possible % Now reopen the file, and confirm that a fetch is still possible
{ok, Bookie2} = book_start([{root_path, RootPath}]), {ok, Bookie2} = book_start([{root_path, RootPath}]),
{ok, F1C} = book_riakget(Bookie2, B1, K1), {ok, F1C} = book_get(Bookie2, B1, K1, ?RIAK_TAG),
?assertMatch(F1C, Obj1), ?assertMatch(F1C, Obj1),
{ok, F2C} = book_riakget(Bookie2, B2, K2), {ok, F2C} = book_get(Bookie2, B2, K2, ?RIAK_TAG),
?assertMatch(F2C, Obj2), ?assertMatch(F2C, Obj2),
ObjL3 = generate_multiple_robjects(100, 203), ObjL3 = generate_multiple_robjects(100, 203),
SW3 = os:timestamp(), SW3 = os:timestamp(),
lists:foreach(fun({O, S}) -> ok = book_riakput(Bookie2, O, S) end, ObjL3), lists:foreach(fun({O, S}) ->
{B, K} = leveled_codec:riakto_keydetails(O),
ok = book_put(Bookie2, B, K, O, S, ?RIAK_TAG)
end,
ObjL3),
io:format("PUT of 100 objects completed in ~w microseconds~n", io:format("PUT of 100 objects completed in ~w microseconds~n",
[timer:now_diff(os:timestamp(),SW3)]), [timer:now_diff(os:timestamp(),SW3)]),
{ok, F1D} = book_riakget(Bookie2, B1, K1), {ok, F1D} = book_get(Bookie2, B1, K1, ?RIAK_TAG),
?assertMatch(F1D, Obj1), ?assertMatch(F1D, Obj1),
{ok, F2D} = book_riakget(Bookie2, B2, K2), {ok, F2D} = book_get(Bookie2, B2, K2, ?RIAK_TAG),
?assertMatch(F2D, Obj2), ?assertMatch(F2D, Obj2),
ok = book_close(Bookie2), ok = book_close(Bookie2),
reset_filestructure(). reset_filestructure().

25
src/leveled_penciller.erl
View file

@ -1298,13 +1298,18 @@ simple_server_test() ->
clean_testdir(RootPath), clean_testdir(RootPath),
{ok, PCL} = pcl_start(#penciller_options{root_path=RootPath, {ok, PCL} = pcl_start(#penciller_options{root_path=RootPath,
max_inmemory_tablesize=1000}), max_inmemory_tablesize=1000}),
Key1 = {{o,"Bucket0001", "Key0001", null}, {1, {active, infinity}, null}}, Key1 = {{o,"Bucket0001", "Key0001", null},
{1, {active, infinity}, null}},
KL1 = leveled_sft:generate_randomkeys({1000, 2}), KL1 = leveled_sft:generate_randomkeys({1000, 2}),
Key2 = {{o,"Bucket0002", "Key0002", null}, {1002, {active, infinity}, null}}, Key2 = {{o,"Bucket0002", "Key0002", null},
KL2 = leveled_sft:generate_randomkeys({1000, 1003}), {1002, {active, infinity}, null}},
Key3 = {{o,"Bucket0003", "Key0003", null}, {2003, {active, infinity}, null}}, KL2 = leveled_sft:generate_randomkeys({900, 1003}),
KL3 = leveled_sft:generate_randomkeys({1000, 2004}), % Keep below the max table size by having 900 not 1000
Key4 = {{o,"Bucket0004", "Key0004", null}, {3004, {active, infinity}, null}}, Key3 = {{o,"Bucket0003", "Key0003", null},
{2003, {active, infinity}, null}},
KL3 = leveled_sft:generate_randomkeys({1000, 2004}),
Key4 = {{o,"Bucket0004", "Key0004", null},
{3004, {active, infinity}, null}},
KL4 = leveled_sft:generate_randomkeys({1000, 3005}), KL4 = leveled_sft:generate_randomkeys({1000, 3005}),
ok = maybe_pause_push(PCL, [Key1]), ok = maybe_pause_push(PCL, [Key1]),
?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})),
@ -1321,13 +1326,15 @@ simple_server_test() ->
?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})),
?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})),
?assertMatch(Key3, pcl_fetch(PCL, {o,"Bucket0003", "Key0003", null})), ?assertMatch(Key3, pcl_fetch(PCL, {o,"Bucket0003", "Key0003", null})),
timer:sleep(200),
% This sleep should make sure that the merge to L1 has occurred
% This will free up the L0 slot for the remainder to be written in shutdown
ok = pcl_close(PCL), ok = pcl_close(PCL),
{ok, PCLr} = pcl_start(#penciller_options{root_path=RootPath, {ok, PCLr} = pcl_start(#penciller_options{root_path=RootPath,
max_inmemory_tablesize=1000}), max_inmemory_tablesize=1000}),
?assertMatch(1001, pcl_getstartupsequencenumber(PCLr)), ?assertMatch(2003, pcl_getstartupsequencenumber(PCLr)),
ok = maybe_pause_push(PCLr, [Key2] ++ KL2 ++ [Key3]), % ok = maybe_pause_push(PCLr, [Key2] ++ KL2 ++ [Key3]),
io:format("Back to starting position with lost data recovered~n"),
?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001", null})), ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001", null})),
?assertMatch(Key2, pcl_fetch(PCLr, {o,"Bucket0002", "Key0002", null})), ?assertMatch(Key2, pcl_fetch(PCLr, {o,"Bucket0002", "Key0002", null})),

50
test/end_to_end/basic_SUITE.erl
View file

@ -27,7 +27,7 @@ simple_put_fetch_head_delete(_Config) ->
StartOpts1 = [{root_path, RootPath}], StartOpts1 = [{root_path, RootPath}],
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
{TestObject, TestSpec} = testutil:generate_testobject(), {TestObject, TestSpec} = testutil:generate_testobject(),
ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
testutil:check_forobject(Bookie1, TestObject), testutil:check_forobject(Bookie1, TestObject),
testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"), testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"),
ok = leveled_bookie:book_close(Bookie1), ok = leveled_bookie:book_close(Bookie1),
@ -37,7 +37,7 @@ simple_put_fetch_head_delete(_Config) ->
testutil:check_forobject(Bookie2, TestObject), testutil:check_forobject(Bookie2, TestObject),
ObjList1 = testutil:generate_objects(5000, 2), ObjList1 = testutil:generate_objects(5000, 2),
lists:foreach(fun({_RN, Obj, Spc}) -> lists:foreach(fun({_RN, Obj, Spc}) ->
leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, testutil:book_riakput(Bookie2, Obj, Spc) end,
ObjList1), ObjList1),
ChkList1 = lists:sublist(lists:sort(ObjList1), 100), ChkList1 = lists:sublist(lists:sort(ObjList1), 100),
testutil:check_forlist(Bookie2, ChkList1), testutil:check_forlist(Bookie2, ChkList1),
@ -72,7 +72,7 @@ many_put_fetch_head(_Config) ->
StartOpts1 = [{root_path, RootPath}, {max_pencillercachesize, 16000}], StartOpts1 = [{root_path, RootPath}, {max_pencillercachesize, 16000}],
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
{TestObject, TestSpec} = testutil:generate_testobject(), {TestObject, TestSpec} = testutil:generate_testobject(),
ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
testutil:check_forobject(Bookie1, TestObject), testutil:check_forobject(Bookie1, TestObject),
ok = leveled_bookie:book_close(Bookie1), ok = leveled_bookie:book_close(Bookie1),
StartOpts2 = [{root_path, RootPath}, StartOpts2 = [{root_path, RootPath},
@ -89,7 +89,7 @@ many_put_fetch_head(_Config) ->
testutil:check_forlist(Bookie2, CL1A), testutil:check_forlist(Bookie2, CL1A),
ObjList2A = testutil:generate_objects(5000, 2), ObjList2A = testutil:generate_objects(5000, 2),
lists:foreach(fun({_RN, Obj, Spc}) -> lists:foreach(fun({_RN, Obj, Spc}) ->
leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, testutil:book_riakput(Bookie2, Obj, Spc) end,
ObjList2A), ObjList2A),
ChkList2A = lists:sublist(lists:sort(ObjList2A), 1000), ChkList2A = lists:sublist(lists:sort(ObjList2A), 1000),
testutil:check_forlist(Bookie2, ChkList2A), testutil:check_forlist(Bookie2, ChkList2A),
@ -113,11 +113,11 @@ journal_compaction(_Config) ->
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
ok = leveled_bookie:book_compactjournal(Bookie1, 30000), ok = leveled_bookie:book_compactjournal(Bookie1, 30000),
{TestObject, TestSpec} = testutil:generate_testobject(), {TestObject, TestSpec} = testutil:generate_testobject(),
ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
testutil:check_forobject(Bookie1, TestObject), testutil:check_forobject(Bookie1, TestObject),
ObjList1 = testutil:generate_objects(20000, 2), ObjList1 = testutil:generate_objects(20000, 2),
lists:foreach(fun({_RN, Obj, Spc}) -> lists:foreach(fun({_RN, Obj, Spc}) ->
leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, testutil:book_riakput(Bookie1, Obj, Spc) end,
ObjList1), ObjList1),
ChkList1 = lists:sublist(lists:sort(ObjList1), 10000), ChkList1 = lists:sublist(lists:sort(ObjList1), 10000),
testutil:check_forlist(Bookie1, ChkList1), testutil:check_forlist(Bookie1, ChkList1),
@ -129,7 +129,7 @@ journal_compaction(_Config) ->
{"MDK1", "MDV1"}}, {"MDK1", "MDV1"}},
{TestObject2, TestSpec2} = testutil:generate_testobject(B2, K2, {TestObject2, TestSpec2} = testutil:generate_testobject(B2, K2,
V2, Spec2, MD), V2, Spec2, MD),
ok = leveled_bookie:book_riakput(Bookie1, TestObject2, TestSpec2), ok = testutil:book_riakput(Bookie1, TestObject2, TestSpec2),
ok = leveled_bookie:book_compactjournal(Bookie1, 30000), ok = leveled_bookie:book_compactjournal(Bookie1, 30000),
testutil:check_forlist(Bookie1, ChkList1), testutil:check_forlist(Bookie1, ChkList1),
testutil:check_forobject(Bookie1, TestObject), testutil:check_forobject(Bookie1, TestObject),
@ -140,17 +140,17 @@ journal_compaction(_Config) ->
%% Delete some of the objects %% Delete some of the objects
ObjListD = testutil:generate_objects(10000, 2), ObjListD = testutil:generate_objects(10000, 2),
lists:foreach(fun({_R, O, _S}) -> lists:foreach(fun({_R, O, _S}) ->
ok = leveled_bookie:book_riakdelete(Bookie1, ok = testutil:book_riakdelete(Bookie1,
O#r_object.bucket, O#r_object.bucket,
O#r_object.key, O#r_object.key,
[]) [])
end, end,
ObjListD), ObjListD),
%% Now replace all the other objects %% Now replace all the other objects
ObjList2 = testutil:generate_objects(40000, 10002), ObjList2 = testutil:generate_objects(40000, 10002),
lists:foreach(fun({_RN, Obj, Spc}) -> lists:foreach(fun({_RN, Obj, Spc}) ->
leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, testutil:book_riakput(Bookie1, Obj, Spc) end,
ObjList2), ObjList2),
ok = leveled_bookie:book_compactjournal(Bookie1, 30000), ok = leveled_bookie:book_compactjournal(Bookie1, 30000),
@ -184,10 +184,10 @@ fetchput_snapshot(_Config) ->
StartOpts1 = [{root_path, RootPath}, {max_journalsize, 30000000}], StartOpts1 = [{root_path, RootPath}, {max_journalsize, 30000000}],
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
{TestObject, TestSpec} = testutil:generate_testobject(), {TestObject, TestSpec} = testutil:generate_testobject(),
ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
ObjList1 = testutil:generate_objects(5000, 2), ObjList1 = testutil:generate_objects(5000, 2),
lists:foreach(fun({_RN, Obj, Spc}) -> lists:foreach(fun({_RN, Obj, Spc}) ->
leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, testutil:book_riakput(Bookie1, Obj, Spc) end,
ObjList1), ObjList1),
SnapOpts1 = [{snapshot_bookie, Bookie1}], SnapOpts1 = [{snapshot_bookie, Bookie1}],
{ok, SnapBookie1} = leveled_bookie:book_start(SnapOpts1), {ok, SnapBookie1} = leveled_bookie:book_start(SnapOpts1),
@ -212,7 +212,7 @@ fetchput_snapshot(_Config) ->
ObjList2 = testutil:generate_objects(5000, 2), ObjList2 = testutil:generate_objects(5000, 2),
lists:foreach(fun({_RN, Obj, Spc}) -> lists:foreach(fun({_RN, Obj, Spc}) ->
leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, testutil:book_riakput(Bookie2, Obj, Spc) end,
ObjList2), ObjList2),
io:format("Replacement objects put~n"), io:format("Replacement objects put~n"),
@ -226,7 +226,7 @@ fetchput_snapshot(_Config) ->
{ok, FNsA} = file:list_dir(RootPath ++ "/ledger/ledger_files"), {ok, FNsA} = file:list_dir(RootPath ++ "/ledger/ledger_files"),
ObjList3 = testutil:generate_objects(15000, 5002), ObjList3 = testutil:generate_objects(15000, 5002),
lists:foreach(fun({_RN, Obj, Spc}) -> lists:foreach(fun({_RN, Obj, Spc}) ->
leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, testutil:book_riakput(Bookie2, Obj, Spc) end,
ObjList3), ObjList3),
ChkList3 = lists:sublist(lists:sort(ObjList3), 100), ChkList3 = lists:sublist(lists:sort(ObjList3), 100),
testutil:check_forlist(Bookie2, ChkList3), testutil:check_forlist(Bookie2, ChkList3),
@ -291,7 +291,7 @@ load_and_count(_Config) ->
StartOpts1 = [{root_path, RootPath}, {max_journalsize, 50000000}], StartOpts1 = [{root_path, RootPath}, {max_journalsize, 50000000}],
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
{TestObject, TestSpec} = testutil:generate_testobject(), {TestObject, TestSpec} = testutil:generate_testobject(),
ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
testutil:check_forobject(Bookie1, TestObject), testutil:check_forobject(Bookie1, TestObject),
io:format("Loading initial small objects~n"), io:format("Loading initial small objects~n"),
G1 = fun testutil:generate_smallobjects/2, G1 = fun testutil:generate_smallobjects/2,
@ -374,7 +374,7 @@ load_and_count_withdelete(_Config) ->
StartOpts1 = [{root_path, RootPath}, {max_journalsize, 50000000}], StartOpts1 = [{root_path, RootPath}, {max_journalsize, 50000000}],
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
{TestObject, TestSpec} = testutil:generate_testobject(), {TestObject, TestSpec} = testutil:generate_testobject(),
ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
testutil:check_forobject(Bookie1, TestObject), testutil:check_forobject(Bookie1, TestObject),
io:format("Loading initial small objects~n"), io:format("Loading initial small objects~n"),
G1 = fun testutil:generate_smallobjects/2, G1 = fun testutil:generate_smallobjects/2,
@ -396,8 +396,8 @@ load_and_count_withdelete(_Config) ->
testutil:check_forobject(Bookie1, TestObject), testutil:check_forobject(Bookie1, TestObject),
{BucketD, KeyD} = leveled_codec:riakto_keydetails(TestObject), {BucketD, KeyD} = leveled_codec:riakto_keydetails(TestObject),
{_, 1} = testutil:check_bucket_stats(Bookie1, BucketD), {_, 1} = testutil:check_bucket_stats(Bookie1, BucketD),
ok = leveled_bookie:book_riakdelete(Bookie1, BucketD, KeyD, []), ok = testutil:book_riakdelete(Bookie1, BucketD, KeyD, []),
not_found = leveled_bookie:book_riakget(Bookie1, BucketD, KeyD), not_found = testutil:book_riakget(Bookie1, BucketD, KeyD),
{_, 0} = testutil:check_bucket_stats(Bookie1, BucketD), {_, 0} = testutil:check_bucket_stats(Bookie1, BucketD),
io:format("Loading larger compressible objects~n"), io:format("Loading larger compressible objects~n"),
G2 = fun testutil:generate_compressibleobjects/2, G2 = fun testutil:generate_compressibleobjects/2,
@ -416,7 +416,7 @@ load_and_count_withdelete(_Config) ->
Acc + 5000 end, Acc + 5000 end,
100000, 100000,
lists:seq(1, 20)), lists:seq(1, 20)),
not_found = leveled_bookie:book_riakget(Bookie1, BucketD, KeyD), not_found = testutil:book_riakget(Bookie1, BucketD, KeyD),
ok = leveled_bookie:book_close(Bookie1), ok = leveled_bookie:book_close(Bookie1),
{ok, Bookie2} = leveled_bookie:book_start(StartOpts1), {ok, Bookie2} = leveled_bookie:book_start(StartOpts1),
testutil:check_formissingobject(Bookie2, BucketD, KeyD), testutil:check_formissingobject(Bookie2, BucketD, KeyD),
@ -461,10 +461,10 @@ space_clear_ondelete(_Config) ->
% Delete the keys % Delete the keys
SW2 = os:timestamp(), SW2 = os:timestamp(),
lists:foreach(fun({Bucket, Key}) -> lists:foreach(fun({Bucket, Key}) ->
ok = leveled_bookie:book_riakdelete(Book1, ok = testutil:book_riakdelete(Book1,
Bucket, Bucket,
Key, Key,
[]) [])
end, end,
KL1), KL1),
io:format("Deletion took ~w microseconds for 80K keys~n", io:format("Deletion took ~w microseconds for 80K keys~n",

16
test/end_to_end/iterator_SUITE.erl
View file

@ -24,7 +24,7 @@ small_load_with2i(_Config) ->
% low journal size to make sure > 1 created % low journal size to make sure > 1 created
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
{TestObject, TestSpec} = testutil:generate_testobject(), {TestObject, TestSpec} = testutil:generate_testobject(),
ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
testutil:check_forobject(Bookie1, TestObject), testutil:check_forobject(Bookie1, TestObject),
testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"), testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"),
testutil:check_forobject(Bookie1, TestObject), testutil:check_forobject(Bookie1, TestObject),
@ -36,7 +36,7 @@ small_load_with2i(_Config) ->
ObjectGen, ObjectGen,
IndexGen), IndexGen),
lists:foreach(fun({_RN, Obj, Spc}) -> lists:foreach(fun({_RN, Obj, Spc}) ->
leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, testutil:book_riakput(Bookie1, Obj, Spc) end,
ObjL1), ObjL1),
ChkList1 = lists:sublist(lists:sort(ObjL1), 100), ChkList1 = lists:sublist(lists:sort(ObjL1), 100),
testutil:check_forlist(Bookie1, ChkList1), testutil:check_forlist(Bookie1, ChkList1),
@ -48,7 +48,7 @@ small_load_with2i(_Config) ->
end, end,
Spc), Spc),
{B, K} = leveled_codec:riakto_keydetails(Obj), {B, K} = leveled_codec:riakto_keydetails(Obj),
leveled_bookie:book_riakdelete(Bookie1, B, K, DSpc) testutil:book_riakdelete(Bookie1, B, K, DSpc)
end, end,
ChkList1), ChkList1),
%% Get the Buckets Keys and Hashes for the whole bucket %% Get the Buckets Keys and Hashes for the whole bucket
@ -116,7 +116,7 @@ query_count(_Config) ->
"Value1", "Value1",
[], [],
{"MDK1", "MDV1"}), {"MDK1", "MDV1"}),
ok = leveled_bookie:book_riakput(Book1, TestObject, TestSpec), ok = testutil:book_riakput(Book1, TestObject, TestSpec),
testutil:check_forobject(Book1, TestObject), testutil:check_forobject(Book1, TestObject),
testutil:check_formissingobject(Book1, "Bucket1", "Key2"), testutil:check_formissingobject(Book1, "Bucket1", "Key2"),
testutil:check_forobject(Book1, TestObject), testutil:check_forobject(Book1, TestObject),
@ -130,7 +130,7 @@ query_count(_Config) ->
V, V,
Indexes), Indexes),
lists:foreach(fun({_RN, Obj, Spc}) -> lists:foreach(fun({_RN, Obj, Spc}) ->
leveled_bookie:book_riakput(Book1, testutil:book_riakput(Book1,
Obj, Obj,
Spc) Spc)
end, end,
@ -243,7 +243,7 @@ query_count(_Config) ->
V9 = testutil:get_compressiblevalue(), V9 = testutil:get_compressiblevalue(),
Indexes9 = testutil:get_randomindexes_generator(8), Indexes9 = testutil:get_randomindexes_generator(8),
[{_RN, Obj9, Spc9}] = testutil:generate_objects(1, uuid, [], V9, Indexes9), [{_RN, Obj9, Spc9}] = testutil:generate_objects(1, uuid, [], V9, Indexes9),
ok = leveled_bookie:book_riakput(Book2, Obj9, Spc9), ok = testutil:book_riakput(Book2, Obj9, Spc9),
R9 = lists:map(fun({add, IdxF, IdxT}) -> R9 = lists:map(fun({add, IdxF, IdxT}) ->
R = leveled_bookie:book_returnfolder(Book2, R = leveled_bookie:book_returnfolder(Book2,
{index_query, {index_query,
@ -261,7 +261,7 @@ query_count(_Config) ->
Spc9), Spc9),
Spc9Del = lists:map(fun({add, IdxF, IdxT}) -> {remove, IdxF, IdxT} end, Spc9Del = lists:map(fun({add, IdxF, IdxT}) -> {remove, IdxF, IdxT} end,
Spc9), Spc9),
ok = leveled_bookie:book_riakput(Book2, Obj9, Spc9Del), ok = testutil:book_riakput(Book2, Obj9, Spc9Del),
lists:foreach(fun({IdxF, IdxT, X}) -> lists:foreach(fun({IdxF, IdxT, X}) ->
R = leveled_bookie:book_returnfolder(Book2, R = leveled_bookie:book_returnfolder(Book2,
{index_query, {index_query,
@ -294,7 +294,7 @@ query_count(_Config) ->
end end
end, end,
R9), R9),
ok = leveled_bookie:book_riakput(Book3, Obj9, Spc9), ok = testutil:book_riakput(Book3, Obj9, Spc9),
ok = leveled_bookie:book_close(Book3), ok = leveled_bookie:book_close(Book3),
{ok, Book4} = leveled_bookie:book_start(RootPath, 2000, 50000000), {ok, Book4} = leveled_bookie:book_start(RootPath, 2000, 50000000),
lists:foreach(fun({IdxF, IdxT, X}) -> lists:foreach(fun({IdxF, IdxT, X}) ->

12
test/end_to_end/recovery_SUITE.erl
View file

@ -47,7 +47,7 @@ aae_bustedjournal(_Config) ->
{max_journalsize, 20000000}], {max_journalsize, 20000000}],
{ok, Bookie1} = leveled_bookie:book_start(StartOpts), {ok, Bookie1} = leveled_bookie:book_start(StartOpts),
{TestObject, TestSpec} = testutil:generate_testobject(), {TestObject, TestSpec} = testutil:generate_testobject(),
ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
testutil:check_forobject(Bookie1, TestObject), testutil:check_forobject(Bookie1, TestObject),
GenList = [2], GenList = [2],
_CLs = testutil:load_objects(20000, GenList, Bookie1, TestObject, _CLs = testutil:load_objects(20000, GenList, Bookie1, TestObject,
@ -64,7 +64,7 @@ aae_bustedjournal(_Config) ->
KeyList = KeyF(), KeyList = KeyF(),
20001 = length(KeyList), 20001 = length(KeyList),
HeadCount = lists:foldl(fun({B, K}, Acc) -> HeadCount = lists:foldl(fun({B, K}, Acc) ->
case leveled_bookie:book_riakhead(Bookie2, case testutil:book_riakhead(Bookie2,
B, B,
K) of K) of
{ok, _} -> Acc + 1; {ok, _} -> Acc + 1;
@ -75,7 +75,7 @@ aae_bustedjournal(_Config) ->
KeyList), KeyList),
20001 = HeadCount, 20001 = HeadCount,
GetCount = lists:foldl(fun({B, K}, Acc) -> GetCount = lists:foldl(fun({B, K}, Acc) ->
case leveled_bookie:book_riakget(Bookie2, case testutil:book_riakget(Bookie2,
B, B,
K) of K) of
{ok, _} -> Acc + 1; {ok, _} -> Acc + 1;
@ -199,16 +199,16 @@ journal_compaction_bustedjournal(_Config) ->
{max_run_length, 10}], {max_run_length, 10}],
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
{TestObject, TestSpec} = testutil:generate_testobject(), {TestObject, TestSpec} = testutil:generate_testobject(),
ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
testutil:check_forobject(Bookie1, TestObject), testutil:check_forobject(Bookie1, TestObject),
ObjList1 = testutil:generate_objects(50000, 2), ObjList1 = testutil:generate_objects(50000, 2),
lists:foreach(fun({_RN, Obj, Spc}) -> lists:foreach(fun({_RN, Obj, Spc}) ->
leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, testutil:book_riakput(Bookie1, Obj, Spc) end,
ObjList1), ObjList1),
%% Now replace all the objects %% Now replace all the objects
ObjList2 = testutil:generate_objects(50000, 2), ObjList2 = testutil:generate_objects(50000, 2),
lists:foreach(fun({_RN, Obj, Spc}) -> lists:foreach(fun({_RN, Obj, Spc}) ->
leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, testutil:book_riakput(Bookie1, Obj, Spc) end,
ObjList2), ObjList2),
ok = leveled_bookie:book_close(Bookie1), ok = leveled_bookie:book_close(Bookie1),

64
test/end_to_end/testutil.erl
View file

@ -2,7 +2,11 @@
-include("../include/leveled.hrl"). -include("../include/leveled.hrl").
-export([reset_filestructure/0, -export([book_riakput/3,
book_riakdelete/4,
book_riakget/3,
book_riakhead/3,
reset_filestructure/0,
reset_filestructure/1, reset_filestructure/1,
check_bucket_stats/2, check_bucket_stats/2,
check_forlist/2, check_forlist/2,
@ -39,6 +43,24 @@
-define(RETURN_TERMS, {true, undefined}). -define(RETURN_TERMS, {true, undefined}).
book_riakput(Pid, RiakObject, IndexSpecs) ->
{Bucket, Key} = leveled_codec:riakto_keydetails(RiakObject),
leveled_book:book_put(Pid, Bucket, Key, RiakObject, IndexSpecs, ?RIAK_TAG).
book_riakdelete(Pid, Bucket, Key, IndexSpecs) ->
leveled_book:book_put(Pid, Bucket, Key, delete, IndexSpecs, ?RIAK_TAG).
book_riakget(Pid, Bucket, Key) ->
leveled_book:book_get(Pid, Bucket, Key, ?RIAK_TAG).
book_riakhead(Pid, Bucket, Key) ->
leveled_book:book_head(Pid, Bucket, Key, ?RIAK_TAG).
reset_filestructure() -> reset_filestructure() ->
reset_filestructure(0). reset_filestructure(0).
@ -81,9 +103,9 @@ check_forlist(Bookie, ChkList, Log) ->
true -> true ->
ok ok
end, end,
R = leveled_bookie:book_riakget(Bookie, R = book_riakget(Bookie,
Obj#r_object.bucket, Obj#r_object.bucket,
Obj#r_object.key), Obj#r_object.key),
ok = case R of ok = case R of
{ok, Obj} -> {ok, Obj} ->
ok; ok;
@ -100,21 +122,21 @@ check_forlist(Bookie, ChkList, Log) ->
check_formissinglist(Bookie, ChkList) -> check_formissinglist(Bookie, ChkList) ->
SW = os:timestamp(), SW = os:timestamp(),
lists:foreach(fun({_RN, Obj, _Spc}) -> lists:foreach(fun({_RN, Obj, _Spc}) ->
R = leveled_bookie:book_riakget(Bookie, R = book_riakget(Bookie,
Obj#r_object.bucket, Obj#r_object.bucket,
Obj#r_object.key), Obj#r_object.key),
R = not_found end, R = not_found end,
ChkList), ChkList),
io:format("Miss check took ~w microseconds checking list of length ~w~n", io:format("Miss check took ~w microseconds checking list of length ~w~n",
[timer:now_diff(os:timestamp(), SW), length(ChkList)]). [timer:now_diff(os:timestamp(), SW), length(ChkList)]).
check_forobject(Bookie, TestObject) -> check_forobject(Bookie, TestObject) ->
{ok, TestObject} = leveled_bookie:book_riakget(Bookie, {ok, TestObject} = book_riakget(Bookie,
TestObject#r_object.bucket, TestObject#r_object.bucket,
TestObject#r_object.key), TestObject#r_object.key),
{ok, HeadObject} = leveled_bookie:book_riakhead(Bookie, {ok, HeadObject} = book_riakhead(Bookie,
TestObject#r_object.bucket, TestObject#r_object.bucket,
TestObject#r_object.key), TestObject#r_object.key),
ok = case {HeadObject#r_object.bucket, ok = case {HeadObject#r_object.bucket,
HeadObject#r_object.key, HeadObject#r_object.key,
HeadObject#r_object.vclock} of HeadObject#r_object.vclock} of
@ -125,8 +147,8 @@ check_forobject(Bookie, TestObject) ->
end. end.
check_formissingobject(Bookie, Bucket, Key) -> check_formissingobject(Bookie, Bucket, Key) ->
not_found = leveled_bookie:book_riakget(Bookie, Bucket, Key), not_found = book_riakget(Bookie, Bucket, Key),
not_found = leveled_bookie:book_riakhead(Bookie, Bucket, Key). not_found = book_riakhead(Bookie, Bucket, Key).
generate_testobject() -> generate_testobject() ->
@ -236,7 +258,7 @@ load_objects(ChunkSize, GenList, Bookie, TestObject, Generator) ->
ObjListA = Generator(ChunkSize, KN), ObjListA = Generator(ChunkSize, KN),
StartWatchA = os:timestamp(), StartWatchA = os:timestamp(),
lists:foreach(fun({_RN, Obj, Spc}) -> lists:foreach(fun({_RN, Obj, Spc}) ->
leveled_bookie:book_riakput(Bookie, Obj, Spc) book_riakput(Bookie, Obj, Spc)
end, end,
ObjListA), ObjListA),
Time = timer:now_diff(os:timestamp(), StartWatchA), Time = timer:now_diff(os:timestamp(), StartWatchA),
@ -289,7 +311,7 @@ check_indexed_objects(Book, B, KSpecL, V) ->
% Check all objects match, return what should be the results of an all % Check all objects match, return what should be the results of an all
% index query % index query
IdxR = lists:map(fun({K, Spc}) -> IdxR = lists:map(fun({K, Spc}) ->
{ok, O} = leveled_bookie:book_riakget(Book, B, K), {ok, O} = book_riakget(Book, B, K),
V = testutil:get_value(O), V = testutil:get_value(O),
{add, {add,
"idx1_bin", "idx1_bin",
@ -333,9 +355,7 @@ put_indexed_objects(Book, Bucket, Count) ->
IndexGen, IndexGen,
Bucket), Bucket),
KSpecL = lists:map(fun({_RN, Obj, Spc}) -> KSpecL = lists:map(fun({_RN, Obj, Spc}) ->
leveled_bookie:book_riakput(Book, book_riakput(Book, Obj, Spc),
Obj,
Spc),
{testutil:get_key(Obj), Spc} {testutil:get_key(Obj), Spc}
end, end,
ObjL1), ObjL1),
@ -364,9 +384,7 @@ put_altered_indexed_objects(Book, Bucket, KSpecL, RemoveOld2i) ->
V, V,
IndexGen, IndexGen,
AddSpc), AddSpc),
ok = leveled_bookie:book_riakput(Book, ok = book_riakput(Book, O, AltSpc),
O,
AltSpc),
{K, AltSpc} end, {K, AltSpc} end,
KSpecL), KSpecL),
{RplKSpecL, V}. {RplKSpecL, V}.