umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Split code for folders - introduce runner actor

Browse source 
Introduce a dedicated module for all the different fold types.  Also simplify the list of folders by deprecating those folds that should eb achieveable by fold_heads/fold_objects type folds but with smarter functions.

Makes sure that the fold functiosn also have better spec coverage, and are dialyzer checked.
This commit is contained in:
Martin Sumner 2017-10-17 20:39:11 +01:00
parent d0b8e47f77
commit bfaed921e6
7 changed files with 817 additions and 735 deletions
Download patch file Download diff file Expand all files Collapse all files

14
test/end_to_end/basic_SUITE.erl
View file

@ -156,11 +156,8 @@ journal_compaction(_Config) ->
testutil:wait_for_compaction(Bookie1),
% Start snapshot - should not stop deletions
{ok,
PclClone,
InkClone} = leveled_bookie:book_snapshotstore(Bookie1,
self(),
300000),
{ok, PclClone, InkClone} =
leveled_bookie:book_snapshot(Bookie1, store, undefined, false),
% Wait 2 seconds for files to be deleted
WasteFP = RootPath ++ "/journal/journal_files/waste",
lists:foldl(fun(X, Found) ->
@ -500,7 +497,8 @@ space_clear_ondelete(_Config) ->
{async, HTreeF1} = leveled_bookie:book_returnfolder(Book1,
{foldobjects_allkeys,
?RIAK_TAG,
FoldObjectsFun}),
FoldObjectsFun,
false}),
{async, KF1} = leveled_bookie:book_returnfolder(Book1, AllKeyQuery),
% Delete the keys
@ -534,8 +532,8 @@ space_clear_ondelete(_Config) ->
io:format("Waiting for journal deletes - blocked~n"),
timer:sleep(20000),
% for this query snapshot is made at fold time - EDIT now uses a snapshot!
true = length(HTreeF1()) == 80000,
% for this query snapshot is made at fold time
true = length(HTreeF1()) == 0,
% This query uses a genuine async fold on a snasphot made at request time
true = length(KF1()) == 80000,

13
test/end_to_end/iterator_SUITE.erl
View file

@ -128,13 +128,16 @@ small_load_with2i(_Config) ->
{async, HTreeF1} = leveled_bookie:book_returnfolder(Bookie1,
{foldobjects_allkeys,
?RIAK_TAG,
FoldObjectsFun}),
FoldObjectsFun,
false}),
KeyHashList1 = HTreeF1(),
{async, HTreeF2} = leveled_bookie:book_returnfolder(Bookie1,
{foldobjects_bybucket,
?RIAK_TAG,
"Bucket",
FoldObjectsFun}),
all,
FoldObjectsFun,
false}),
KeyHashList2 = HTreeF2(),
{async, HTreeF3} = leveled_bookie:book_returnfolder(Bookie1,
{foldobjects_byindex,
@ -142,7 +145,8 @@ small_load_with2i(_Config) ->
"Bucket",
{"idx1_bin",
"#", "|"},
FoldObjectsFun}),
FoldObjectsFun,
false}),
KeyHashList3 = HTreeF3(),
true = 9901 == length(KeyHashList1), % also includes the test object
true = 9900 == length(KeyHashList2),
@ -152,7 +156,8 @@ small_load_with2i(_Config) ->
{I, _Bin} = testutil:get_value(Obj),
Acc + I
end,
BucketObjQ = {foldobjects_bybucket, ?RIAK_TAG, "Bucket", {SumIntFun, 0}},
BucketObjQ =
{foldobjects_bybucket, ?RIAK_TAG, "Bucket", all, {SumIntFun, 0}, true},
{async, Sum1} = leveled_bookie:book_returnfolder(Bookie1, BucketObjQ),
Total1 = Sum1(),
true = Total1 > 100000,

16
test/end_to_end/recovery_SUITE.erl
View file

@ -202,7 +202,7 @@ aae_bustedjournal(_Config) ->
{async, HashTreeF2} = leveled_bookie:book_returnfolder(Bookie2,
{hashlist_query,
?RIAK_TAG,
check_presence}),
true}),
KeyHashList2 = HashTreeF2(),
% The file is still there, and the hashtree is not corrupted
KeyHashList2 = KeyHashList1,
@ -219,7 +219,8 @@ aae_bustedjournal(_Config) ->
{async, HashTreeF3} = leveled_bookie:book_returnfolder(Bookie2,
{foldobjects_allkeys,
?RIAK_TAG,
FoldObjectsFun}),
FoldObjectsFun,
false}),
KeyHashList3 = HashTreeF3(),
true = length(KeyHashList3) > 19000,
@ -241,7 +242,8 @@ aae_bustedjournal(_Config) ->
{async, HashTreeF4} = leveled_bookie:book_returnfolder(Bookie3,
{foldobjects_allkeys,
?RIAK_TAG,
FoldObjectsFun}),
FoldObjectsFun,
false}),
KeyHashList4 = HashTreeF4(),
true = length(KeyHashList4) == 20001,
@ -258,7 +260,8 @@ aae_bustedjournal(_Config) ->
{async, HashTreeF5} = leveled_bookie:book_returnfolder(Bookie4,
{foldobjects_allkeys,
?RIAK_TAG,
FoldObjectsFun}),
FoldObjectsFun,
false}),
KeyHashList5 = HashTreeF5(),
true = length(KeyHashList5) > 19000,
@ -274,7 +277,7 @@ aae_bustedjournal(_Config) ->
{async, HashTreeF6} = leveled_bookie:book_returnfolder(Bookie4,
{hashlist_query,
?RIAK_TAG,
check_presence}),
true}),
KeyHashList6 = HashTreeF6(),
true = length(KeyHashList6) > 19000,
true = length(KeyHashList6) < HeadCount,
@ -289,7 +292,8 @@ aae_bustedjournal(_Config) ->
{async, HashTreeF7} = leveled_bookie:book_returnfolder(Bookie5,
{foldobjects_allkeys,
?RIAK_TAG,
FoldObjectsFun}),
FoldObjectsFun,
false}),
KeyHashList7 = HashTreeF7(),
true = length(KeyHashList7) == 20001,