Merge pull request #130 from russelldb/rdb/backend_eqc_test
WIP: changes for backend_eqc test
This commit is contained in:
Martin Sumner
GitHub
commit
e182f8c95c
4 changed files with 174 additions and 47 deletions
|
|
@ -915,6 +915,9 @@ get_runner(State, {keylist, Tag, FoldAccT}) ->
|
|||
get_runner(State, {keylist, Tag, Bucket, FoldAccT}) ->
|
||||
SnapFun = return_snapfun(State, ledger, no_lookup, true, true),
|
||||
leveled_runner:bucketkey_query(SnapFun, Tag, Bucket, FoldAccT);
|
||||
get_runner(State, {keylist, Tag, Bucket, KeyRange, FoldAccT}) ->
|
||||
SnapFun = return_snapfun(State, ledger, no_lookup, true, true),
|
||||
leveled_runner:bucketkey_query(SnapFun, Tag, Bucket, KeyRange, FoldAccT);
|
||||
|
||||
%% Set of runners for object or metadata folds
|
||||
get_runner(State,
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@
|
|||
binary_bucketlist/5,
|
||||
index_query/3,
|
||||
bucketkey_query/4,
|
||||
bucketkey_query/5,
|
||||
hashlist_query/3,
|
||||
tictactree/5,
|
||||
foldheads_allkeys/5,
|
||||
|
|
@ -41,6 +42,8 @@
|
|||
|
||||
-define(CHECKJOURNAL_PROB, 0.2).
|
||||
|
||||
-type key_range() :: {StartKey:: any(), EndKey :: any()}.
|
||||
|
||||
%%%============================================================================
|
||||
%%% External functions
|
||||
%%%============================================================================
|
||||
|
|
@ -118,12 +121,12 @@ index_query(SnapFun, {StartKey, EndKey, TermHandling}, FoldAccT) ->
|
|||
end,
|
||||
{async, Runner}.
|
||||
|
||||
-spec bucketkey_query(fun(), atom(), any(), tuple()) -> {async, fun()}.
|
||||
-spec bucketkey_query(fun(), atom(), any(), key_range(), tuple()) -> {async, fun()}.
|
||||
%% @doc
|
||||
%% Fold over all keys under tak (potentially restricted to a given bucket)
|
||||
bucketkey_query(SnapFun, Tag, Bucket, {FoldKeysFun, InitAcc}) ->
|
||||
SK = leveled_codec:to_ledgerkey(Bucket, null, Tag),
|
||||
EK = leveled_codec:to_ledgerkey(Bucket, null, Tag),
|
||||
%% Fold over all keys in `KeyRange' under tag (restricted to a given bucket)
|
||||
bucketkey_query(SnapFun, Tag, Bucket, {StartKey, EndKey}, {FoldKeysFun, InitAcc}) ->
|
||||
SK = leveled_codec:to_ledgerkey(Bucket, StartKey, Tag),
|
||||
EK = leveled_codec:to_ledgerkey(Bucket, EndKey, Tag),
|
||||
AccFun = accumulate_keys(FoldKeysFun),
|
||||
Runner =
|
||||
fun() ->
|
||||
|
|
@ -138,6 +141,12 @@ bucketkey_query(SnapFun, Tag, Bucket, {FoldKeysFun, InitAcc}) ->
|
|||
end,
|
||||
{async, Runner}.
|
||||
|
||||
-spec bucketkey_query(fun(), atom(), any(), tuple()) -> {async, fun()}.
|
||||
%% @doc
|
||||
%% Fold over all keys under tag (potentially restricted to a given bucket)
|
||||
bucketkey_query(SnapFun, Tag, Bucket, FunAcc) ->
|
||||
bucketkey_query(SnapFun, Tag, Bucket, {null, null}, FunAcc).
|
||||
|
||||
-spec hashlist_query(fun(), atom(), boolean()) -> {async, fun()}.
|
||||
%% @doc
|
||||
%% Fold pver the key accumulating the hashes
|
||||
|
|
|
|||
|
|
@ -4,12 +4,16 @@
|
|||
-export([all/0]).
|
||||
-export([
|
||||
crossbucket_aae/1,
|
||||
handoff/1
|
||||
handoff/1,
|
||||
dollar_bucket_index/1,
|
||||
dollar_key_index/1
|
||||
]).
|
||||
|
||||
all() -> [
|
||||
crossbucket_aae,
|
||||
handoff
|
||||
handoff,
|
||||
dollar_bucket_index,
|
||||
dollar_key_index
|
||||
].
|
||||
|
||||
-define(MAGIC, 53). % riak_kv -> riak_object
|
||||
|
|
@ -419,3 +423,87 @@ handoff(_Config) ->
|
|||
ok = leveled_bookie:book_close(Bookie2),
|
||||
ok = leveled_bookie:book_close(Bookie3),
|
||||
ok = leveled_bookie:book_close(Bookie4).
|
||||
|
||||
%% @doc test that the riak specific $key index can be iterated using
|
||||
%% leveled's existing folders
|
||||
dollar_key_index(_Config) ->
|
||||
RootPath = testutil:reset_filestructure(),
|
||||
{ok, Bookie1} = leveled_bookie:book_start(RootPath,
|
||||
2000,
|
||||
50000000,
|
||||
testutil:sync_strategy()),
|
||||
ObjectGen = testutil:get_compressiblevalue_andinteger(),
|
||||
IndexGen = fun() -> [] end,
|
||||
ObjL1 = testutil:generate_objects(1300,
|
||||
{fixed_binary, 1},
|
||||
[],
|
||||
ObjectGen,
|
||||
IndexGen,
|
||||
<<"Bucket1">>),
|
||||
testutil:riakload(Bookie1, ObjL1),
|
||||
|
||||
FoldKeysFun = fun(_B, K, Acc) ->
|
||||
[ K |Acc]
|
||||
end,
|
||||
|
||||
StartKey = testutil:fixed_bin_key(123),
|
||||
EndKey = testutil:fixed_bin_key(779),
|
||||
|
||||
Query = {keylist, ?RIAK_TAG, <<"Bucket1">>, {StartKey, EndKey}, {FoldKeysFun, []}},
|
||||
|
||||
{async, Folder} = leveled_bookie:book_returnfolder(Bookie1, Query),
|
||||
ResLen = length(Folder()),
|
||||
io:format("Length of Result of folder ~w~n", [ResLen]),
|
||||
true = 657 == ResLen,
|
||||
|
||||
ok = leveled_bookie:book_close(Bookie1),
|
||||
testutil:reset_filestructure().
|
||||
|
||||
%% @doc test that the riak specific $bucket indexes can be iterated
|
||||
%% using leveled's existing folders
|
||||
dollar_bucket_index(_Config) ->
|
||||
RootPath = testutil:reset_filestructure(),
|
||||
{ok, Bookie1} = leveled_bookie:book_start(RootPath,
|
||||
2000,
|
||||
50000000,
|
||||
testutil:sync_strategy()),
|
||||
ObjectGen = testutil:get_compressiblevalue_andinteger(),
|
||||
IndexGen = fun() -> [] end,
|
||||
ObjL1 = testutil:generate_objects(1300,
|
||||
uuid,
|
||||
[],
|
||||
ObjectGen,
|
||||
IndexGen,
|
||||
<<"Bucket1">>),
|
||||
testutil:riakload(Bookie1, ObjL1),
|
||||
ObjL2 = testutil:generate_objects(1700,
|
||||
uuid,
|
||||
[],
|
||||
ObjectGen,
|
||||
IndexGen,
|
||||
<<"Bucket2">>),
|
||||
testutil:riakload(Bookie1, ObjL2),
|
||||
ObjL3 = testutil:generate_objects(7000,
|
||||
uuid,
|
||||
[],
|
||||
ObjectGen,
|
||||
IndexGen,
|
||||
<<"Bucket3">>),
|
||||
|
||||
testutil:riakload(Bookie1, ObjL3),
|
||||
|
||||
FoldKeysFun = fun(B, K, Acc) ->
|
||||
[{B, K}|Acc]
|
||||
end,
|
||||
|
||||
Query = {keylist, ?RIAK_TAG, <<"Bucket2">>, {FoldKeysFun, []}},
|
||||
|
||||
{async, Folder} = leveled_bookie:book_returnfolder(Bookie1, Query),
|
||||
ResLen = length(Folder()),
|
||||
|
||||
io:format("Length of Result of folder ~w~n", [ResLen]),
|
||||
|
||||
true = 1700 == ResLen,
|
||||
|
||||
ok = leveled_bookie:book_close(Bookie1),
|
||||
testutil:reset_filestructure().
|
||||
|
|
|
|||
|
|
@ -48,7 +48,9 @@
|
|||
wait_for_compaction/1,
|
||||
foldkeysfun/3,
|
||||
foldkeysfun_returnbucket/3,
|
||||
sync_strategy/0]).
|
||||
sync_strategy/0,
|
||||
numbered_key/1,
|
||||
fixed_bin_key/1]).
|
||||
|
||||
-define(RETURN_TERMS, {true, undefined}).
|
||||
-define(SLOWOFFER_DELAY, 5).
|
||||
|
|
@ -398,7 +400,7 @@ generate_objects(Count, uuid, ObjL, Value, IndexGen, Bucket) ->
|
|||
generate_objects(Count, {binary, KeyNumber}, ObjL, Value, IndexGen, Bucket) ->
|
||||
{Obj1, Spec1} =
|
||||
set_object(list_to_binary(Bucket),
|
||||
list_to_binary("Key" ++ integer_to_list(KeyNumber)),
|
||||
list_to_binary(numbered_key(KeyNumber)),
|
||||
Value,
|
||||
IndexGen),
|
||||
generate_objects(Count - 1,
|
||||
|
|
@ -407,9 +409,21 @@ generate_objects(Count, {binary, KeyNumber}, ObjL, Value, IndexGen, Bucket) ->
|
|||
Value,
|
||||
IndexGen,
|
||||
Bucket);
|
||||
generate_objects(Count, {fixed_binary, KeyNumber}, ObjL, Value, IndexGen, Bucket) ->
|
||||
{Obj1, Spec1} =
|
||||
set_object(Bucket,
|
||||
fixed_bin_key(KeyNumber),
|
||||
Value,
|
||||
IndexGen),
|
||||
generate_objects(Count - 1,
|
||||
{fixed_binary, KeyNumber + 1},
|
||||
ObjL ++ [{leveled_rand:uniform(), Obj1, Spec1}],
|
||||
Value,
|
||||
IndexGen,
|
||||
Bucket);
|
||||
generate_objects(Count, KeyNumber, ObjL, Value, IndexGen, Bucket) ->
|
||||
{Obj1, Spec1} = set_object(Bucket,
|
||||
"Key" ++ integer_to_list(KeyNumber),
|
||||
numbered_key(KeyNumber),
|
||||
Value,
|
||||
IndexGen),
|
||||
generate_objects(Count - 1,
|
||||
|
|
@ -419,6 +433,19 @@ generate_objects(Count, KeyNumber, ObjL, Value, IndexGen, Bucket) ->
|
|||
IndexGen,
|
||||
Bucket).
|
||||
|
||||
%% @doc generates a key, exported so tests can use it without copying
|
||||
%% code
|
||||
-spec numbered_key(integer()) -> list().
|
||||
numbered_key(KeyNumber) when is_integer(KeyNumber) ->
|
||||
"Key" ++ integer_to_list(KeyNumber).
|
||||
|
||||
%% @doc generates a key for `KeyNumber' of a fixed size (64bits),
|
||||
%% again, exported for tests to generate the same keys as
|
||||
%% generate_objects/N without peeking.
|
||||
-spec fixed_bin_key(integer()) -> binary().
|
||||
fixed_bin_key(KeyNumber) ->
|
||||
<<$K, $e, $y, KeyNumber:64/integer>>.
|
||||
|
||||
set_object(Bucket, Key, Value, IndexGen) ->
|
||||
set_object(Bucket, Key, Value, IndexGen, []).
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue