d45356a4f7
* Extend perf_SUITE This is v6 of the perf_SUITE tests. The test adds a complex index entry to every object, and then adds a new test phase to test regex queries. There are three profiles added so the full, mini and profiling versions of perf_SUITE can be run without having to edit the file itself: e.g. ./rebar3 as perf_mini do ct --suite=test/end_to_end/perf_SUITE When testing as `perf_prof` summarised versions of the eprof results are now printed to screen. The volume of keys within the full test suite has been dropped ... just to make life easier so that test run times are not excessively increase by the new features. * Load chunk in spawned processes Assume to make the job of gs easier - name makes a massive difference to load time in OTP 24. * Correctly account for pause alos try and improve test stability by increasing pause * Add microstate accounting to profile * Add memory tracking during test phases Identify and log out memory usage by test phase * Use macros instead (#437) * Don't print memory to screen in standard ct test --------- Co-authored-by: Thomas Arts <thomas.arts@quviq.com>
1001 lines
38 KiB
Erlang
1001 lines
38 KiB
Erlang
-module(iterator_SUITE).
|
|
|
|
-include_lib("common_test/include/ct.hrl").
|
|
-include("include/leveled.hrl").
|
|
|
|
-define(KEY_ONLY, {false, undefined}).
|
|
|
|
-export([all/0]).
|
|
-export([expiring_indexes/1,
|
|
breaking_folds/1,
|
|
single_object_with2i/1,
|
|
small_load_with2i/1,
|
|
query_count/1,
|
|
multibucket_fold/1,
|
|
foldobjects_bybucket_range/1,
|
|
rotating_objects/1]).
|
|
|
|
all() -> [
|
|
expiring_indexes,
|
|
breaking_folds,
|
|
single_object_with2i,
|
|
small_load_with2i,
|
|
query_count,
|
|
multibucket_fold,
|
|
rotating_objects,
|
|
foldobjects_bybucket_range
|
|
].
|
|
|
|
|
|
expiring_indexes(_Config) ->
|
|
% Add objects to the store with index entries, where the objects (and hence
|
|
% the indexes have an expiry time. Confirm that the indexes and the
|
|
% objects are no longer present after the expiry time (and are present
|
|
% before). Confirm that replacing an object has the expected outcome, if
|
|
% the IndexSpecs are updated as part of the request.
|
|
KeyCount = 50000,
|
|
Future = 120,
|
|
% 2 minutes - if running tests on a slow machine, may need to increase
|
|
% this value
|
|
RootPath = testutil:reset_filestructure(),
|
|
StartOpts1 =
|
|
[{root_path, RootPath},
|
|
{max_pencillercachesize, 16000},
|
|
{max_journalobjectcount, 30000},
|
|
{sync_strategy, testutil:sync_strategy()}],
|
|
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
|
|
|
|
SW1 = os:timestamp(),
|
|
timer:sleep(1000),
|
|
|
|
V9 = testutil:get_compressiblevalue(),
|
|
Indexes9 = testutil:get_randomindexes_generator(2),
|
|
TempRiakObjects =
|
|
testutil:generate_objects(
|
|
KeyCount, binary_uuid, [], V9, Indexes9, "riakBucket"),
|
|
|
|
IBKL1 = testutil:stdload_expiring(Bookie1, KeyCount, Future),
|
|
lists:foreach(
|
|
fun({_RN, Obj, Spc}) ->
|
|
testutil:book_tempriakput(
|
|
Bookie1, Obj, Spc, leveled_util:integer_now() + Future)
|
|
end,
|
|
TempRiakObjects
|
|
),
|
|
timer:sleep(1000),
|
|
% Wait a second after last key so that none loaded in the last second
|
|
LoadTime = timer:now_diff(os:timestamp(), SW1)/1000000,
|
|
io:format("Load of ~w std objects in ~w seconds~n", [KeyCount, LoadTime]),
|
|
|
|
timer:sleep(1000),
|
|
SW2 = os:timestamp(),
|
|
|
|
FilterFun = fun({I, _B, _K}) -> lists:member(I, [5, 6, 7, 8]) end,
|
|
LoadedEntriesInRange = lists:sort(lists:filter(FilterFun, IBKL1)),
|
|
|
|
true = LoadTime < (Future - 20),
|
|
% need 20 seconds spare to run query
|
|
% and add replacements
|
|
|
|
{I0, B0, K0} = hd(IBKL1),
|
|
false = FilterFun(hd(IBKL1)),
|
|
% The head entry should not have index between 5 and 8
|
|
|
|
CountI0Fold =
|
|
fun() ->
|
|
leveled_bookie:book_indexfold(
|
|
Bookie1,
|
|
B0,
|
|
{fun(_BF, _KT, Acc) -> Acc + 1 end, 0},
|
|
{<<"temp_int">>, I0, I0},
|
|
{true, undefined})
|
|
end,
|
|
{async, I0Counter1} = CountI0Fold(),
|
|
I0Count1 = I0Counter1(),
|
|
|
|
HeadFold =
|
|
fun(LowTS, HighTS) ->
|
|
leveled_bookie:book_headfold(
|
|
Bookie1,
|
|
?RIAK_TAG,
|
|
{range, <<"riakBucket">>, all},
|
|
{fun(_B, _K, _V, Acc) -> Acc + 1 end, 0},
|
|
false, true, false,
|
|
{testutil:convert_to_seconds(LowTS),
|
|
testutil:convert_to_seconds(HighTS)},
|
|
false
|
|
)
|
|
end,
|
|
{async, HeadCount0Fun} = HeadFold(SW1, SW2),
|
|
{async, HeadCount1Fun} = HeadFold(SW2, os:timestamp()),
|
|
HeadCounts = {HeadCount0Fun(), HeadCount1Fun()},
|
|
io:format("HeadCounts ~w before expiry~n", [HeadCounts]),
|
|
{KeyCount, 0} = HeadCounts,
|
|
|
|
FoldFun = fun(BF, {IdxV, KeyF}, Acc) -> [{IdxV, BF, KeyF}|Acc] end,
|
|
InitAcc = [],
|
|
IndexFold =
|
|
fun() ->
|
|
leveled_bookie:book_indexfold(
|
|
Bookie1,
|
|
B0,
|
|
{FoldFun, InitAcc},
|
|
{<<"temp_int">>, 5, 8},
|
|
{true, undefined})
|
|
end,
|
|
|
|
{async, Folder1} = IndexFold(),
|
|
QR1 = Folder1(),
|
|
true = lists:sort(QR1) == LoadedEntriesInRange,
|
|
% Replace object with one with an index value of 6
|
|
testutil:stdload_object(
|
|
Bookie1, B0, K0, 6, <<"value">>, leveled_util:integer_now() + 600),
|
|
% Confirm that this has reduced the index entries in I0 by 1
|
|
{async, I0Counter2} = CountI0Fold(),
|
|
I0Count2 = I0Counter2(),
|
|
io:format("Count with index value ~w changed from ~w to ~w~n",
|
|
[I0, I0Count1, I0Count2]),
|
|
true = I0Count2 == (I0Count1 - 1),
|
|
% Now replace again, shortening the timeout to 10s,
|
|
% this time index value of 6
|
|
testutil:stdload_object(
|
|
Bookie1, B0, K0, 5, <<"value">>, leveled_util:integer_now() + 10),
|
|
timer:sleep(1000),
|
|
{async, Folder2} = IndexFold(),
|
|
leveled_bookie:book_indexfold(Bookie1,
|
|
B0,
|
|
{FoldFun, InitAcc},
|
|
{<<"temp_int">>, 5, 8},
|
|
{true, undefined}),
|
|
QR2 = Folder2(),
|
|
io:format("Query with additional entry length ~w~n", [length(QR2)]),
|
|
true = lists:sort(QR2) == lists:sort([{5, B0, K0}|LoadedEntriesInRange]),
|
|
% Wait for a 10s timeout plus a second to be sure
|
|
timer:sleep(10000 + 1000),
|
|
{async, Folder3} = IndexFold(),
|
|
QR3 = Folder3(),
|
|
% Now the entry should be missing (and the 600s TTL entry should not have
|
|
% resurfaced)
|
|
io:format("Query results length ~w following sleep~n", [length(QR3)]),
|
|
true = lists:sort(QR3) == LoadedEntriesInRange,
|
|
|
|
FoldTime = timer:now_diff(os:timestamp(), SW1)/1000000 - LoadTime,
|
|
io:format("Query returned ~w entries in ~w seconds - 3 queries + 10s wait~n",
|
|
[length(QR3), FoldTime]),
|
|
|
|
SleepTime =
|
|
(Future - (timer:now_diff(os:timestamp(), SW2) div (1000 * 1000))) + 1,
|
|
|
|
io:format("Sleeping ~w s for all to expire~n", [SleepTime]),
|
|
timer:sleep(SleepTime * 1000),
|
|
|
|
% Index entries should now have expired
|
|
{async, Folder4} = IndexFold(),
|
|
QR4 = Folder4(),
|
|
io:format("QR4 Unexpired indexes of length ~w~n", [length(QR4)]),
|
|
timer:sleep(1000),
|
|
{async, Folder5} = IndexFold(),
|
|
QR5 = Folder5(),
|
|
io:format("QR5 Unexpired indexes of length ~w~n", [length(QR5)]),
|
|
|
|
true = QR4 == [],
|
|
true = QR5 == [],
|
|
|
|
{async, HeadCount0ExpFun} = HeadFold(SW1, SW2),
|
|
{async, HeadCount1ExpFun} = HeadFold(SW2, os:timestamp()),
|
|
HeadCountsExp = {HeadCount0ExpFun(), HeadCount1ExpFun()},
|
|
io:format("HeadCounts ~w after expiry~n", [HeadCountsExp]),
|
|
{0, 0} = HeadCountsExp,
|
|
|
|
ok = leveled_bookie:book_close(Bookie1),
|
|
testutil:reset_filestructure().
|
|
|
|
|
|
breaking_folds(_Config) ->
|
|
% Run various iterators and show that they can be broken by throwing an
|
|
% exception from within the fold
|
|
KeyCount = 10000,
|
|
|
|
RootPath = testutil:reset_filestructure(),
|
|
StartOpts1 = [{root_path, RootPath},
|
|
{max_journalsize, 10000000},
|
|
{sync_strategy, testutil:sync_strategy()}],
|
|
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
|
|
ObjectGen = testutil:get_compressiblevalue_andinteger(),
|
|
IndexGen = testutil:get_randomindexes_generator(8),
|
|
ObjL1 = testutil:generate_objects(KeyCount,
|
|
binary_uuid,
|
|
[],
|
|
ObjectGen,
|
|
IndexGen),
|
|
testutil:riakload(Bookie1, ObjL1),
|
|
|
|
% Find all keys index, and then same again but stop at a midpoint using a
|
|
% throw
|
|
{async, IdxFolder} =
|
|
leveled_bookie:book_indexfold(
|
|
Bookie1,
|
|
list_to_binary("Bucket"),
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{<<"idx1_bin">>, <<"#">>, <<"|">>},
|
|
{true, undefined}),
|
|
KeyList1 = lists:reverse(IdxFolder()),
|
|
io:format("Index fold with result size ~w~n", [length(KeyList1)]),
|
|
true = KeyCount == length(KeyList1),
|
|
|
|
|
|
{MidTerm, MidKey} = lists:nth(KeyCount div 2, KeyList1),
|
|
|
|
FoldKeyThrowFun =
|
|
fun(_B, {Term, Key}, Acc) ->
|
|
case {Term, Key} > {MidTerm, MidKey} of
|
|
true ->
|
|
throw({stop_fold, Acc});
|
|
false ->
|
|
[{Term, Key}|Acc]
|
|
end
|
|
end,
|
|
{async, IdxFolderToMidK} =
|
|
leveled_bookie:book_indexfold(
|
|
Bookie1,
|
|
list_to_binary("Bucket"),
|
|
{FoldKeyThrowFun, []},
|
|
{<<"idx1_bin">>, <<"#">>, <<"|">>},
|
|
{true, undefined}),
|
|
CatchingFold =
|
|
fun(AsyncFolder) ->
|
|
try
|
|
AsyncFolder()
|
|
catch
|
|
throw:{stop_fold, Acc} ->
|
|
Acc
|
|
end
|
|
end,
|
|
|
|
KeyList2 = lists:reverse(CatchingFold(IdxFolderToMidK)),
|
|
io:format("Index fold with result size ~w~n", [length(KeyList2)]),
|
|
true = KeyCount div 2 == length(KeyList2),
|
|
|
|
|
|
HeadFoldFun =
|
|
fun(_B, K, PO, Acc) ->
|
|
{proxy_object, _MDBin, Size, _FF} = binary_to_term(PO),
|
|
[{K, Size}|Acc]
|
|
end,
|
|
{async, HeadFolder} =
|
|
leveled_bookie:book_headfold(
|
|
Bookie1, ?RIAK_TAG, {HeadFoldFun, []}, true, true, false),
|
|
KeySizeList1 = lists:reverse(HeadFolder()),
|
|
io:format("Head fold with result size ~w~n", [length(KeySizeList1)]),
|
|
true = KeyCount == length(KeySizeList1),
|
|
|
|
{MidHeadKey, _MidSize} = lists:nth(KeyCount div 2, KeySizeList1),
|
|
FoldThrowFun =
|
|
fun(FoldFun) ->
|
|
fun(B, K, PO, Acc) ->
|
|
case K > MidHeadKey of
|
|
true ->
|
|
throw({stop_fold, Acc});
|
|
false ->
|
|
FoldFun(B, K, PO, Acc)
|
|
end
|
|
end
|
|
end,
|
|
{async, HeadFolderToMidK} =
|
|
leveled_bookie:book_headfold(Bookie1,
|
|
?RIAK_TAG,
|
|
{FoldThrowFun(HeadFoldFun), []},
|
|
true, true, false),
|
|
KeySizeList2 = lists:reverse(CatchingFold(HeadFolderToMidK)),
|
|
io:format("Head fold with result size ~w~n", [length(KeySizeList2)]),
|
|
true = KeyCount div 2 == length(KeySizeList2),
|
|
|
|
ObjFoldFun =
|
|
fun(_B, K, V, Acc) ->
|
|
[{K,byte_size(V)}|Acc]
|
|
end,
|
|
{async, ObjectFolderKO} =
|
|
leveled_bookie:book_objectfold(Bookie1,
|
|
?RIAK_TAG,
|
|
{ObjFoldFun, []},
|
|
false,
|
|
key_order),
|
|
ObjSizeList1 = lists:reverse(ObjectFolderKO()),
|
|
io:format("Obj fold with result size ~w~n", [length(ObjSizeList1)]),
|
|
true = KeyCount == length(ObjSizeList1),
|
|
|
|
{async, ObjFolderToMidK} =
|
|
leveled_bookie:book_objectfold(Bookie1,
|
|
?RIAK_TAG,
|
|
{FoldThrowFun(ObjFoldFun), []},
|
|
false,
|
|
key_order),
|
|
ObjSizeList2 = lists:reverse(CatchingFold(ObjFolderToMidK)),
|
|
io:format("Object fold with result size ~w~n", [length(ObjSizeList2)]),
|
|
true = KeyCount div 2 == length(ObjSizeList2),
|
|
|
|
% Object folds which are SQN order use a different path through the code,
|
|
% so testing that here. Note that it would not make sense to have a fold
|
|
% that was terminated by reaching a point in the key range .. as results
|
|
% will not be passed to the fold function in key order
|
|
{async, ObjectFolderSO} =
|
|
leveled_bookie:book_objectfold(Bookie1,
|
|
?RIAK_TAG,
|
|
{ObjFoldFun, []},
|
|
false,
|
|
sqn_order),
|
|
ObjSizeList1_SO = lists:reverse(ObjectFolderSO()),
|
|
io:format("Obj fold with result size ~w~n", [length(ObjSizeList1_SO)]),
|
|
true = KeyCount == length(ObjSizeList1_SO),
|
|
|
|
% Exit fold when we've reached a thousand accumulated obects
|
|
FoldThrowThousandFun =
|
|
fun(FoldFun) ->
|
|
fun(B, K, PO, Acc) ->
|
|
case length(Acc) == 1000 of
|
|
true ->
|
|
throw({stop_fold, Acc});
|
|
false ->
|
|
FoldFun(B, K, PO, Acc)
|
|
end
|
|
end
|
|
end,
|
|
{async, ObjFolderTo1K} =
|
|
leveled_bookie:book_objectfold(Bookie1,
|
|
?RIAK_TAG,
|
|
{FoldThrowThousandFun(ObjFoldFun), []},
|
|
false,
|
|
sqn_order),
|
|
ObjSizeList2_SO = lists:reverse(CatchingFold(ObjFolderTo1K)),
|
|
io:format("Object fold with result size ~w~n", [length(ObjSizeList2_SO)]),
|
|
true = 1000 == length(ObjSizeList2_SO),
|
|
|
|
ObjL2 = testutil:generate_objects(10,
|
|
binary_uuid,
|
|
[],
|
|
ObjectGen,
|
|
IndexGen,
|
|
"B2"),
|
|
ObjL3 = testutil:generate_objects(10,
|
|
binary_uuid,
|
|
[],
|
|
ObjectGen,
|
|
IndexGen,
|
|
"B3"),
|
|
ObjL4 = testutil:generate_objects(10,
|
|
binary_uuid,
|
|
[],
|
|
ObjectGen,
|
|
IndexGen,
|
|
"B4"),
|
|
testutil:riakload(Bookie1, ObjL2),
|
|
testutil:riakload(Bookie1, ObjL3),
|
|
testutil:riakload(Bookie1, ObjL4),
|
|
|
|
FBAccT = {fun(B, Acc) -> [B|Acc] end, []},
|
|
{async, BucketFolder} =
|
|
leveled_bookie:book_bucketlist(Bookie1, ?RIAK_TAG, FBAccT, all),
|
|
BucketList1 = lists:reverse(BucketFolder()),
|
|
io:format("bucket list with result size ~w~n", [length(BucketList1)]),
|
|
true = 4 == length(BucketList1),
|
|
|
|
StopAt3Fun =
|
|
fun(B, Acc) ->
|
|
Acc0 = [B|Acc],
|
|
case B of
|
|
<<"B3">> ->
|
|
throw({stop_fold, Acc0});
|
|
_ ->
|
|
Acc0
|
|
end
|
|
end,
|
|
|
|
{async, StopAt3BucketFolder} =
|
|
leveled_bookie:book_bucketlist(Bookie1,
|
|
?RIAK_TAG,
|
|
{StopAt3Fun, []},
|
|
all),
|
|
BucketListSA3 = lists:reverse(CatchingFold(StopAt3BucketFolder)),
|
|
io:format("bucket list with result ~w~n", [BucketListSA3]),
|
|
true = [<<"B2">>, <<"B3">>] == BucketListSA3,
|
|
|
|
|
|
ok = leveled_bookie:book_close(Bookie1),
|
|
testutil:reset_filestructure().
|
|
|
|
|
|
|
|
single_object_with2i(_Config) ->
|
|
% Load a single object with an integer and a binary
|
|
% index and query for it
|
|
RootPath = testutil:reset_filestructure(),
|
|
StartOpts1 = [{root_path, RootPath},
|
|
{max_journalsize, 5000000},
|
|
{sync_strategy, testutil:sync_strategy()}],
|
|
% low journal size to make sure > 1 created
|
|
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
|
|
{TestObject, _TestSpec} = testutil:generate_testobject(),
|
|
TestSpec = [{add, list_to_binary("integer_int"), 100},
|
|
{add, list_to_binary("binary_bin"), <<100:32/integer>>}],
|
|
ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
|
|
|
|
%% @TODO replace all index queries with new Top-Level API if tests
|
|
%% pass
|
|
{async, IdxFolder1} =
|
|
leveled_bookie:book_indexfold(
|
|
Bookie1,
|
|
"Bucket1",
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{list_to_binary("binary_bin"),
|
|
<<99:32/integer>>, <<101:32/integer>>},
|
|
{true, undefined}),
|
|
R1 = IdxFolder1(),
|
|
io:format("R1 of ~w~n", [R1]),
|
|
true = [{<<100:32/integer>>,"Key1"}] == R1,
|
|
|
|
IdxQ2 = {index_query,
|
|
"Bucket1",
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{list_to_binary("integer_int"),
|
|
99, 101},
|
|
{true, undefined}},
|
|
{async, IdxFolder2} = leveled_bookie:book_returnfolder(Bookie1, IdxQ2),
|
|
R2 = IdxFolder2(),
|
|
io:format("R2 of ~w~n", [R2]),
|
|
true = [{100,"Key1"}] == R2,
|
|
|
|
IdxQ3 = {index_query,
|
|
{"Bucket1", "Key1"},
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{list_to_binary("integer_int"),
|
|
99, 101},
|
|
{true, undefined}},
|
|
{async, IdxFolder3} = leveled_bookie:book_returnfolder(Bookie1, IdxQ3),
|
|
R3 = IdxFolder3(),
|
|
io:format("R2 of ~w~n", [R3]),
|
|
true = [{100,"Key1"}] == R3,
|
|
|
|
ok = leveled_bookie:book_close(Bookie1),
|
|
testutil:reset_filestructure().
|
|
|
|
small_load_with2i(_Config) ->
|
|
RootPath = testutil:reset_filestructure(),
|
|
StartOpts1 = [{root_path, RootPath},
|
|
{max_journalsize, 5000000},
|
|
{sync_strategy, testutil:sync_strategy()}],
|
|
% low journal size to make sure > 1 created
|
|
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
|
|
{TestObject, TestSpec} = testutil:generate_testobject(),
|
|
ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
|
|
testutil:check_forobject(Bookie1, TestObject),
|
|
testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"),
|
|
testutil:check_forobject(Bookie1, TestObject),
|
|
ObjectGen = testutil:get_compressiblevalue_andinteger(),
|
|
IndexGen = testutil:get_randomindexes_generator(8),
|
|
ObjL1 =
|
|
testutil:generate_objects(
|
|
10000, uuid, [], ObjectGen, IndexGen),
|
|
testutil:riakload(Bookie1, ObjL1),
|
|
ChkList1 = lists:sublist(lists:sort(ObjL1), 100),
|
|
testutil:check_forlist(Bookie1, ChkList1),
|
|
testutil:check_forobject(Bookie1, TestObject),
|
|
|
|
% Find all keys index, and then just the last key
|
|
IdxQ1 = {index_query,
|
|
"Bucket",
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{<<"idx1_bin">>, <<"#">>, <<"|">>},
|
|
{true, undefined}},
|
|
{async, IdxFolder} = leveled_bookie:book_returnfolder(Bookie1, IdxQ1),
|
|
KeyList1 = lists:usort(IdxFolder()),
|
|
true = 10000 == length(KeyList1),
|
|
{LastTerm, LastKey} = lists:last(KeyList1),
|
|
IdxQ2 = {index_query,
|
|
{"Bucket", LastKey},
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{<<"idx1_bin">>, LastTerm, <<"|">>},
|
|
{false, undefined}},
|
|
{async, IdxFolderLK} = leveled_bookie:book_returnfolder(Bookie1, IdxQ2),
|
|
KeyList2 = lists:usort(IdxFolderLK()),
|
|
io:format("List should be last key ~w ~w~n", [LastKey, KeyList2]),
|
|
true = 1 == length(KeyList2),
|
|
|
|
%% Delete the objects from the ChkList removing the indexes
|
|
lists:foreach(fun({_RN, Obj, Spc}) ->
|
|
DSpc = lists:map(fun({add, F, T}) ->
|
|
{remove, F, T}
|
|
end,
|
|
Spc),
|
|
{B, K} =
|
|
{testutil:get_bucket(Obj), testutil:get_key(Obj)},
|
|
testutil:book_riakdelete(Bookie1, B, K, DSpc)
|
|
end,
|
|
ChkList1),
|
|
%% Get the Buckets Keys and Hashes for the whole bucket
|
|
FoldObjectsFun = fun(B, K, V, Acc) -> [{B, K, erlang:phash2(V)}|Acc]
|
|
end,
|
|
|
|
{async, HTreeF1} = leveled_bookie:book_objectfold(Bookie1,
|
|
?RIAK_TAG,
|
|
{FoldObjectsFun, []},
|
|
false),
|
|
|
|
KeyHashList1 = HTreeF1(),
|
|
{async, HTreeF2} = leveled_bookie:book_objectfold(Bookie1,
|
|
?RIAK_TAG,
|
|
"Bucket",
|
|
all,
|
|
{FoldObjectsFun, []},
|
|
false),
|
|
KeyHashList2 = HTreeF2(),
|
|
{async, HTreeF3} =
|
|
leveled_bookie:book_objectfold(
|
|
Bookie1,
|
|
?RIAK_TAG,
|
|
"Bucket",
|
|
{<<"idx1_bin">>, <<"#">>, <<"|">>},
|
|
{FoldObjectsFun, []},
|
|
false),
|
|
KeyHashList3 = HTreeF3(),
|
|
true = 9901 == length(KeyHashList1), % also includes the test object
|
|
true = 9900 == length(KeyHashList2),
|
|
true = 9900 == length(KeyHashList3),
|
|
|
|
SumIntFun = fun(_B, _K, Obj, Acc) ->
|
|
{I, _Bin} = testutil:get_value(Obj),
|
|
Acc + I
|
|
end,
|
|
BucketObjQ =
|
|
{foldobjects_bybucket, ?RIAK_TAG, "Bucket", all, {SumIntFun, 0}, true},
|
|
{async, Sum1} = leveled_bookie:book_returnfolder(Bookie1, BucketObjQ),
|
|
Total1 = Sum1(),
|
|
io:format("Total from summing all I is ~w~n", [Total1]),
|
|
SumFromObjLFun =
|
|
fun(Obj, Acc) ->
|
|
{I, _Bin} = testutil:get_value_from_objectlistitem(Obj),
|
|
Acc + I
|
|
end,
|
|
ObjL1Total =
|
|
lists:foldl(SumFromObjLFun, 0, ObjL1),
|
|
ChkList1Total =
|
|
lists:foldl(SumFromObjLFun, 0, ChkList1),
|
|
io:format("Total in original object list ~w and from removed list ~w~n",
|
|
[ObjL1Total, ChkList1Total]),
|
|
|
|
Total1 = ObjL1Total - ChkList1Total,
|
|
|
|
ok = leveled_bookie:book_close(Bookie1),
|
|
|
|
{ok, Bookie2} = leveled_bookie:book_start(StartOpts1),
|
|
|
|
{async, Sum2} = leveled_bookie:book_returnfolder(Bookie2, BucketObjQ),
|
|
Total2 = Sum2(),
|
|
true = Total2 == Total1,
|
|
|
|
FoldBucketsFun = fun(B, Acc) -> sets:add_element(B, Acc) end,
|
|
|
|
% this should find Bucket and Bucket1 - as we can now find string-based
|
|
% buckets using bucket_list - i.e. it isn't just binary buckets now
|
|
{async, BL} = leveled_bookie:book_bucketlist(Bookie2, ?RIAK_TAG, {FoldBucketsFun, sets:new()}, all),
|
|
true = sets:size(BL()) == 2,
|
|
|
|
ok = leveled_bookie:book_close(Bookie2),
|
|
testutil:reset_filestructure().
|
|
|
|
|
|
query_count(_Config) ->
|
|
RootPath = testutil:reset_filestructure(),
|
|
{ok, Book1} =
|
|
leveled_bookie:book_start(
|
|
RootPath, 2000, 50000000, testutil:sync_strategy()),
|
|
BucketBin = list_to_binary("Bucket"),
|
|
{TestObject, TestSpec} =
|
|
testutil:generate_testobject(
|
|
BucketBin, term_to_binary("Key1"), "Value1", [], [{"MDK1", "MDV1"}]),
|
|
ok = testutil:book_riakput(Book1, TestObject, TestSpec),
|
|
testutil:check_forobject(Book1, TestObject),
|
|
testutil:check_formissingobject(Book1, "Bucket1", "Key2"),
|
|
testutil:check_forobject(Book1, TestObject),
|
|
lists:foreach(
|
|
fun(_X) ->
|
|
V = testutil:get_compressiblevalue(),
|
|
Indexes = testutil:get_randomindexes_generator(8),
|
|
SW = os:timestamp(),
|
|
ObjL1 = testutil:generate_objects(10000,
|
|
binary_uuid,
|
|
[],
|
|
V,
|
|
Indexes),
|
|
testutil:riakload(Book1, ObjL1),
|
|
io:format(
|
|
"Put of 10000 objects with 8 index entries "
|
|
"each completed in ~w microseconds~n",
|
|
[timer:now_diff(os:timestamp(), SW)])
|
|
end,
|
|
lists:seq(1, 8)),
|
|
testutil:check_forobject(Book1, TestObject),
|
|
Total =
|
|
lists:foldl(
|
|
fun(X, Acc) ->
|
|
IdxF = "idx" ++ integer_to_list(X) ++ "_bin",
|
|
T =
|
|
count_termsonindex(
|
|
BucketBin, list_to_binary(IdxF), Book1, ?KEY_ONLY),
|
|
io:format("~w terms found on index ~s~n", [T, IdxF]),
|
|
Acc + T
|
|
end,
|
|
0,
|
|
lists:seq(1, 8)),
|
|
true = Total == 640000,
|
|
Index1Count =
|
|
count_termsonindex(
|
|
BucketBin, <<"idx1_bin">>, Book1, ?KEY_ONLY),
|
|
ok = leveled_bookie:book_close(Book1),
|
|
{ok, Book2} =
|
|
leveled_bookie:book_start(
|
|
RootPath, 1000, 50000000, testutil:sync_strategy()),
|
|
Index1Count =
|
|
count_termsonindex(
|
|
BucketBin, <<"idx1_bin">>, Book2, ?KEY_ONLY),
|
|
NameList = testutil:name_list(),
|
|
TotalNameByName =
|
|
lists:foldl(
|
|
fun({_X, Name}, Acc) ->
|
|
{ok, Regex} =
|
|
re:compile("[0-9]+" ++ Name),
|
|
SW = os:timestamp(),
|
|
T =
|
|
count_termsonindex(
|
|
BucketBin,
|
|
list_to_binary("idx1_bin"),
|
|
Book2,
|
|
{false, Regex}),
|
|
TD = timer:now_diff(os:timestamp(), SW),
|
|
io:format(
|
|
"~w terms found on index idx1 with a "
|
|
"regex in ~w microseconds~n",
|
|
[T, TD]),
|
|
Acc + T
|
|
end,
|
|
0,
|
|
NameList),
|
|
true = TotalNameByName == Index1Count,
|
|
{ok, RegMia} = re:compile("[0-9]+Mia"),
|
|
Query1 = {index_query,
|
|
BucketBin,
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{<<"idx2_bin">>, <<"2000">>, <<"2000|">>},
|
|
{false, RegMia}},
|
|
{async,
|
|
Mia2KFolder1} = leveled_bookie:book_returnfolder(Book2, Query1),
|
|
Mia2000Count1 = length(Mia2KFolder1()),
|
|
Query2 = {index_query,
|
|
BucketBin,
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{<<"idx2_bin">>, <<"2000">>, <<"2001">>},
|
|
{true, undefined}},
|
|
{async,
|
|
Mia2KFolder2} = leveled_bookie:book_returnfolder(Book2, Query2),
|
|
Mia2000Count2 = lists:foldl(fun({Term, _Key}, Acc) ->
|
|
case re:run(Term, RegMia) of
|
|
nomatch ->
|
|
Acc;
|
|
_ ->
|
|
Acc + 1
|
|
end end,
|
|
0,
|
|
Mia2KFolder2()),
|
|
ok = case Mia2000Count2 of
|
|
Mia2000Count1 when Mia2000Count1 > 0 ->
|
|
io:format("Mia2000 counts match at ~w~n",
|
|
[Mia2000Count1]),
|
|
ok
|
|
end,
|
|
{ok, RxMia2K} = re:compile("^2000[0-9]+Mia"),
|
|
Query3 = {index_query,
|
|
BucketBin,
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{<<"idx2_bin">>, <<"1980">>, <<"2100">>},
|
|
{false, RxMia2K}},
|
|
{async,
|
|
Mia2KFolder3} = leveled_bookie:book_returnfolder(Book2, Query3),
|
|
Mia2000Count1 = length(Mia2KFolder3()),
|
|
|
|
V9 = testutil:get_compressiblevalue(),
|
|
Indexes9 = testutil:get_randomindexes_generator(8),
|
|
[{_RN, Obj9, Spc9}] =
|
|
testutil:generate_objects(
|
|
1, binary_uuid, [], V9, Indexes9),
|
|
ok = testutil:book_riakput(Book2, Obj9, Spc9),
|
|
R9 =
|
|
lists:map(
|
|
fun({add, IdxF, IdxT}) ->
|
|
Q = {index_query,
|
|
BucketBin,
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{IdxF, IdxT, IdxT},
|
|
?KEY_ONLY},
|
|
R = leveled_bookie:book_returnfolder(Book2, Q),
|
|
{async, Fldr} = R,
|
|
case length(Fldr()) of
|
|
X when X > 0 ->
|
|
{IdxF, IdxT, X}
|
|
end
|
|
end,
|
|
Spc9),
|
|
Spc9Del = lists:map(fun({add, IdxF, IdxT}) -> {remove, IdxF, IdxT} end,
|
|
Spc9),
|
|
ok = testutil:book_riakput(Book2, Obj9, Spc9Del),
|
|
lists:foreach(fun({IdxF, IdxT, X}) ->
|
|
Q = {index_query,
|
|
BucketBin,
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{IdxF, IdxT, IdxT},
|
|
?KEY_ONLY},
|
|
R = leveled_bookie:book_returnfolder(Book2, Q),
|
|
{async, Fldr} = R,
|
|
case length(Fldr()) of
|
|
Y ->
|
|
Y = X - 1
|
|
end
|
|
end,
|
|
R9),
|
|
ok = leveled_bookie:book_close(Book2),
|
|
{ok, Book3} =
|
|
leveled_bookie:book_start(
|
|
RootPath, 2000, 50000000, testutil:sync_strategy()),
|
|
lists:foreach(
|
|
fun({IdxF, IdxT, X}) ->
|
|
Q = {index_query,
|
|
BucketBin,
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{IdxF, IdxT, IdxT},
|
|
?KEY_ONLY},
|
|
R = leveled_bookie:book_returnfolder(Book3, Q),
|
|
{async, Fldr} = R,
|
|
case length(Fldr()) of
|
|
Y ->
|
|
Y = X - 1
|
|
end
|
|
end,
|
|
R9),
|
|
ok = testutil:book_riakput(Book3, Obj9, Spc9),
|
|
ok = leveled_bookie:book_close(Book3),
|
|
{ok, Book4} =
|
|
leveled_bookie:book_start(
|
|
RootPath, 2000, 50000000, testutil:sync_strategy()),
|
|
lists:foreach(
|
|
fun({IdxF, IdxT, X}) ->
|
|
Q = {index_query,
|
|
BucketBin,
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{IdxF, IdxT, IdxT},
|
|
?KEY_ONLY},
|
|
R = leveled_bookie:book_returnfolder(Book4, Q),
|
|
{async, Fldr} = R,
|
|
case length(Fldr()) of
|
|
X ->
|
|
ok
|
|
end
|
|
end,
|
|
R9),
|
|
testutil:check_forobject(Book4, TestObject),
|
|
|
|
FoldBucketsFun = fun(B, Acc) -> sets:add_element(B, Acc) end,
|
|
BucketListQuery = {bucket_list,
|
|
?RIAK_TAG,
|
|
{FoldBucketsFun, sets:new()}},
|
|
{async, BLF1} = leveled_bookie:book_returnfolder(Book4, BucketListQuery),
|
|
SW_QA = os:timestamp(),
|
|
BucketSet1 = BLF1(),
|
|
io:format("Bucket set returned in ~w microseconds",
|
|
[timer:now_diff(os:timestamp(), SW_QA)]),
|
|
|
|
true = sets:size(BucketSet1) == 1,
|
|
|
|
ObjList10A =
|
|
testutil:generate_objects(
|
|
5000, binary_uuid, [], V9, Indexes9, "BucketA"),
|
|
ObjList10B =
|
|
testutil:generate_objects(
|
|
5000, binary_uuid, [], V9, Indexes9, "BucketB"),
|
|
ObjList10C =
|
|
testutil:generate_objects(
|
|
5000, binary_uuid, [], V9, Indexes9, "BucketC"),
|
|
testutil:riakload(Book4, ObjList10A),
|
|
testutil:riakload(Book4, ObjList10B),
|
|
testutil:riakload(Book4, ObjList10C),
|
|
{async, BLF2} = leveled_bookie:book_returnfolder(Book4, BucketListQuery),
|
|
SW_QB = os:timestamp(),
|
|
BucketSet2 = BLF2(),
|
|
io:format("Bucket set returned in ~w microseconds",
|
|
[timer:now_diff(os:timestamp(), SW_QB)]),
|
|
true = sets:size(BucketSet2) == 4,
|
|
|
|
ok = leveled_bookie:book_close(Book4),
|
|
|
|
{ok, Book5} = leveled_bookie:book_start(RootPath,
|
|
2000,
|
|
50000000,
|
|
testutil:sync_strategy()),
|
|
{async, BLF3} = leveled_bookie:book_returnfolder(Book5, BucketListQuery),
|
|
SW_QC = os:timestamp(),
|
|
BucketSet3 = BLF3(),
|
|
io:format("Bucket set returned in ~w microseconds",
|
|
[timer:now_diff(os:timestamp(), SW_QC)]),
|
|
true = sets:size(BucketSet3) == 4,
|
|
|
|
ok = leveled_bookie:book_close(Book5),
|
|
|
|
testutil:reset_filestructure().
|
|
|
|
|
|
count_termsonindex(Bucket, IdxField, Book, QType) ->
|
|
lists:foldl(
|
|
fun(X, Acc) ->
|
|
SW = os:timestamp(),
|
|
ST = list_to_binary(integer_to_list(X)),
|
|
Pipe = <<"|">>,
|
|
ET = <<ST/binary, Pipe/binary>>,
|
|
Q = {index_query,
|
|
Bucket,
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{IdxField, ST, ET},
|
|
QType},
|
|
R = leveled_bookie:book_returnfolder(Book, Q),
|
|
{async, Folder} = R,
|
|
Items = length(Folder()),
|
|
io:format(
|
|
"2i query from term ~s on index ~s took ~w microseconds~n",
|
|
[ST, IdxField, timer:now_diff(os:timestamp(), SW)]),
|
|
Acc + Items
|
|
end,
|
|
0,
|
|
lists:seq(190, 221)).
|
|
|
|
multibucket_fold(_Config) ->
|
|
RootPath = testutil:reset_filestructure(),
|
|
{ok, Bookie1} = leveled_bookie:book_start(RootPath,
|
|
2000,
|
|
50000000,
|
|
testutil:sync_strategy()),
|
|
ObjectGen = testutil:get_compressiblevalue_andinteger(),
|
|
IndexGen = fun() -> [] end,
|
|
ObjL1 = testutil:generate_objects(13000,
|
|
uuid,
|
|
[],
|
|
ObjectGen,
|
|
IndexGen,
|
|
{<<"Type1">>, <<"Bucket1">>}),
|
|
testutil:riakload(Bookie1, ObjL1),
|
|
ObjL2 = testutil:generate_objects(17000,
|
|
uuid,
|
|
[],
|
|
ObjectGen,
|
|
IndexGen,
|
|
<<"Bucket2">>),
|
|
testutil:riakload(Bookie1, ObjL2),
|
|
ObjL3 = testutil:generate_objects(7000,
|
|
uuid,
|
|
[],
|
|
ObjectGen,
|
|
IndexGen,
|
|
<<"Bucket3">>),
|
|
testutil:riakload(Bookie1, ObjL3),
|
|
ObjL4 = testutil:generate_objects(23000,
|
|
uuid,
|
|
[],
|
|
ObjectGen,
|
|
IndexGen,
|
|
{<<"Type2">>, <<"Bucket4">>}),
|
|
testutil:riakload(Bookie1, ObjL4),
|
|
|
|
FF = fun(B, K, _PO, Acc) ->
|
|
[{B, K}|Acc]
|
|
end,
|
|
FoldAccT = {FF, []},
|
|
|
|
{async, R1} =
|
|
leveled_bookie:book_headfold(Bookie1,
|
|
?RIAK_TAG,
|
|
{bucket_list,
|
|
[{<<"Type1">>, <<"Bucket1">>},
|
|
{<<"Type2">>, <<"Bucket4">>}]},
|
|
FoldAccT,
|
|
false,
|
|
true,
|
|
false),
|
|
|
|
O1 = length(R1()),
|
|
io:format("Result R1 of length ~w~n", [O1]),
|
|
|
|
{async, R2} =
|
|
leveled_bookie:book_headfold(Bookie1,
|
|
?RIAK_TAG,
|
|
{bucket_list,
|
|
[<<"Bucket2">>,
|
|
<<"Bucket3">>]},
|
|
{fun(_B, _K, _PO, Acc) ->
|
|
Acc +1
|
|
end,
|
|
0},
|
|
false, true, false),
|
|
O2 = R2(),
|
|
io:format("Result R2 of ~w~n", [O2]),
|
|
|
|
true = 36000 == O1,
|
|
true = 24000 == O2,
|
|
|
|
FoldBucketsFun = fun(B, Acc) -> [B|Acc] end,
|
|
{async, Folder} =
|
|
leveled_bookie:book_bucketlist(Bookie1,
|
|
?RIAK_TAG,
|
|
{FoldBucketsFun, []},
|
|
all),
|
|
BucketList = lists:reverse(Folder()),
|
|
ExpectedBucketList =
|
|
[{<<"Type1">>, <<"Bucket1">>}, {<<"Type2">>, <<"Bucket4">>},
|
|
<<"Bucket2">>, <<"Bucket3">>],
|
|
io:format("BucketList ~w", [BucketList]),
|
|
true = ExpectedBucketList == BucketList,
|
|
|
|
ok = leveled_bookie:book_close(Bookie1),
|
|
testutil:reset_filestructure().
|
|
|
|
rotating_objects(_Config) ->
|
|
RootPath = testutil:reset_filestructure(),
|
|
ok = testutil:rotating_object_check(RootPath, "Bucket1", 10),
|
|
ok = testutil:rotating_object_check(RootPath, "Bucket2", 200),
|
|
ok = testutil:rotating_object_check(RootPath, "Bucket3", 800),
|
|
ok = testutil:rotating_object_check(RootPath, "Bucket4", 1600),
|
|
ok = testutil:rotating_object_check(RootPath, "Bucket5", 3200),
|
|
ok = testutil:rotating_object_check(RootPath, "Bucket6", 9600),
|
|
testutil:reset_filestructure().
|
|
|
|
foldobjects_bybucket_range(_Config) ->
|
|
RootPath = testutil:reset_filestructure(),
|
|
{ok, Bookie1} = leveled_bookie:book_start(RootPath,
|
|
2000,
|
|
50000000,
|
|
testutil:sync_strategy()),
|
|
ObjectGen = testutil:get_compressiblevalue_andinteger(),
|
|
IndexGen = fun() -> [] end,
|
|
ObjL1 = testutil:generate_objects(1300,
|
|
{fixed_binary, 1},
|
|
[],
|
|
ObjectGen,
|
|
IndexGen,
|
|
<<"Bucket1">>),
|
|
testutil:riakload(Bookie1, ObjL1),
|
|
|
|
FoldKeysFun = fun(_B, K,_V, Acc) ->
|
|
[ K |Acc]
|
|
end,
|
|
|
|
StartKey = testutil:fixed_bin_key(123),
|
|
EndKey = testutil:fixed_bin_key(779),
|
|
|
|
{async, Folder} = leveled_bookie:book_objectfold(Bookie1,
|
|
?RIAK_TAG,
|
|
<<"Bucket1">>,
|
|
{StartKey, EndKey}, {FoldKeysFun, []},
|
|
true
|
|
),
|
|
ResLen = length(Folder()),
|
|
io:format("Length of Result of folder ~w~n", [ResLen]),
|
|
true = 657 == ResLen,
|
|
|
|
{async, AllFolder} = leveled_bookie:book_objectfold(Bookie1,
|
|
?RIAK_TAG,
|
|
<<"Bucket1">>,
|
|
all,
|
|
{FoldKeysFun, []},
|
|
true
|
|
),
|
|
|
|
AllResLen = length(AllFolder()),
|
|
io:format("Length of Result of all keys folder ~w~n", [AllResLen]),
|
|
true = 1300 == AllResLen,
|
|
|
|
ok = leveled_bookie:book_close(Bookie1),
|
|
testutil:reset_filestructure().
|