6223b801f3
* Mas i410 looptoclose (#420) * Stop waiting full SHUTDOWN_PAUSE If there is a snapshot outstanding at shutdown time, there was a wait of SHUTDOWN_PAUSE to give the snapshot time to close down. This causes an issue in kv_index_tictactree when rebuilds complete, when an exchange was in flight at the point the rebuild completed - the aae_controller will become blocked for the full shutdown pause, whilst it waits for the replaced key store to be closed. This change is to loop within the shutdown pause, so that if the snapshot supporting the exchange is closed, the paused bookie can close more quickly (unblocking the controller). Without this fix, there are intermittent issues in kv_index_tictactree's mockvnode_SUITE tests. * Address test reliability Be a bit clearer with waiting round seconds, Was intermittently failing on QR4 previously (but QR5 1s later was always OK). * Update iterator_SUITE.erl * Refine test assertion At Stage C there might be 0 files left, in which case equality with Stage D result is ok.
977 lines
41 KiB
Erlang
977 lines
41 KiB
Erlang
-module(iterator_SUITE).
|
|
|
|
-include_lib("common_test/include/ct.hrl").
|
|
-include("include/leveled.hrl").
|
|
|
|
-define(KEY_ONLY, {false, undefined}).
|
|
|
|
-export([all/0]).
|
|
-export([expiring_indexes/1,
|
|
breaking_folds/1,
|
|
single_object_with2i/1,
|
|
small_load_with2i/1,
|
|
query_count/1,
|
|
multibucket_fold/1,
|
|
foldobjects_bybucket_range/1,
|
|
rotating_objects/1]).
|
|
|
|
all() -> [
|
|
expiring_indexes,
|
|
breaking_folds,
|
|
single_object_with2i,
|
|
small_load_with2i,
|
|
query_count,
|
|
multibucket_fold,
|
|
rotating_objects,
|
|
foldobjects_bybucket_range
|
|
].
|
|
|
|
|
|
expiring_indexes(_Config) ->
|
|
% Add objects to the store with index entries, where the objects (and hence
|
|
% the indexes have an expiry time. Confirm that the indexes and the
|
|
% objects are no longer present after the expiry time (and are present
|
|
% before). Confirm that replacing an object has the expected outcome, if
|
|
% the IndexSpecs are updated as part of the request.
|
|
KeyCount = 50000,
|
|
Future = 60,
|
|
% 1 minute - if running tests on a slow machine, may need to increase
|
|
% this value
|
|
RootPath = testutil:reset_filestructure(),
|
|
StartOpts1 =
|
|
[{root_path, RootPath},
|
|
{max_pencillercachesize, 16000},
|
|
{max_journalobjectcount, 30000},
|
|
{sync_strategy, testutil:sync_strategy()}],
|
|
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
|
|
|
|
SW1 = os:timestamp(),
|
|
IBKL1 = testutil:stdload_expiring(Bookie1, KeyCount, Future),
|
|
timer:sleep(1000),
|
|
% Wait a second after last key so that none loaded in the last second
|
|
LoadTime = timer:now_diff(os:timestamp(), SW1)/1000000,
|
|
io:format("Load of ~w std objects in ~w seconds~n", [KeyCount, LoadTime]),
|
|
SW2 = os:timestamp(),
|
|
|
|
FilterFun = fun({I, _B, _K}) -> lists:member(I, [5, 6, 7, 8]) end,
|
|
LoadedEntriesInRange = lists:sort(lists:filter(FilterFun, IBKL1)),
|
|
|
|
true = LoadTime < (Future - 20),
|
|
% need 20 seconds spare to run query
|
|
% and add replacements
|
|
|
|
{I0, B0, K0} = hd(IBKL1),
|
|
false = FilterFun(hd(IBKL1)),
|
|
% The head entry should not have index between 5 and 8
|
|
|
|
CountI0Fold =
|
|
fun() ->
|
|
leveled_bookie:book_indexfold(
|
|
Bookie1,
|
|
B0,
|
|
{fun(_BF, _KT, Acc) -> Acc + 1 end, 0},
|
|
{<<"temp_int">>, I0, I0},
|
|
{true, undefined})
|
|
end,
|
|
{async, I0Counter1} = CountI0Fold(),
|
|
I0Count1 = I0Counter1(),
|
|
|
|
FoldFun = fun(BF, {IdxV, KeyF}, Acc) -> [{IdxV, BF, KeyF}|Acc] end,
|
|
InitAcc = [],
|
|
IndexFold =
|
|
fun() ->
|
|
leveled_bookie:book_indexfold(
|
|
Bookie1,
|
|
B0,
|
|
{FoldFun, InitAcc},
|
|
{<<"temp_int">>, 5, 8},
|
|
{true, undefined})
|
|
end,
|
|
|
|
{async, Folder1} = IndexFold(),
|
|
QR1 = Folder1(),
|
|
true = lists:sort(QR1) == LoadedEntriesInRange,
|
|
% Replace object with one with an index value of 6
|
|
testutil:stdload_object(
|
|
Bookie1, B0, K0, 6, <<"value">>, leveled_util:integer_now() + 600),
|
|
% Confirm that this has reduced the index entries in I0 by 1
|
|
{async, I0Counter2} = CountI0Fold(),
|
|
I0Count2 = I0Counter2(),
|
|
io:format("Count with index value ~w changed from ~w to ~w~n",
|
|
[I0, I0Count1, I0Count2]),
|
|
true = I0Count2 == (I0Count1 - 1),
|
|
% Now replace again, shortening the timeout to 10s,
|
|
% this time index value of 6
|
|
testutil:stdload_object(
|
|
Bookie1, B0, K0, 5, <<"value">>, leveled_util:integer_now() + 10),
|
|
timer:sleep(1000),
|
|
{async, Folder2} = IndexFold(),
|
|
leveled_bookie:book_indexfold(Bookie1,
|
|
B0,
|
|
{FoldFun, InitAcc},
|
|
{<<"temp_int">>, 5, 8},
|
|
{true, undefined}),
|
|
QR2 = Folder2(),
|
|
io:format("Query with additional entry length ~w~n", [length(QR2)]),
|
|
true = lists:sort(QR2) == lists:sort([{5, B0, K0}|LoadedEntriesInRange]),
|
|
% Wait for a 10s timeout plus a second to be sure
|
|
timer:sleep(10000 + 1000),
|
|
{async, Folder3} = IndexFold(),
|
|
QR3 = Folder3(),
|
|
% Now the entry should be missing (and the 600s TTL entry should not have
|
|
% resurfaced)
|
|
io:format("Query results length ~w following sleep~n", [length(QR3)]),
|
|
true = lists:sort(QR3) == LoadedEntriesInRange,
|
|
|
|
FoldTime = timer:now_diff(os:timestamp(), SW1)/1000000 - LoadTime,
|
|
io:format("Query returned ~w entries in ~w seconds - 3 queries + 10s wait~n",
|
|
[length(QR3), FoldTime]),
|
|
|
|
SleepTime =
|
|
(Future - (timer:now_diff(os:timestamp(), SW2) div (1000 * 1000))) + 1,
|
|
|
|
io:format("Sleeping ~w s for all to expire~n", [SleepTime]),
|
|
timer:sleep(SleepTime * 1000),
|
|
|
|
% Index entries should now have expired
|
|
{async, Folder4} = IndexFold(),
|
|
QR4 = Folder4(),
|
|
io:format("QR4 Unexpired indexes of length ~w~n", [length(QR4)]),
|
|
timer:sleep(1000),
|
|
{async, Folder5} = IndexFold(),
|
|
QR5 = Folder5(),
|
|
io:format("QR5 Unexpired indexes of length ~w~n", [length(QR5)]),
|
|
|
|
true = QR4 == [],
|
|
true = QR5 == [],
|
|
|
|
ok = leveled_bookie:book_close(Bookie1),
|
|
testutil:reset_filestructure().
|
|
|
|
|
|
breaking_folds(_Config) ->
|
|
% Run various iterators and show that they can be broken by throwing an
|
|
% exception from within the fold
|
|
KeyCount = 10000,
|
|
|
|
RootPath = testutil:reset_filestructure(),
|
|
StartOpts1 = [{root_path, RootPath},
|
|
{max_journalsize, 10000000},
|
|
{sync_strategy, testutil:sync_strategy()}],
|
|
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
|
|
ObjectGen = testutil:get_compressiblevalue_andinteger(),
|
|
IndexGen = testutil:get_randomindexes_generator(8),
|
|
ObjL1 = testutil:generate_objects(KeyCount,
|
|
binary_uuid,
|
|
[],
|
|
ObjectGen,
|
|
IndexGen),
|
|
testutil:riakload(Bookie1, ObjL1),
|
|
|
|
% Find all keys index, and then same again but stop at a midpoint using a
|
|
% throw
|
|
{async, IdxFolder} =
|
|
leveled_bookie:book_indexfold(Bookie1,
|
|
list_to_binary("Bucket"),
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{"idx1_bin", "#", "|"},
|
|
{true, undefined}),
|
|
KeyList1 = lists:reverse(IdxFolder()),
|
|
io:format("Index fold with result size ~w~n", [length(KeyList1)]),
|
|
true = KeyCount == length(KeyList1),
|
|
|
|
|
|
{MidTerm, MidKey} = lists:nth(KeyCount div 2, KeyList1),
|
|
|
|
FoldKeyThrowFun =
|
|
fun(_B, {Term, Key}, Acc) ->
|
|
case {Term, Key} > {MidTerm, MidKey} of
|
|
true ->
|
|
throw({stop_fold, Acc});
|
|
false ->
|
|
[{Term, Key}|Acc]
|
|
end
|
|
end,
|
|
{async, IdxFolderToMidK} =
|
|
leveled_bookie:book_indexfold(Bookie1,
|
|
list_to_binary("Bucket"),
|
|
{FoldKeyThrowFun, []},
|
|
{"idx1_bin", "#", "|"},
|
|
{true, undefined}),
|
|
CatchingFold =
|
|
fun(AsyncFolder) ->
|
|
try
|
|
AsyncFolder()
|
|
catch
|
|
throw:{stop_fold, Acc} ->
|
|
Acc
|
|
end
|
|
end,
|
|
|
|
KeyList2 = lists:reverse(CatchingFold(IdxFolderToMidK)),
|
|
io:format("Index fold with result size ~w~n", [length(KeyList2)]),
|
|
true = KeyCount div 2 == length(KeyList2),
|
|
|
|
|
|
HeadFoldFun =
|
|
fun(_B, K, PO, Acc) ->
|
|
{proxy_object, _MDBin, Size, _FF} = binary_to_term(PO),
|
|
[{K, Size}|Acc]
|
|
end,
|
|
{async, HeadFolder} =
|
|
leveled_bookie:book_headfold(Bookie1,
|
|
?RIAK_TAG,
|
|
{HeadFoldFun, []},
|
|
true, true, false),
|
|
KeySizeList1 = lists:reverse(HeadFolder()),
|
|
io:format("Head fold with result size ~w~n", [length(KeySizeList1)]),
|
|
true = KeyCount == length(KeySizeList1),
|
|
|
|
{MidHeadKey, _MidSize} = lists:nth(KeyCount div 2, KeySizeList1),
|
|
FoldThrowFun =
|
|
fun(FoldFun) ->
|
|
fun(B, K, PO, Acc) ->
|
|
case K > MidHeadKey of
|
|
true ->
|
|
throw({stop_fold, Acc});
|
|
false ->
|
|
FoldFun(B, K, PO, Acc)
|
|
end
|
|
end
|
|
end,
|
|
{async, HeadFolderToMidK} =
|
|
leveled_bookie:book_headfold(Bookie1,
|
|
?RIAK_TAG,
|
|
{FoldThrowFun(HeadFoldFun), []},
|
|
true, true, false),
|
|
KeySizeList2 = lists:reverse(CatchingFold(HeadFolderToMidK)),
|
|
io:format("Head fold with result size ~w~n", [length(KeySizeList2)]),
|
|
true = KeyCount div 2 == length(KeySizeList2),
|
|
|
|
ObjFoldFun =
|
|
fun(_B, K, V, Acc) ->
|
|
[{K,byte_size(V)}|Acc]
|
|
end,
|
|
{async, ObjectFolderKO} =
|
|
leveled_bookie:book_objectfold(Bookie1,
|
|
?RIAK_TAG,
|
|
{ObjFoldFun, []},
|
|
false,
|
|
key_order),
|
|
ObjSizeList1 = lists:reverse(ObjectFolderKO()),
|
|
io:format("Obj fold with result size ~w~n", [length(ObjSizeList1)]),
|
|
true = KeyCount == length(ObjSizeList1),
|
|
|
|
{async, ObjFolderToMidK} =
|
|
leveled_bookie:book_objectfold(Bookie1,
|
|
?RIAK_TAG,
|
|
{FoldThrowFun(ObjFoldFun), []},
|
|
false,
|
|
key_order),
|
|
ObjSizeList2 = lists:reverse(CatchingFold(ObjFolderToMidK)),
|
|
io:format("Object fold with result size ~w~n", [length(ObjSizeList2)]),
|
|
true = KeyCount div 2 == length(ObjSizeList2),
|
|
|
|
% Object folds which are SQN order use a different path through the code,
|
|
% so testing that here. Note that it would not make sense to have a fold
|
|
% that was terminated by reaching a point in the key range .. as results
|
|
% will not be passed to the fold function in key order
|
|
{async, ObjectFolderSO} =
|
|
leveled_bookie:book_objectfold(Bookie1,
|
|
?RIAK_TAG,
|
|
{ObjFoldFun, []},
|
|
false,
|
|
sqn_order),
|
|
ObjSizeList1_SO = lists:reverse(ObjectFolderSO()),
|
|
io:format("Obj fold with result size ~w~n", [length(ObjSizeList1_SO)]),
|
|
true = KeyCount == length(ObjSizeList1_SO),
|
|
|
|
% Exit fold when we've reached a thousand accumulated obects
|
|
FoldThrowThousandFun =
|
|
fun(FoldFun) ->
|
|
fun(B, K, PO, Acc) ->
|
|
case length(Acc) == 1000 of
|
|
true ->
|
|
throw({stop_fold, Acc});
|
|
false ->
|
|
FoldFun(B, K, PO, Acc)
|
|
end
|
|
end
|
|
end,
|
|
{async, ObjFolderTo1K} =
|
|
leveled_bookie:book_objectfold(Bookie1,
|
|
?RIAK_TAG,
|
|
{FoldThrowThousandFun(ObjFoldFun), []},
|
|
false,
|
|
sqn_order),
|
|
ObjSizeList2_SO = lists:reverse(CatchingFold(ObjFolderTo1K)),
|
|
io:format("Object fold with result size ~w~n", [length(ObjSizeList2_SO)]),
|
|
true = 1000 == length(ObjSizeList2_SO),
|
|
|
|
ObjL2 = testutil:generate_objects(10,
|
|
binary_uuid,
|
|
[],
|
|
ObjectGen,
|
|
IndexGen,
|
|
"B2"),
|
|
ObjL3 = testutil:generate_objects(10,
|
|
binary_uuid,
|
|
[],
|
|
ObjectGen,
|
|
IndexGen,
|
|
"B3"),
|
|
ObjL4 = testutil:generate_objects(10,
|
|
binary_uuid,
|
|
[],
|
|
ObjectGen,
|
|
IndexGen,
|
|
"B4"),
|
|
testutil:riakload(Bookie1, ObjL2),
|
|
testutil:riakload(Bookie1, ObjL3),
|
|
testutil:riakload(Bookie1, ObjL4),
|
|
|
|
FBAccT = {fun(B, Acc) -> [B|Acc] end, []},
|
|
{async, BucketFolder} =
|
|
leveled_bookie:book_bucketlist(Bookie1, ?RIAK_TAG, FBAccT, all),
|
|
BucketList1 = lists:reverse(BucketFolder()),
|
|
io:format("bucket list with result size ~w~n", [length(BucketList1)]),
|
|
true = 4 == length(BucketList1),
|
|
|
|
StopAt3Fun =
|
|
fun(B, Acc) ->
|
|
Acc0 = [B|Acc],
|
|
case B of
|
|
<<"B3">> ->
|
|
throw({stop_fold, Acc0});
|
|
_ ->
|
|
Acc0
|
|
end
|
|
end,
|
|
|
|
{async, StopAt3BucketFolder} =
|
|
leveled_bookie:book_bucketlist(Bookie1,
|
|
?RIAK_TAG,
|
|
{StopAt3Fun, []},
|
|
all),
|
|
BucketListSA3 = lists:reverse(CatchingFold(StopAt3BucketFolder)),
|
|
io:format("bucket list with result ~w~n", [BucketListSA3]),
|
|
true = [<<"B2">>, <<"B3">>] == BucketListSA3,
|
|
|
|
|
|
ok = leveled_bookie:book_close(Bookie1),
|
|
testutil:reset_filestructure().
|
|
|
|
|
|
|
|
single_object_with2i(_Config) ->
|
|
% Load a single object with an integer and a binary
|
|
% index and query for it
|
|
RootPath = testutil:reset_filestructure(),
|
|
StartOpts1 = [{root_path, RootPath},
|
|
{max_journalsize, 5000000},
|
|
{sync_strategy, testutil:sync_strategy()}],
|
|
% low journal size to make sure > 1 created
|
|
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
|
|
{TestObject, _TestSpec} = testutil:generate_testobject(),
|
|
TestSpec = [{add, list_to_binary("integer_int"), 100},
|
|
{add, list_to_binary("binary_bin"), <<100:32/integer>>}],
|
|
ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
|
|
|
|
%% @TODO replace all index queries with new Top-Level API if tests
|
|
%% pass
|
|
{async, IdxFolder1} = leveled_bookie:book_indexfold(Bookie1,
|
|
"Bucket1",
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{list_to_binary("binary_bin"),
|
|
<<99:32/integer>>, <<101:32/integer>>},
|
|
{true, undefined}),
|
|
R1 = IdxFolder1(),
|
|
io:format("R1 of ~w~n", [R1]),
|
|
true = [{<<100:32/integer>>,"Key1"}] == R1,
|
|
|
|
IdxQ2 = {index_query,
|
|
"Bucket1",
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{list_to_binary("integer_int"),
|
|
99, 101},
|
|
{true, undefined}},
|
|
{async, IdxFolder2} = leveled_bookie:book_returnfolder(Bookie1, IdxQ2),
|
|
R2 = IdxFolder2(),
|
|
io:format("R2 of ~w~n", [R2]),
|
|
true = [{100,"Key1"}] == R2,
|
|
|
|
IdxQ3 = {index_query,
|
|
{"Bucket1", "Key1"},
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{list_to_binary("integer_int"),
|
|
99, 101},
|
|
{true, undefined}},
|
|
{async, IdxFolder3} = leveled_bookie:book_returnfolder(Bookie1, IdxQ3),
|
|
R3 = IdxFolder3(),
|
|
io:format("R2 of ~w~n", [R3]),
|
|
true = [{100,"Key1"}] == R3,
|
|
|
|
ok = leveled_bookie:book_close(Bookie1),
|
|
testutil:reset_filestructure().
|
|
|
|
small_load_with2i(_Config) ->
|
|
RootPath = testutil:reset_filestructure(),
|
|
StartOpts1 = [{root_path, RootPath},
|
|
{max_journalsize, 5000000},
|
|
{sync_strategy, testutil:sync_strategy()}],
|
|
% low journal size to make sure > 1 created
|
|
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
|
|
{TestObject, TestSpec} = testutil:generate_testobject(),
|
|
ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
|
|
testutil:check_forobject(Bookie1, TestObject),
|
|
testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"),
|
|
testutil:check_forobject(Bookie1, TestObject),
|
|
ObjectGen = testutil:get_compressiblevalue_andinteger(),
|
|
IndexGen = testutil:get_randomindexes_generator(8),
|
|
ObjL1 = testutil:generate_objects(10000,
|
|
uuid,
|
|
[],
|
|
ObjectGen,
|
|
IndexGen),
|
|
testutil:riakload(Bookie1, ObjL1),
|
|
ChkList1 = lists:sublist(lists:sort(ObjL1), 100),
|
|
testutil:check_forlist(Bookie1, ChkList1),
|
|
testutil:check_forobject(Bookie1, TestObject),
|
|
|
|
% Find all keys index, and then just the last key
|
|
IdxQ1 = {index_query,
|
|
"Bucket",
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{"idx1_bin", "#", "|"},
|
|
{true, undefined}},
|
|
{async, IdxFolder} = leveled_bookie:book_returnfolder(Bookie1, IdxQ1),
|
|
KeyList1 = lists:usort(IdxFolder()),
|
|
true = 10000 == length(KeyList1),
|
|
{LastTerm, LastKey} = lists:last(KeyList1),
|
|
IdxQ2 = {index_query,
|
|
{"Bucket", LastKey},
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{"idx1_bin", LastTerm, "|"},
|
|
{false, undefined}},
|
|
{async, IdxFolderLK} = leveled_bookie:book_returnfolder(Bookie1, IdxQ2),
|
|
KeyList2 = lists:usort(IdxFolderLK()),
|
|
io:format("List should be last key ~w ~w~n", [LastKey, KeyList2]),
|
|
true = 1 == length(KeyList2),
|
|
|
|
%% Delete the objects from the ChkList removing the indexes
|
|
lists:foreach(fun({_RN, Obj, Spc}) ->
|
|
DSpc = lists:map(fun({add, F, T}) ->
|
|
{remove, F, T}
|
|
end,
|
|
Spc),
|
|
{B, K} =
|
|
{testutil:get_bucket(Obj), testutil:get_key(Obj)},
|
|
testutil:book_riakdelete(Bookie1, B, K, DSpc)
|
|
end,
|
|
ChkList1),
|
|
%% Get the Buckets Keys and Hashes for the whole bucket
|
|
FoldObjectsFun = fun(B, K, V, Acc) -> [{B, K, erlang:phash2(V)}|Acc]
|
|
end,
|
|
|
|
{async, HTreeF1} = leveled_bookie:book_objectfold(Bookie1,
|
|
?RIAK_TAG,
|
|
{FoldObjectsFun, []},
|
|
false),
|
|
|
|
KeyHashList1 = HTreeF1(),
|
|
{async, HTreeF2} = leveled_bookie:book_objectfold(Bookie1,
|
|
?RIAK_TAG,
|
|
"Bucket",
|
|
all,
|
|
{FoldObjectsFun, []},
|
|
false),
|
|
KeyHashList2 = HTreeF2(),
|
|
{async, HTreeF3} = leveled_bookie:book_objectfold(Bookie1,
|
|
?RIAK_TAG,
|
|
"Bucket",
|
|
{"idx1_bin", "#", "|"},
|
|
{FoldObjectsFun, []},
|
|
false),
|
|
KeyHashList3 = HTreeF3(),
|
|
true = 9901 == length(KeyHashList1), % also includes the test object
|
|
true = 9900 == length(KeyHashList2),
|
|
true = 9900 == length(KeyHashList3),
|
|
|
|
SumIntFun = fun(_B, _K, Obj, Acc) ->
|
|
{I, _Bin} = testutil:get_value(Obj),
|
|
Acc + I
|
|
end,
|
|
BucketObjQ =
|
|
{foldobjects_bybucket, ?RIAK_TAG, "Bucket", all, {SumIntFun, 0}, true},
|
|
{async, Sum1} = leveled_bookie:book_returnfolder(Bookie1, BucketObjQ),
|
|
Total1 = Sum1(),
|
|
io:format("Total from summing all I is ~w~n", [Total1]),
|
|
SumFromObjLFun =
|
|
fun(Obj, Acc) ->
|
|
{I, _Bin} = testutil:get_value_from_objectlistitem(Obj),
|
|
Acc + I
|
|
end,
|
|
ObjL1Total =
|
|
lists:foldl(SumFromObjLFun, 0, ObjL1),
|
|
ChkList1Total =
|
|
lists:foldl(SumFromObjLFun, 0, ChkList1),
|
|
io:format("Total in original object list ~w and from removed list ~w~n",
|
|
[ObjL1Total, ChkList1Total]),
|
|
|
|
Total1 = ObjL1Total - ChkList1Total,
|
|
|
|
ok = leveled_bookie:book_close(Bookie1),
|
|
|
|
{ok, Bookie2} = leveled_bookie:book_start(StartOpts1),
|
|
|
|
{async, Sum2} = leveled_bookie:book_returnfolder(Bookie2, BucketObjQ),
|
|
Total2 = Sum2(),
|
|
true = Total2 == Total1,
|
|
|
|
FoldBucketsFun = fun(B, Acc) -> sets:add_element(B, Acc) end,
|
|
|
|
% this should find Bucket and Bucket1 - as we can now find string-based
|
|
% buckets using bucket_list - i.e. it isn't just binary buckets now
|
|
{async, BL} = leveled_bookie:book_bucketlist(Bookie2, ?RIAK_TAG, {FoldBucketsFun, sets:new()}, all),
|
|
true = sets:size(BL()) == 2,
|
|
|
|
ok = leveled_bookie:book_close(Bookie2),
|
|
testutil:reset_filestructure().
|
|
|
|
|
|
query_count(_Config) ->
|
|
RootPath = testutil:reset_filestructure(),
|
|
{ok, Book1} = leveled_bookie:book_start(RootPath,
|
|
2000,
|
|
50000000,
|
|
testutil:sync_strategy()),
|
|
BucketBin = list_to_binary("Bucket"),
|
|
{TestObject, TestSpec} = testutil:generate_testobject(BucketBin,
|
|
term_to_binary("Key1"),
|
|
"Value1",
|
|
[],
|
|
[{"MDK1", "MDV1"}]),
|
|
ok = testutil:book_riakput(Book1, TestObject, TestSpec),
|
|
testutil:check_forobject(Book1, TestObject),
|
|
testutil:check_formissingobject(Book1, "Bucket1", "Key2"),
|
|
testutil:check_forobject(Book1, TestObject),
|
|
lists:foreach(fun(_X) ->
|
|
V = testutil:get_compressiblevalue(),
|
|
Indexes = testutil:get_randomindexes_generator(8),
|
|
SW = os:timestamp(),
|
|
ObjL1 = testutil:generate_objects(10000,
|
|
binary_uuid,
|
|
[],
|
|
V,
|
|
Indexes),
|
|
testutil:riakload(Book1, ObjL1),
|
|
io:format("Put of 10000 objects with 8 index entries "
|
|
++
|
|
"each completed in ~w microseconds~n",
|
|
[timer:now_diff(os:timestamp(), SW)])
|
|
end,
|
|
lists:seq(1, 8)),
|
|
testutil:check_forobject(Book1, TestObject),
|
|
Total = lists:foldl(fun(X, Acc) ->
|
|
IdxF = "idx" ++ integer_to_list(X) ++ "_bin",
|
|
T = count_termsonindex(BucketBin,
|
|
IdxF,
|
|
Book1,
|
|
?KEY_ONLY),
|
|
io:format("~w terms found on index ~s~n",
|
|
[T, IdxF]),
|
|
Acc + T
|
|
end,
|
|
0,
|
|
lists:seq(1, 8)),
|
|
ok = case Total of
|
|
640000 ->
|
|
ok
|
|
end,
|
|
Index1Count = count_termsonindex(BucketBin,
|
|
"idx1_bin",
|
|
Book1,
|
|
?KEY_ONLY),
|
|
ok = leveled_bookie:book_close(Book1),
|
|
{ok, Book2} = leveled_bookie:book_start(RootPath,
|
|
1000,
|
|
50000000,
|
|
testutil:sync_strategy()),
|
|
Index1Count = count_termsonindex(BucketBin,
|
|
"idx1_bin",
|
|
Book2,
|
|
?KEY_ONLY),
|
|
NameList = testutil:name_list(),
|
|
TotalNameByName = lists:foldl(fun({_X, Name}, Acc) ->
|
|
{ok, Regex} = re:compile("[0-9]+" ++
|
|
Name),
|
|
SW = os:timestamp(),
|
|
T = count_termsonindex(BucketBin,
|
|
"idx1_bin",
|
|
Book2,
|
|
{false,
|
|
Regex}),
|
|
TD = timer:now_diff(os:timestamp(),
|
|
SW),
|
|
io:format("~w terms found on " ++
|
|
"index idx1 with a " ++
|
|
"regex in ~w " ++
|
|
"microseconds~n",
|
|
[T, TD]),
|
|
Acc + T
|
|
end,
|
|
0,
|
|
NameList),
|
|
ok = case TotalNameByName of
|
|
Index1Count ->
|
|
ok
|
|
end,
|
|
{ok, RegMia} = re:compile("[0-9]+Mia"),
|
|
Query1 = {index_query,
|
|
BucketBin,
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{"idx2_bin", "2000", "2000|"},
|
|
{false, RegMia}},
|
|
{async,
|
|
Mia2KFolder1} = leveled_bookie:book_returnfolder(Book2, Query1),
|
|
Mia2000Count1 = length(Mia2KFolder1()),
|
|
Query2 = {index_query,
|
|
BucketBin,
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{"idx2_bin", "2000", "2001"},
|
|
{true, undefined}},
|
|
{async,
|
|
Mia2KFolder2} = leveled_bookie:book_returnfolder(Book2, Query2),
|
|
Mia2000Count2 = lists:foldl(fun({Term, _Key}, Acc) ->
|
|
case re:run(Term, RegMia) of
|
|
nomatch ->
|
|
Acc;
|
|
_ ->
|
|
Acc + 1
|
|
end end,
|
|
0,
|
|
Mia2KFolder2()),
|
|
ok = case Mia2000Count2 of
|
|
Mia2000Count1 when Mia2000Count1 > 0 ->
|
|
io:format("Mia2000 counts match at ~w~n",
|
|
[Mia2000Count1]),
|
|
ok
|
|
end,
|
|
{ok, RxMia2K} = re:compile("^2000[0-9]+Mia"),
|
|
Query3 = {index_query,
|
|
BucketBin,
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{"idx2_bin", "1980", "2100"},
|
|
{false, RxMia2K}},
|
|
{async,
|
|
Mia2KFolder3} = leveled_bookie:book_returnfolder(Book2, Query3),
|
|
Mia2000Count1 = length(Mia2KFolder3()),
|
|
|
|
V9 = testutil:get_compressiblevalue(),
|
|
Indexes9 = testutil:get_randomindexes_generator(8),
|
|
[{_RN, Obj9, Spc9}] = testutil:generate_objects(1,
|
|
binary_uuid,
|
|
[],
|
|
V9,
|
|
Indexes9),
|
|
ok = testutil:book_riakput(Book2, Obj9, Spc9),
|
|
R9 = lists:map(fun({add, IdxF, IdxT}) ->
|
|
Q = {index_query,
|
|
BucketBin,
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{IdxF, IdxT, IdxT},
|
|
?KEY_ONLY},
|
|
R = leveled_bookie:book_returnfolder(Book2, Q),
|
|
{async, Fldr} = R,
|
|
case length(Fldr()) of
|
|
X when X > 0 ->
|
|
{IdxF, IdxT, X}
|
|
end
|
|
end,
|
|
Spc9),
|
|
Spc9Del = lists:map(fun({add, IdxF, IdxT}) -> {remove, IdxF, IdxT} end,
|
|
Spc9),
|
|
ok = testutil:book_riakput(Book2, Obj9, Spc9Del),
|
|
lists:foreach(fun({IdxF, IdxT, X}) ->
|
|
Q = {index_query,
|
|
BucketBin,
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{IdxF, IdxT, IdxT},
|
|
?KEY_ONLY},
|
|
R = leveled_bookie:book_returnfolder(Book2, Q),
|
|
{async, Fldr} = R,
|
|
case length(Fldr()) of
|
|
Y ->
|
|
Y = X - 1
|
|
end
|
|
end,
|
|
R9),
|
|
ok = leveled_bookie:book_close(Book2),
|
|
{ok, Book3} = leveled_bookie:book_start(RootPath,
|
|
2000,
|
|
50000000,
|
|
testutil:sync_strategy()),
|
|
lists:foreach(fun({IdxF, IdxT, X}) ->
|
|
Q = {index_query,
|
|
BucketBin,
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{IdxF, IdxT, IdxT},
|
|
?KEY_ONLY},
|
|
R = leveled_bookie:book_returnfolder(Book3, Q),
|
|
{async, Fldr} = R,
|
|
case length(Fldr()) of
|
|
Y ->
|
|
Y = X - 1
|
|
end
|
|
end,
|
|
R9),
|
|
ok = testutil:book_riakput(Book3, Obj9, Spc9),
|
|
ok = leveled_bookie:book_close(Book3),
|
|
{ok, Book4} = leveled_bookie:book_start(RootPath,
|
|
2000,
|
|
50000000,
|
|
testutil:sync_strategy()),
|
|
lists:foreach(fun({IdxF, IdxT, X}) ->
|
|
Q = {index_query,
|
|
BucketBin,
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{IdxF, IdxT, IdxT},
|
|
?KEY_ONLY},
|
|
R = leveled_bookie:book_returnfolder(Book4, Q),
|
|
{async, Fldr} = R,
|
|
case length(Fldr()) of
|
|
X ->
|
|
ok
|
|
end
|
|
end,
|
|
R9),
|
|
testutil:check_forobject(Book4, TestObject),
|
|
|
|
FoldBucketsFun = fun(B, Acc) -> sets:add_element(B, Acc) end,
|
|
BucketListQuery = {bucket_list,
|
|
?RIAK_TAG,
|
|
{FoldBucketsFun, sets:new()}},
|
|
{async, BLF1} = leveled_bookie:book_returnfolder(Book4, BucketListQuery),
|
|
SW_QA = os:timestamp(),
|
|
BucketSet1 = BLF1(),
|
|
io:format("Bucket set returned in ~w microseconds",
|
|
[timer:now_diff(os:timestamp(), SW_QA)]),
|
|
|
|
true = sets:size(BucketSet1) == 1,
|
|
|
|
ObjList10A = testutil:generate_objects(5000,
|
|
binary_uuid,
|
|
[],
|
|
V9,
|
|
Indexes9,
|
|
"BucketA"),
|
|
ObjList10B = testutil:generate_objects(5000,
|
|
binary_uuid,
|
|
[],
|
|
V9,
|
|
Indexes9,
|
|
"BucketB"),
|
|
ObjList10C = testutil:generate_objects(5000,
|
|
binary_uuid,
|
|
[],
|
|
V9,
|
|
Indexes9,
|
|
"BucketC"),
|
|
testutil:riakload(Book4, ObjList10A),
|
|
testutil:riakload(Book4, ObjList10B),
|
|
testutil:riakload(Book4, ObjList10C),
|
|
{async, BLF2} = leveled_bookie:book_returnfolder(Book4, BucketListQuery),
|
|
SW_QB = os:timestamp(),
|
|
BucketSet2 = BLF2(),
|
|
io:format("Bucket set returned in ~w microseconds",
|
|
[timer:now_diff(os:timestamp(), SW_QB)]),
|
|
true = sets:size(BucketSet2) == 4,
|
|
|
|
ok = leveled_bookie:book_close(Book4),
|
|
|
|
{ok, Book5} = leveled_bookie:book_start(RootPath,
|
|
2000,
|
|
50000000,
|
|
testutil:sync_strategy()),
|
|
{async, BLF3} = leveled_bookie:book_returnfolder(Book5, BucketListQuery),
|
|
SW_QC = os:timestamp(),
|
|
BucketSet3 = BLF3(),
|
|
io:format("Bucket set returned in ~w microseconds",
|
|
[timer:now_diff(os:timestamp(), SW_QC)]),
|
|
true = sets:size(BucketSet3) == 4,
|
|
|
|
ok = leveled_bookie:book_close(Book5),
|
|
|
|
testutil:reset_filestructure().
|
|
|
|
|
|
|
|
count_termsonindex(Bucket, IdxField, Book, QType) ->
|
|
lists:foldl(fun(X, Acc) ->
|
|
SW = os:timestamp(),
|
|
ST = integer_to_list(X),
|
|
ET = ST ++ "|",
|
|
Q = {index_query,
|
|
Bucket,
|
|
{fun testutil:foldkeysfun/3, []},
|
|
{IdxField, ST, ET},
|
|
QType},
|
|
R = leveled_bookie:book_returnfolder(Book, Q),
|
|
{async, Folder} = R,
|
|
Items = length(Folder()),
|
|
io:format("2i query from term ~s on index ~s took " ++
|
|
"~w microseconds~n",
|
|
[ST,
|
|
IdxField,
|
|
timer:now_diff(os:timestamp(), SW)]),
|
|
Acc + Items
|
|
end,
|
|
0,
|
|
lists:seq(190, 221)).
|
|
|
|
multibucket_fold(_Config) ->
|
|
RootPath = testutil:reset_filestructure(),
|
|
{ok, Bookie1} = leveled_bookie:book_start(RootPath,
|
|
2000,
|
|
50000000,
|
|
testutil:sync_strategy()),
|
|
ObjectGen = testutil:get_compressiblevalue_andinteger(),
|
|
IndexGen = fun() -> [] end,
|
|
ObjL1 = testutil:generate_objects(13000,
|
|
uuid,
|
|
[],
|
|
ObjectGen,
|
|
IndexGen,
|
|
{<<"Type1">>, <<"Bucket1">>}),
|
|
testutil:riakload(Bookie1, ObjL1),
|
|
ObjL2 = testutil:generate_objects(17000,
|
|
uuid,
|
|
[],
|
|
ObjectGen,
|
|
IndexGen,
|
|
<<"Bucket2">>),
|
|
testutil:riakload(Bookie1, ObjL2),
|
|
ObjL3 = testutil:generate_objects(7000,
|
|
uuid,
|
|
[],
|
|
ObjectGen,
|
|
IndexGen,
|
|
<<"Bucket3">>),
|
|
testutil:riakload(Bookie1, ObjL3),
|
|
ObjL4 = testutil:generate_objects(23000,
|
|
uuid,
|
|
[],
|
|
ObjectGen,
|
|
IndexGen,
|
|
{<<"Type2">>, <<"Bucket4">>}),
|
|
testutil:riakload(Bookie1, ObjL4),
|
|
|
|
FF = fun(B, K, _PO, Acc) ->
|
|
[{B, K}|Acc]
|
|
end,
|
|
FoldAccT = {FF, []},
|
|
|
|
{async, R1} =
|
|
leveled_bookie:book_headfold(Bookie1,
|
|
?RIAK_TAG,
|
|
{bucket_list,
|
|
[{<<"Type1">>, <<"Bucket1">>},
|
|
{<<"Type2">>, <<"Bucket4">>}]},
|
|
FoldAccT,
|
|
false,
|
|
true,
|
|
false),
|
|
|
|
O1 = length(R1()),
|
|
io:format("Result R1 of length ~w~n", [O1]),
|
|
|
|
{async, R2} =
|
|
leveled_bookie:book_headfold(Bookie1,
|
|
?RIAK_TAG,
|
|
{bucket_list,
|
|
[<<"Bucket2">>,
|
|
<<"Bucket3">>]},
|
|
{fun(_B, _K, _PO, Acc) ->
|
|
Acc +1
|
|
end,
|
|
0},
|
|
false, true, false),
|
|
O2 = R2(),
|
|
io:format("Result R2 of ~w~n", [O2]),
|
|
|
|
true = 36000 == O1,
|
|
true = 24000 == O2,
|
|
|
|
FoldBucketsFun = fun(B, Acc) -> [B|Acc] end,
|
|
{async, Folder} =
|
|
leveled_bookie:book_bucketlist(Bookie1,
|
|
?RIAK_TAG,
|
|
{FoldBucketsFun, []},
|
|
all),
|
|
BucketList = lists:reverse(Folder()),
|
|
ExpectedBucketList =
|
|
[{<<"Type1">>, <<"Bucket1">>}, {<<"Type2">>, <<"Bucket4">>},
|
|
<<"Bucket2">>, <<"Bucket3">>],
|
|
io:format("BucketList ~w", [BucketList]),
|
|
true = ExpectedBucketList == BucketList,
|
|
|
|
ok = leveled_bookie:book_close(Bookie1),
|
|
testutil:reset_filestructure().
|
|
|
|
rotating_objects(_Config) ->
|
|
RootPath = testutil:reset_filestructure(),
|
|
ok = testutil:rotating_object_check(RootPath, "Bucket1", 10),
|
|
ok = testutil:rotating_object_check(RootPath, "Bucket2", 200),
|
|
ok = testutil:rotating_object_check(RootPath, "Bucket3", 800),
|
|
ok = testutil:rotating_object_check(RootPath, "Bucket4", 1600),
|
|
ok = testutil:rotating_object_check(RootPath, "Bucket5", 3200),
|
|
ok = testutil:rotating_object_check(RootPath, "Bucket6", 9600),
|
|
testutil:reset_filestructure().
|
|
|
|
foldobjects_bybucket_range(_Config) ->
|
|
RootPath = testutil:reset_filestructure(),
|
|
{ok, Bookie1} = leveled_bookie:book_start(RootPath,
|
|
2000,
|
|
50000000,
|
|
testutil:sync_strategy()),
|
|
ObjectGen = testutil:get_compressiblevalue_andinteger(),
|
|
IndexGen = fun() -> [] end,
|
|
ObjL1 = testutil:generate_objects(1300,
|
|
{fixed_binary, 1},
|
|
[],
|
|
ObjectGen,
|
|
IndexGen,
|
|
<<"Bucket1">>),
|
|
testutil:riakload(Bookie1, ObjL1),
|
|
|
|
FoldKeysFun = fun(_B, K,_V, Acc) ->
|
|
[ K |Acc]
|
|
end,
|
|
|
|
StartKey = testutil:fixed_bin_key(123),
|
|
EndKey = testutil:fixed_bin_key(779),
|
|
|
|
{async, Folder} = leveled_bookie:book_objectfold(Bookie1,
|
|
?RIAK_TAG,
|
|
<<"Bucket1">>,
|
|
{StartKey, EndKey}, {FoldKeysFun, []},
|
|
true
|
|
),
|
|
ResLen = length(Folder()),
|
|
io:format("Length of Result of folder ~w~n", [ResLen]),
|
|
true = 657 == ResLen,
|
|
|
|
{async, AllFolder} = leveled_bookie:book_objectfold(Bookie1,
|
|
?RIAK_TAG,
|
|
<<"Bucket1">>,
|
|
all,
|
|
{FoldKeysFun, []},
|
|
true
|
|
),
|
|
|
|
AllResLen = length(AllFolder()),
|
|
io:format("Length of Result of all keys folder ~w~n", [AllResLen]),
|
|
true = 1300 == AllResLen,
|
|
|
|
ok = leveled_bookie:book_close(Bookie1),
|
|
testutil:reset_filestructure().
|