8f29a6c40f
The 2i work now has tests for removals as well as regex etc. Some initial refactoring work has also been tried - to try and take some tasks of the critical path of push_mem. The primary change has been to avoid putting index keys into the gb_tree, and building the KeyChanges list in parallel to the gb_tree (now known as ObjectTree) within the Ledger Cache. Some initial experiments done as to changing the ETS table in the Penciller now that it will now be used for iterating - but that has been reverted for now.
269 lines
14 KiB
Erlang
269 lines
14 KiB
Erlang
-module(iterator_SUITE).
|
|
|
|
-include_lib("common_test/include/ct.hrl").
|
|
-include("include/leveled.hrl").
|
|
|
|
-define(KEY_ONLY, {false, undefined}).
|
|
|
|
-export([all/0]).
|
|
-export([simple_load_with2i/1,
|
|
simple_querycount/1]).
|
|
|
|
all() -> [simple_load_with2i,
|
|
simple_querycount].
|
|
|
|
|
|
simple_load_with2i(_Config) ->
|
|
RootPath = testutil:reset_filestructure(),
|
|
StartOpts1 = #bookie_options{root_path=RootPath,
|
|
max_journalsize=50000000},
|
|
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
|
|
{TestObject, TestSpec} = testutil:generate_testobject(),
|
|
ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec),
|
|
testutil:check_forobject(Bookie1, TestObject),
|
|
testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"),
|
|
testutil:check_forobject(Bookie1, TestObject),
|
|
ObjL1 = testutil:generate_objects(10000,
|
|
uuid,
|
|
[],
|
|
testutil:get_compressiblevalue(),
|
|
testutil:get_randomindexes_generator(8)),
|
|
lists:foreach(fun({_RN, Obj, Spc}) ->
|
|
leveled_bookie:book_riakput(Bookie1, Obj, Spc) end,
|
|
ObjL1),
|
|
ChkList1 = lists:sublist(lists:sort(ObjL1), 100),
|
|
testutil:check_forlist(Bookie1, ChkList1),
|
|
testutil:check_forobject(Bookie1, TestObject),
|
|
ok = leveled_bookie:book_close(Bookie1),
|
|
testutil:reset_filestructure().
|
|
|
|
|
|
simple_querycount(_Config) ->
|
|
RootPath = testutil:reset_filestructure(),
|
|
StartOpts1 = #bookie_options{root_path=RootPath,
|
|
max_journalsize=50000000},
|
|
{ok, Book1} = leveled_bookie:book_start(StartOpts1),
|
|
{TestObject, TestSpec} = testutil:generate_testobject(),
|
|
ok = leveled_bookie:book_riakput(Book1, TestObject, TestSpec),
|
|
testutil:check_forobject(Book1, TestObject),
|
|
testutil:check_formissingobject(Book1, "Bucket1", "Key2"),
|
|
testutil:check_forobject(Book1, TestObject),
|
|
lists:foreach(fun(_X) ->
|
|
V = testutil:get_compressiblevalue(),
|
|
Indexes = testutil:get_randomindexes_generator(8),
|
|
SW = os:timestamp(),
|
|
ObjL1 = testutil:generate_objects(10000,
|
|
uuid,
|
|
[],
|
|
V,
|
|
Indexes),
|
|
lists:foreach(fun({_RN, Obj, Spc}) ->
|
|
leveled_bookie:book_riakput(Book1,
|
|
Obj,
|
|
Spc)
|
|
end,
|
|
ObjL1),
|
|
io:format("Put of 10000 objects with 8 index entries "
|
|
++
|
|
"each completed in ~w microseconds~n",
|
|
[timer:now_diff(os:timestamp(), SW)])
|
|
end,
|
|
lists:seq(1, 8)),
|
|
testutil:check_forobject(Book1, TestObject),
|
|
Total = lists:foldl(fun(X, Acc) ->
|
|
IdxF = "idx" ++ integer_to_list(X) ++ "_bin",
|
|
T = count_termsonindex("Bucket",
|
|
IdxF,
|
|
Book1,
|
|
?KEY_ONLY),
|
|
io:format("~w terms found on index ~s~n",
|
|
[T, IdxF]),
|
|
Acc + T
|
|
end,
|
|
0,
|
|
lists:seq(1, 8)),
|
|
ok = case Total of
|
|
640000 ->
|
|
ok
|
|
end,
|
|
Index1Count = count_termsonindex("Bucket",
|
|
"idx1_bin",
|
|
Book1,
|
|
?KEY_ONLY),
|
|
ok = leveled_bookie:book_close(Book1),
|
|
{ok, Book2} = leveled_bookie:book_start(StartOpts1),
|
|
Index1Count = count_termsonindex("Bucket",
|
|
"idx1_bin",
|
|
Book2,
|
|
?KEY_ONLY),
|
|
NameList = testutil:name_list(),
|
|
TotalNameByName = lists:foldl(fun({_X, Name}, Acc) ->
|
|
{ok, Regex} = re:compile("[0-9]+" ++
|
|
Name),
|
|
SW = os:timestamp(),
|
|
T = count_termsonindex("Bucket",
|
|
"idx1_bin",
|
|
Book2,
|
|
{false,
|
|
Regex}),
|
|
TD = timer:now_diff(os:timestamp(),
|
|
SW),
|
|
io:format("~w terms found on " ++
|
|
"index idx1 with a " ++
|
|
"regex in ~w " ++
|
|
"microseconds~n",
|
|
[T, TD]),
|
|
Acc + T
|
|
end,
|
|
0,
|
|
NameList),
|
|
ok = case TotalNameByName of
|
|
Index1Count ->
|
|
ok
|
|
end,
|
|
{ok, RegMia} = re:compile("[0-9]+Mia"),
|
|
{async,
|
|
Mia2KFolder1} = leveled_bookie:book_returnfolder(Book2,
|
|
{index_query,
|
|
"Bucket",
|
|
{"idx2_bin",
|
|
"2000",
|
|
"2000~"},
|
|
{false,
|
|
RegMia}}),
|
|
Mia2000Count1 = length(Mia2KFolder1()),
|
|
{async,
|
|
Mia2KFolder2} = leveled_bookie:book_returnfolder(Book2,
|
|
{index_query,
|
|
"Bucket",
|
|
{"idx2_bin",
|
|
"2000",
|
|
"2001"},
|
|
{true,
|
|
undefined}}),
|
|
Mia2000Count2 = lists:foldl(fun({Term, _Key}, Acc) ->
|
|
case re:run(Term, RegMia) of
|
|
nomatch ->
|
|
Acc;
|
|
_ ->
|
|
Acc + 1
|
|
end end,
|
|
0,
|
|
Mia2KFolder2()),
|
|
ok = case Mia2000Count2 of
|
|
Mia2000Count1 when Mia2000Count1 > 0 ->
|
|
io:format("Mia2000 counts match at ~w~n",
|
|
[Mia2000Count1]),
|
|
ok
|
|
end,
|
|
{ok, RxMia2K} = re:compile("^2000[0-9]+Mia"),
|
|
{async,
|
|
Mia2KFolder3} = leveled_bookie:book_returnfolder(Book2,
|
|
{index_query,
|
|
"Bucket",
|
|
{"idx2_bin",
|
|
"1980",
|
|
"2100"},
|
|
{false,
|
|
RxMia2K}}),
|
|
Mia2000Count1 = length(Mia2KFolder3()),
|
|
|
|
V9 = testutil:get_compressiblevalue(),
|
|
Indexes9 = testutil:get_randomindexes_generator(8),
|
|
[{_RN, Obj9, Spc9}] = testutil:generate_objects(1, uuid, [], V9, Indexes9),
|
|
ok = leveled_bookie:book_riakput(Book2, Obj9, Spc9),
|
|
R9 = lists:map(fun({add, IdxF, IdxT}) ->
|
|
R = leveled_bookie:book_returnfolder(Book2,
|
|
{index_query,
|
|
"Bucket",
|
|
{IdxF,
|
|
IdxT,
|
|
IdxT},
|
|
?KEY_ONLY}),
|
|
{async, Fldr} = R,
|
|
case length(Fldr()) of
|
|
X when X > 0 ->
|
|
{IdxF, IdxT, X}
|
|
end
|
|
end,
|
|
Spc9),
|
|
Spc9Del = lists:map(fun({add, IdxF, IdxT}) -> {remove, IdxF, IdxT} end,
|
|
Spc9),
|
|
ok = leveled_bookie:book_riakput(Book2, Obj9, Spc9Del),
|
|
lists:foreach(fun({IdxF, IdxT, X}) ->
|
|
R = leveled_bookie:book_returnfolder(Book2,
|
|
{index_query,
|
|
"Bucket",
|
|
{IdxF,
|
|
IdxT,
|
|
IdxT},
|
|
?KEY_ONLY}),
|
|
{async, Fldr} = R,
|
|
case length(Fldr()) of
|
|
Y ->
|
|
Y = X - 1
|
|
end
|
|
end,
|
|
R9),
|
|
ok = leveled_bookie:book_close(Book2),
|
|
{ok, Book3} = leveled_bookie:book_start(StartOpts1),
|
|
lists:foreach(fun({IdxF, IdxT, X}) ->
|
|
R = leveled_bookie:book_returnfolder(Book3,
|
|
{index_query,
|
|
"Bucket",
|
|
{IdxF,
|
|
IdxT,
|
|
IdxT},
|
|
?KEY_ONLY}),
|
|
{async, Fldr} = R,
|
|
case length(Fldr()) of
|
|
Y ->
|
|
Y = X - 1
|
|
end
|
|
end,
|
|
R9),
|
|
ok = leveled_bookie:book_riakput(Book3, Obj9, Spc9),
|
|
{ok, Book4} = leveled_bookie:book_start(StartOpts1),
|
|
lists:foreach(fun({IdxF, IdxT, X}) ->
|
|
R = leveled_bookie:book_returnfolder(Book4,
|
|
{index_query,
|
|
"Bucket",
|
|
{IdxF,
|
|
IdxT,
|
|
IdxT},
|
|
?KEY_ONLY}),
|
|
{async, Fldr} = R,
|
|
case length(Fldr()) of
|
|
X ->
|
|
ok
|
|
end
|
|
end,
|
|
R9),
|
|
ok = leveled_bookie:book_close(Book4),
|
|
testutil:reset_filestructure().
|
|
|
|
|
|
|
|
count_termsonindex(Bucket, IdxField, Book, QType) ->
|
|
lists:foldl(fun(X, Acc) ->
|
|
SW = os:timestamp(),
|
|
ST = integer_to_list(X),
|
|
ET = ST ++ "~",
|
|
R = leveled_bookie:book_returnfolder(Book,
|
|
{index_query,
|
|
Bucket,
|
|
{IdxField,
|
|
ST,
|
|
ET},
|
|
QType}),
|
|
{async, Folder} = R,
|
|
Items = length(Folder()),
|
|
io:format("2i query from term ~s on index ~s took " ++
|
|
"~w microseconds~n",
|
|
[ST,
|
|
IdxField,
|
|
timer:now_diff(os:timestamp(), SW)]),
|
|
Acc + Items
|
|
end,
|
|
0,
|
|
lists:seq(1901, 2218)).
|