Mas d31 nhskv16sst (#428)
* Add performance/profiling test Add test to perf_SUITE to do performance tests and also profile different activities in leveled. This can then be used to highlight functions with unexpectedly high execution times, and prove the impact of changes. Switch between riak_ctperf and riak_fullperf to change from standard test (with profile option) to full-scale performance test * Change shape of default perfTest * Refactor SST Compare and contrast profile for guess, before and after refactor: pre ``` lists:map_1/2 313370 2.33 32379 [ 0.10] lists:foldl_1/3 956590 4.81 66992 [ 0.07] leveled_sst:'-expand_list_by_pointer/5-fun-0-'/4 925020 6.13 85318 [ 0.09] erlang:binary_to_term/1 3881 8.55 119012 [ 30.67] erlang:'++'/2 974322 11.55 160724 [ 0.16] lists:member/2 4000180 15.00 208697 [ 0.05] leveled_sst:find_pos/4 4029220 21.01 292347 [ 0.07] leveled_sst:member_check/2 4000000 21.17 294601 [ 0.07] -------------------------------------------------- -------- ------- ------- [----------] Total: 16894665 100.00% 1391759 [ 0.08] ``` post ``` lists:map_1/2 63800 0.79 6795 [ 0.11] erlang:term_to_binary/1 15726 0.81 6950 [ 0.44] lists:keyfind/3 180967 0.92 7884 [ 0.04] erlang:spawn_link/3 15717 1.08 9327 [ 0.59] leveled_sst:'-read_slots/5-fun-1-'/8 31270 1.15 9895 [ 0.32] gen:do_call/4 7881 1.31 11243 [ 1.43] leveled_penciller:find_nextkey/8 180936 2.01 17293 [ 0.10] prim_file:pread_nif/3 15717 3.89 33437 [ 2.13] leveled_sst:find_pos/4 4028940 17.85 153554 [ 0.04] erlang:binary_to_term/1 15717 51.97 447048 [ 28.44] -------------------------------------------------- ------- ------- ------ [----------] Total: 6704100 100.00% 860233 [ 0.13] ``` * Update leveled_penciller.erl * Mas d31 nhskv16sstpcl (#426) Performance updates to leveled: - Refactoring of pointer expansion when fetching from leveled_sst files to avoid expensive list concatenation. - Refactoring of leveled_ebloom to make more flexible, reduce code, and improve check time. - Refactoring of querying within leveled_sst to reduce the number of blocks that need to be de-serialised per query. - Refactoring of the leveled_penciller's query key comparator, to make use of maps and simplify the filtering. - General speed-up of frequently called functions.
This commit is contained in:
Martin Sumner
GitHub
parent
49490c38ef
commit
c294570bce
12 changed files with 1817 additions and 2113 deletions
|
|
@ -34,8 +34,8 @@ expiring_indexes(_Config) ->
|
|||
% before). Confirm that replacing an object has the expected outcome, if
|
||||
% the IndexSpecs are updated as part of the request.
|
||||
KeyCount = 50000,
|
||||
Future = 60,
|
||||
% 1 minute - if running tests on a slow machine, may need to increase
|
||||
Future = 120,
|
||||
% 2 minutes - if running tests on a slow machine, may need to increase
|
||||
% this value
|
||||
RootPath = testutil:reset_filestructure(),
|
||||
StartOpts1 =
|
||||
|
|
@ -44,13 +44,30 @@ expiring_indexes(_Config) ->
|
|||
{max_journalobjectcount, 30000},
|
||||
{sync_strategy, testutil:sync_strategy()}],
|
||||
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
|
||||
|
||||
|
||||
SW1 = os:timestamp(),
|
||||
timer:sleep(1000),
|
||||
|
||||
V9 = testutil:get_compressiblevalue(),
|
||||
Indexes9 = testutil:get_randomindexes_generator(2),
|
||||
TempRiakObjects =
|
||||
testutil:generate_objects(
|
||||
KeyCount, binary_uuid, [], V9, Indexes9, "riakBucket"),
|
||||
|
||||
IBKL1 = testutil:stdload_expiring(Bookie1, KeyCount, Future),
|
||||
lists:foreach(
|
||||
fun({_RN, Obj, Spc}) ->
|
||||
testutil:book_tempriakput(
|
||||
Bookie1, Obj, Spc, leveled_util:integer_now() + Future)
|
||||
end,
|
||||
TempRiakObjects
|
||||
),
|
||||
timer:sleep(1000),
|
||||
% Wait a second after last key so that none loaded in the last second
|
||||
LoadTime = timer:now_diff(os:timestamp(), SW1)/1000000,
|
||||
io:format("Load of ~w std objects in ~w seconds~n", [KeyCount, LoadTime]),
|
||||
|
||||
timer:sleep(1000),
|
||||
SW2 = os:timestamp(),
|
||||
|
||||
FilterFun = fun({I, _B, _K}) -> lists:member(I, [5, 6, 7, 8]) end,
|
||||
|
|
@ -76,6 +93,25 @@ expiring_indexes(_Config) ->
|
|||
{async, I0Counter1} = CountI0Fold(),
|
||||
I0Count1 = I0Counter1(),
|
||||
|
||||
HeadFold =
|
||||
fun(LowTS, HighTS) ->
|
||||
leveled_bookie:book_headfold(
|
||||
Bookie1,
|
||||
?RIAK_TAG,
|
||||
{range, <<"riakBucket">>, all},
|
||||
{fun(_B, _K, _V, Acc) -> Acc + 1 end, 0},
|
||||
false, true, false,
|
||||
{testutil:convert_to_seconds(LowTS),
|
||||
testutil:convert_to_seconds(HighTS)},
|
||||
false
|
||||
)
|
||||
end,
|
||||
{async, HeadCount0Fun} = HeadFold(SW1, SW2),
|
||||
{async, HeadCount1Fun} = HeadFold(SW2, os:timestamp()),
|
||||
HeadCounts = {HeadCount0Fun(), HeadCount1Fun()},
|
||||
io:format("HeadCounts ~w before expiry~n", [HeadCounts]),
|
||||
{KeyCount, 0} = HeadCounts,
|
||||
|
||||
FoldFun = fun(BF, {IdxV, KeyF}, Acc) -> [{IdxV, BF, KeyF}|Acc] end,
|
||||
InitAcc = [],
|
||||
IndexFold =
|
||||
|
|
@ -145,6 +181,12 @@ expiring_indexes(_Config) ->
|
|||
true = QR4 == [],
|
||||
true = QR5 == [],
|
||||
|
||||
{async, HeadCount0ExpFun} = HeadFold(SW1, SW2),
|
||||
{async, HeadCount1ExpFun} = HeadFold(SW2, os:timestamp()),
|
||||
HeadCountsExp = {HeadCount0ExpFun(), HeadCount1ExpFun()},
|
||||
io:format("HeadCounts ~w after expiry~n", [HeadCountsExp]),
|
||||
{0, 0} = HeadCountsExp,
|
||||
|
||||
ok = leveled_bookie:book_close(Bookie1),
|
||||
testutil:reset_filestructure().
|
||||
|
||||
|
|
@ -379,12 +421,14 @@ single_object_with2i(_Config) ->
|
|||
|
||||
%% @TODO replace all index queries with new Top-Level API if tests
|
||||
%% pass
|
||||
{async, IdxFolder1} = leveled_bookie:book_indexfold(Bookie1,
|
||||
"Bucket1",
|
||||
{fun testutil:foldkeysfun/3, []},
|
||||
{list_to_binary("binary_bin"),
|
||||
<<99:32/integer>>, <<101:32/integer>>},
|
||||
{true, undefined}),
|
||||
{async, IdxFolder1} =
|
||||
leveled_bookie:book_indexfold(
|
||||
Bookie1,
|
||||
"Bucket1",
|
||||
{fun testutil:foldkeysfun/3, []},
|
||||
{list_to_binary("binary_bin"),
|
||||
<<99:32/integer>>, <<101:32/integer>>},
|
||||
{true, undefined}),
|
||||
R1 = IdxFolder1(),
|
||||
io:format("R1 of ~w~n", [R1]),
|
||||
true = [{<<100:32/integer>>,"Key1"}] == R1,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
-include("../include/leveled.hrl").
|
||||
|
||||
-export([book_riakput/3,
|
||||
book_tempriakput/4,
|
||||
book_riakdelete/4,
|
||||
book_riakget/3,
|
||||
book_riakhead/3,
|
||||
|
|
@ -182,6 +183,16 @@ book_riakput(Pid, RiakObject, IndexSpecs) ->
|
|||
IndexSpecs,
|
||||
?RIAK_TAG).
|
||||
|
||||
book_tempriakput(Pid, RiakObject, IndexSpecs, TTL) ->
|
||||
leveled_bookie:book_tempput(
|
||||
Pid,
|
||||
RiakObject#r_object.bucket,
|
||||
RiakObject#r_object.key,
|
||||
to_binary(v1, RiakObject),
|
||||
IndexSpecs,
|
||||
?RIAK_TAG,
|
||||
TTL).
|
||||
|
||||
book_riakdelete(Pid, Bucket, Key, IndexSpecs) ->
|
||||
leveled_bookie:book_put(Pid, Bucket, Key, delete, IndexSpecs, ?RIAK_TAG).
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue