Mas d31 nhskv16sst (#428)
* Add performance/profiling test Add test to perf_SUITE to do performance tests and also profile different activities in leveled. This can then be used to highlight functions with unexpectedly high execution times, and prove the impact of changes. Switch between riak_ctperf and riak_fullperf to change from standard test (with profile option) to full-scale performance test * Change shape of default perfTest * Refactor SST Compare and contrast profile for guess, before and after refactor: pre ``` lists:map_1/2 313370 2.33 32379 [ 0.10] lists:foldl_1/3 956590 4.81 66992 [ 0.07] leveled_sst:'-expand_list_by_pointer/5-fun-0-'/4 925020 6.13 85318 [ 0.09] erlang:binary_to_term/1 3881 8.55 119012 [ 30.67] erlang:'++'/2 974322 11.55 160724 [ 0.16] lists:member/2 4000180 15.00 208697 [ 0.05] leveled_sst:find_pos/4 4029220 21.01 292347 [ 0.07] leveled_sst:member_check/2 4000000 21.17 294601 [ 0.07] -------------------------------------------------- -------- ------- ------- [----------] Total: 16894665 100.00% 1391759 [ 0.08] ``` post ``` lists:map_1/2 63800 0.79 6795 [ 0.11] erlang:term_to_binary/1 15726 0.81 6950 [ 0.44] lists:keyfind/3 180967 0.92 7884 [ 0.04] erlang:spawn_link/3 15717 1.08 9327 [ 0.59] leveled_sst:'-read_slots/5-fun-1-'/8 31270 1.15 9895 [ 0.32] gen:do_call/4 7881 1.31 11243 [ 1.43] leveled_penciller:find_nextkey/8 180936 2.01 17293 [ 0.10] prim_file:pread_nif/3 15717 3.89 33437 [ 2.13] leveled_sst:find_pos/4 4028940 17.85 153554 [ 0.04] erlang:binary_to_term/1 15717 51.97 447048 [ 28.44] -------------------------------------------------- ------- ------- ------ [----------] Total: 6704100 100.00% 860233 [ 0.13] ``` * Update leveled_penciller.erl * Mas d31 nhskv16sstpcl (#426) Performance updates to leveled: - Refactoring of pointer expansion when fetching from leveled_sst files to avoid expensive list concatenation. - Refactoring of leveled_ebloom to make more flexible, reduce code, and improve check time. - Refactoring of querying within leveled_sst to reduce the number of blocks that need to be de-serialised per query. - Refactoring of the leveled_penciller's query key comparator, to make use of maps and simplify the filtering. - General speed-up of frequently called functions.
This commit is contained in:
Martin Sumner
GitHub
parent
49490c38ef
commit
c294570bce
12 changed files with 1817 additions and 2113 deletions
|
|
@ -451,23 +451,28 @@ key_lookup(Manifest, LevelIdx, Key) ->
|
|||
-spec query_manifest(
|
||||
manifest(),
|
||||
leveled_codec:ledger_key(),
|
||||
leveled_codec:ledger_key()) -> list().
|
||||
leveled_codec:ledger_key())
|
||||
-> list(
|
||||
{lsm_level(),
|
||||
list({next, manifest_entry(), leveled_codec:ledger_key()})}).
|
||||
query_manifest(Manifest, StartKey, EndKey) ->
|
||||
SetupFoldFun =
|
||||
fun(Level, Acc) ->
|
||||
Pointers =
|
||||
range_lookup(Manifest, Level, StartKey, EndKey),
|
||||
case Pointers of
|
||||
[] -> Acc;
|
||||
PL -> Acc ++ [{Level, PL}]
|
||||
case range_lookup(Manifest, Level, StartKey, EndKey) of
|
||||
[] ->
|
||||
Acc;
|
||||
Pointers ->
|
||||
[{Level, Pointers}|Acc]
|
||||
end
|
||||
end,
|
||||
lists:foldl(SetupFoldFun, [], lists:seq(0, ?MAX_LEVELS - 1)).
|
||||
|
||||
-spec range_lookup(manifest(),
|
||||
integer(),
|
||||
leveled_codec:ledger_key(),
|
||||
leveled_codec:ledger_key()) -> list().
|
||||
-spec range_lookup(
|
||||
manifest(),
|
||||
integer(),
|
||||
leveled_codec:ledger_key(),
|
||||
leveled_codec:ledger_key())
|
||||
-> list({next, manifest_entry(), leveled_codec:ledger_key()}).
|
||||
%% @doc
|
||||
%% Return a list of manifest_entry pointers at this level which cover the
|
||||
%% key query range.
|
||||
|
|
@ -478,10 +483,11 @@ range_lookup(Manifest, LevelIdx, StartKey, EndKey) ->
|
|||
end,
|
||||
range_lookup_int(Manifest, LevelIdx, StartKey, EndKey, MakePointerFun).
|
||||
|
||||
-spec merge_lookup(manifest(),
|
||||
integer(),
|
||||
leveled_codec:ledger_key(),
|
||||
leveled_codec:ledger_key()) -> list().
|
||||
-spec merge_lookup(
|
||||
manifest(),
|
||||
integer(),
|
||||
leveled_codec:ledger_key(),
|
||||
leveled_codec:ledger_key()) -> list({next, manifest_entry(), all}).
|
||||
%% @doc
|
||||
%% Return a list of manifest_entry pointers at this level which cover the
|
||||
%% key query range, only all keys in the files should be included in the
|
||||
|
|
@ -494,8 +500,8 @@ merge_lookup(Manifest, LevelIdx, StartKey, EndKey) ->
|
|||
range_lookup_int(Manifest, LevelIdx, StartKey, EndKey, MakePointerFun).
|
||||
|
||||
|
||||
-spec mergefile_selector(manifest(), integer(), selector_strategy())
|
||||
-> manifest_entry().
|
||||
-spec mergefile_selector(
|
||||
manifest(), integer(), selector_strategy()) -> manifest_entry().
|
||||
%% @doc
|
||||
%% An algorithm for discovering which files to merge ....
|
||||
%% We can find the most optimal file:
|
||||
|
|
@ -511,13 +517,15 @@ mergefile_selector(Manifest, LevelIdx, _Strategy) when LevelIdx =< 1 ->
|
|||
Level = array:get(LevelIdx, Manifest#manifest.levels),
|
||||
lists:nth(leveled_rand:uniform(length(Level)), Level);
|
||||
mergefile_selector(Manifest, LevelIdx, random) ->
|
||||
Level = leveled_tree:to_list(array:get(LevelIdx,
|
||||
Manifest#manifest.levels)),
|
||||
Level =
|
||||
leveled_tree:to_list(
|
||||
array:get(LevelIdx, Manifest#manifest.levels)),
|
||||
{_SK, ME} = lists:nth(leveled_rand:uniform(length(Level)), Level),
|
||||
ME;
|
||||
mergefile_selector(Manifest, LevelIdx, {grooming, ScoringFun}) ->
|
||||
Level = leveled_tree:to_list(array:get(LevelIdx,
|
||||
Manifest#manifest.levels)),
|
||||
Level =
|
||||
leveled_tree:to_list(
|
||||
array:get(LevelIdx, Manifest#manifest.levels)),
|
||||
SelectorFun =
|
||||
fun(_I, Acc) ->
|
||||
{_SK, ME} = lists:nth(leveled_rand:uniform(length(Level)), Level),
|
||||
|
|
@ -555,12 +563,12 @@ add_snapshot(Manifest, Pid, Timeout) ->
|
|||
ManSQN = Manifest#manifest.manifest_sqn,
|
||||
case Manifest#manifest.min_snapshot_sqn of
|
||||
0 ->
|
||||
Manifest#manifest{snapshots = SnapList0,
|
||||
min_snapshot_sqn = ManSQN};
|
||||
Manifest#manifest{
|
||||
snapshots = SnapList0, min_snapshot_sqn = ManSQN};
|
||||
N ->
|
||||
N0 = min(N, ManSQN),
|
||||
Manifest#manifest{snapshots = SnapList0,
|
||||
min_snapshot_sqn = N0}
|
||||
Manifest#manifest{
|
||||
snapshots = SnapList0, min_snapshot_sqn = N0}
|
||||
end.
|
||||
|
||||
-spec release_snapshot(manifest(), pid()|atom()) -> manifest().
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue