Merge pull request #118 from martinsumner/mas-i117-factor4scale
Mas i117 factor4scale
This commit is contained in:
Martin Sumner
GitHub
commit
4d3151752f
3 changed files with 39 additions and 12 deletions
|
|
@ -125,10 +125,12 @@
|
|||
{"P0032",
|
||||
{info, "Fetch head timing with sample_count=~w and level timings of"
|
||||
++ " foundmem_time=~w found0_time=~w found1_time=~w"
|
||||
++ " found2_time=~w foundlower_time=~w missed_time=~w"
|
||||
++ " found2_time=~w found3_time=~w foundlower_time=~w"
|
||||
++ " missed_time=~w"
|
||||
++ " with counts of"
|
||||
++ " foundmem_count=~w found0_count=~w found1_count=~w"
|
||||
++ " found2_count=~w foundlower_count=~w missed_count=~w"}},
|
||||
++ " found2_count=~w found3_count=~w foundlower_count=~w"
|
||||
++ " missed_count=~w"}},
|
||||
{"P0033",
|
||||
{error, "Corrupted manifest file at path ~s to be ignored "
|
||||
++ "due to error ~w"}},
|
||||
|
|
|
|||
|
|
@ -200,9 +200,22 @@
|
|||
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
|
||||
-define(LEVEL_SCALEFACTOR, [{0, 0}, {1, 8}, {2, 64}, {3, 512},
|
||||
{4, 4096}, {5, 32768}, {6, 262144},
|
||||
{7, infinity}]).
|
||||
-define(LEVEL_SCALEFACTOR,
|
||||
[{0, 0},
|
||||
{1, 4}, {2, 16}, {3, 64}, % Factor of 4
|
||||
{4, 384}, {5, 2304}, % Factor of 6
|
||||
{6, 18432}, % Factor of 8
|
||||
{7, infinity}]).
|
||||
% As an alternative to going up by a factor of 8 at each level,
|
||||
% increase by a factor of 4 at young levels - to make early
|
||||
% compaction jobs shorter.
|
||||
%
|
||||
% There are 32K keys per files => with 4096 files there are 100M
|
||||
% keys supported,
|
||||
|
||||
% 600M keys is supported before hitting the infinite level.
|
||||
% At o(10) trillion keys behaviour may become increasingly
|
||||
% difficult to predict.
|
||||
-define(MAX_LEVELS, 8).
|
||||
-define(MAX_WORK_WAIT, 300).
|
||||
-define(MANIFEST_FP, "ledger_manifest").
|
||||
|
|
@ -259,12 +272,14 @@
|
|||
found0_time = 0 :: integer(),
|
||||
found1_time = 0 :: integer(),
|
||||
found2_time = 0 :: integer(),
|
||||
found3_time = 0 :: integer(),
|
||||
foundlower_time = 0 :: integer(),
|
||||
missed_time = 0 :: integer(),
|
||||
foundmem_count = 0 :: integer(),
|
||||
found0_count = 0 :: integer(),
|
||||
found1_count = 0 :: integer(),
|
||||
found2_count = 0 :: integer(),
|
||||
found3_count = 0 :: integer(),
|
||||
foundlower_count = 0 :: integer(),
|
||||
missed_count = 0 :: integer()}).
|
||||
|
||||
|
|
@ -1500,12 +1515,14 @@ log_timings(Timings) ->
|
|||
Timings#pcl_timings.found0_time,
|
||||
Timings#pcl_timings.found1_time,
|
||||
Timings#pcl_timings.found2_time,
|
||||
Timings#pcl_timings.found3_time,
|
||||
Timings#pcl_timings.foundlower_time,
|
||||
Timings#pcl_timings.missed_time,
|
||||
Timings#pcl_timings.foundmem_count,
|
||||
Timings#pcl_timings.found0_count,
|
||||
Timings#pcl_timings.found1_count,
|
||||
Timings#pcl_timings.found2_count,
|
||||
Timings#pcl_timings.found3_count,
|
||||
Timings#pcl_timings.foundlower_count,
|
||||
Timings#pcl_timings.missed_count]).
|
||||
|
||||
|
|
@ -1543,6 +1560,10 @@ update_timings(SW, Timings, Result, Stage) ->
|
|||
L2T = Timings#pcl_timings.found2_time + Timer,
|
||||
L2C = Timings#pcl_timings.found2_count + 1,
|
||||
Timings0#pcl_timings{found2_time = L2T, found2_count = L2C};
|
||||
{_, 3} ->
|
||||
L3T = Timings#pcl_timings.found3_time + Timer,
|
||||
L3C = Timings#pcl_timings.found3_count + 1,
|
||||
Timings0#pcl_timings{found3_time = L3T, found3_count = L3C};
|
||||
_ ->
|
||||
LLT = Timings#pcl_timings.foundlower_time + Timer,
|
||||
LLC = Timings#pcl_timings.foundlower_count + 1,
|
||||
|
|
@ -1956,7 +1977,7 @@ foldwithimm_simple_test() ->
|
|||
create_file_test() ->
|
||||
{RP, Filename} = {"../test/", "new_file.sst"},
|
||||
ok = file:write_file(filename:join(RP, Filename), term_to_binary("hello")),
|
||||
KVL = lists:usort(generate_randomkeys({10000, 0})),
|
||||
KVL = lists:usort(generate_randomkeys({50000, 0})),
|
||||
Tree = leveled_tree:from_orderedlist(KVL, ?CACHE_TYPE),
|
||||
FetchFun = fun(Slot) -> lists:nth(Slot, [Tree]) end,
|
||||
{ok,
|
||||
|
|
@ -1966,7 +1987,7 @@ create_file_test() ->
|
|||
1,
|
||||
FetchFun,
|
||||
undefined,
|
||||
10000,
|
||||
50000,
|
||||
native),
|
||||
lists:foreach(fun(X) ->
|
||||
case checkready(SP) of
|
||||
|
|
@ -2007,7 +2028,8 @@ timings_test() ->
|
|||
?assertMatch(3, T2#pcl_timings.sample_count),
|
||||
?assertMatch(true, T2#pcl_timings.foundlower_time > T2#pcl_timings.found2_time),
|
||||
?assertMatch(1, T2#pcl_timings.found2_count),
|
||||
?assertMatch(2, T2#pcl_timings.foundlower_count).
|
||||
?assertMatch(1, T2#pcl_timings.found3_count),
|
||||
?assertMatch(1, T2#pcl_timings.foundlower_count).
|
||||
|
||||
|
||||
coverage_cheat_test() ->
|
||||
|
|
|
|||
|
|
@ -1517,10 +1517,13 @@ binaryslot_trimmedlist(FullBin, StartKey, EndKey, PressMethod) ->
|
|||
crc_check_slot(FullBin) ->
|
||||
<<CRC32PBL:32/integer,
|
||||
PosBL:32/integer,
|
||||
CRC32H:32/integer,
|
||||
Header:PosBL/binary,
|
||||
Blocks/binary>> = FullBin,
|
||||
case {hmac(Header), hmac(PosBL)} of
|
||||
CRC32H:32/integer,
|
||||
Rest/binary>> = FullBin,
|
||||
PosBL0 = min(PosBL, byte_size(FullBin) - 3),
|
||||
% If the position has been bit-flipped to beyond the maximum paossible
|
||||
% length, use the maximum possible length
|
||||
<<Header:PosBL0/binary, Blocks/binary>> = Rest,
|
||||
case {hmac(Header), hmac(PosBL0)} of
|
||||
{CRC32H, CRC32PBL} ->
|
||||
{Header, Blocks};
|
||||
_ ->
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue