diff --git a/src/leveled_log.erl b/src/leveled_log.erl index 0e475fc..e815bce 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -125,10 +125,12 @@ {"P0032", {info, "Fetch head timing with sample_count=~w and level timings of" ++ " foundmem_time=~w found0_time=~w found1_time=~w" - ++ " found2_time=~w foundlower_time=~w missed_time=~w" + ++ " found2_time=~w found3_time=~w foundlower_time=~w" + ++ " missed_time=~w" ++ " with counts of" ++ " foundmem_count=~w found0_count=~w found1_count=~w" - ++ " found2_count=~w foundlower_count=~w missed_count=~w"}}, + ++ " found2_count=~w found3_count=~w foundlower_count=~w" + ++ " missed_count=~w"}}, {"P0033", {error, "Corrupted manifest file at path ~s to be ignored " ++ "due to error ~w"}}, diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 0f214d1..d184ab6 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -200,9 +200,22 @@ -include_lib("eunit/include/eunit.hrl"). --define(LEVEL_SCALEFACTOR, [{0, 0}, {1, 8}, {2, 64}, {3, 512}, - {4, 4096}, {5, 32768}, {6, 262144}, - {7, infinity}]). +-define(LEVEL_SCALEFACTOR, + [{0, 0}, + {1, 4}, {2, 16}, {3, 64}, % Factor of 4 + {4, 384}, {5, 2304}, % Factor of 6 + {6, 18432}, % Factor of 8 + {7, infinity}]). + % As an alternative to going up by a factor of 8 at each level, + % increase by a factor of 4 at young levels - to make early + % compaction jobs shorter. + % + % There are 32K keys per files => with 4096 files there are 100M + % keys supported, + + % 600M keys is supported before hitting the infinite level. + % At o(10) trillion keys behaviour may become increasingly + % difficult to predict. -define(MAX_LEVELS, 8). -define(MAX_WORK_WAIT, 300). -define(MANIFEST_FP, "ledger_manifest"). @@ -259,12 +272,14 @@ found0_time = 0 :: integer(), found1_time = 0 :: integer(), found2_time = 0 :: integer(), + found3_time = 0 :: integer(), foundlower_time = 0 :: integer(), missed_time = 0 :: integer(), foundmem_count = 0 :: integer(), found0_count = 0 :: integer(), found1_count = 0 :: integer(), found2_count = 0 :: integer(), + found3_count = 0 :: integer(), foundlower_count = 0 :: integer(), missed_count = 0 :: integer()}). @@ -1500,12 +1515,14 @@ log_timings(Timings) -> Timings#pcl_timings.found0_time, Timings#pcl_timings.found1_time, Timings#pcl_timings.found2_time, + Timings#pcl_timings.found3_time, Timings#pcl_timings.foundlower_time, Timings#pcl_timings.missed_time, Timings#pcl_timings.foundmem_count, Timings#pcl_timings.found0_count, Timings#pcl_timings.found1_count, Timings#pcl_timings.found2_count, + Timings#pcl_timings.found3_count, Timings#pcl_timings.foundlower_count, Timings#pcl_timings.missed_count]). @@ -1543,6 +1560,10 @@ update_timings(SW, Timings, Result, Stage) -> L2T = Timings#pcl_timings.found2_time + Timer, L2C = Timings#pcl_timings.found2_count + 1, Timings0#pcl_timings{found2_time = L2T, found2_count = L2C}; + {_, 3} -> + L3T = Timings#pcl_timings.found3_time + Timer, + L3C = Timings#pcl_timings.found3_count + 1, + Timings0#pcl_timings{found3_time = L3T, found3_count = L3C}; _ -> LLT = Timings#pcl_timings.foundlower_time + Timer, LLC = Timings#pcl_timings.foundlower_count + 1, @@ -1956,7 +1977,7 @@ foldwithimm_simple_test() -> create_file_test() -> {RP, Filename} = {"../test/", "new_file.sst"}, ok = file:write_file(filename:join(RP, Filename), term_to_binary("hello")), - KVL = lists:usort(generate_randomkeys({10000, 0})), + KVL = lists:usort(generate_randomkeys({50000, 0})), Tree = leveled_tree:from_orderedlist(KVL, ?CACHE_TYPE), FetchFun = fun(Slot) -> lists:nth(Slot, [Tree]) end, {ok, @@ -1966,7 +1987,7 @@ create_file_test() -> 1, FetchFun, undefined, - 10000, + 50000, native), lists:foreach(fun(X) -> case checkready(SP) of @@ -2007,7 +2028,8 @@ timings_test() -> ?assertMatch(3, T2#pcl_timings.sample_count), ?assertMatch(true, T2#pcl_timings.foundlower_time > T2#pcl_timings.found2_time), ?assertMatch(1, T2#pcl_timings.found2_count), - ?assertMatch(2, T2#pcl_timings.foundlower_count). + ?assertMatch(1, T2#pcl_timings.found3_count), + ?assertMatch(1, T2#pcl_timings.foundlower_count). coverage_cheat_test() -> diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index b13d07f..d0ac95a 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -1517,10 +1517,13 @@ binaryslot_trimmedlist(FullBin, StartKey, EndKey, PressMethod) -> crc_check_slot(FullBin) -> <> = FullBin, - case {hmac(Header), hmac(PosBL)} of + CRC32H:32/integer, + Rest/binary>> = FullBin, + PosBL0 = min(PosBL, byte_size(FullBin) - 3), + % If the position has been bit-flipped to beyond the maximum paossible + % length, use the maximum possible length + <> = Rest, + case {hmac(Header), hmac(PosBL0)} of {CRC32H, CRC32PBL} -> {Header, Blocks}; _ ->