Merge pull request #118 from martinsumner/mas-i117-factor4scale

Mas i117 factor4scale

src/leveled_log.erl

@@ -125,10 +125,12 @@
     {"P0032",
         {info, "Fetch head timing with sample_count=~w and level timings of"
                     ++ " foundmem_time=~w found0_time=~w found1_time=~w" 
-                    ++ " found2_time=~w foundlower_time=~w missed_time=~w"
+                    ++ " found2_time=~w found3_time=~w foundlower_time=~w" 
+                    ++ " missed_time=~w"
                     ++ " with counts of"
                     ++ " foundmem_count=~w found0_count=~w found1_count=~w" 
-                    ++ " found2_count=~w foundlower_count=~w missed_count=~w"}},
+                    ++ " found2_count=~w found3_count=~w foundlower_count=~w"
+                    ++ " missed_count=~w"}},
     {"P0033",
         {error, "Corrupted manifest file at path ~s to be ignored "
                     ++ "due to error ~w"}},

src/leveled_penciller.erl

@@ -200,9 +200,22 @@
 
 -include_lib("eunit/include/eunit.hrl").
 
--define(LEVEL_SCALEFACTOR, [{0, 0}, {1, 8}, {2, 64}, {3, 512},
+-define(LEVEL_SCALEFACTOR, 
-                            {4, 4096}, {5, 32768}, {6, 262144},
+            [{0, 0}, 
-                            {7, infinity}]).
+                {1, 4}, {2, 16}, {3, 64}, % Factor of 4
+                {4, 384}, {5, 2304}, % Factor of 6 
+                {6, 18432}, % Factor of 8 
+                {7, infinity}]).
+            % As an alternative to going up by a factor of 8 at each level, 
+            % increase by a factor of 4 at young levels - to make early  
+            % compaction jobs shorter.
+            %  
+            % There are 32K keys per files => with 4096 files there are 100M
+            % keys supported,
+            
+            % 600M keys is supported before hitting the infinite level.  
+            % At o(10) trillion keys behaviour may become increasingly 
+            % difficult to predict.
 -define(MAX_LEVELS, 8).
 -define(MAX_WORK_WAIT, 300).
 -define(MANIFEST_FP, "ledger_manifest").
@@ -259,12 +272,14 @@
                         found0_time = 0 :: integer(),
                         found1_time = 0 :: integer(),
                         found2_time = 0 :: integer(),
+                        found3_time = 0 :: integer(),
                         foundlower_time = 0 :: integer(),
                         missed_time = 0 :: integer(),
                         foundmem_count = 0 :: integer(),
                         found0_count = 0 :: integer(),
                         found1_count = 0 :: integer(),
                         found2_count = 0 :: integer(),
+                        found3_count = 0 :: integer(),
                         foundlower_count = 0 :: integer(),
                         missed_count = 0 :: integer()}).
 
@@ -1500,12 +1515,14 @@ log_timings(Timings) ->
                                 Timings#pcl_timings.found0_time,
                                 Timings#pcl_timings.found1_time,
                                 Timings#pcl_timings.found2_time,
+                                Timings#pcl_timings.found3_time,
                                 Timings#pcl_timings.foundlower_time,
                                 Timings#pcl_timings.missed_time,
                                 Timings#pcl_timings.foundmem_count,
                                 Timings#pcl_timings.found0_count,
                                 Timings#pcl_timings.found1_count,
                                 Timings#pcl_timings.found2_count,
+                                Timings#pcl_timings.found3_count,
                                 Timings#pcl_timings.foundlower_count,
                                 Timings#pcl_timings.missed_count]).
 
@@ -1543,6 +1560,10 @@ update_timings(SW, Timings, Result, Stage) ->
             L2T = Timings#pcl_timings.found2_time + Timer,
             L2C = Timings#pcl_timings.found2_count + 1,
             Timings0#pcl_timings{found2_time = L2T, found2_count = L2C};
+        {_, 3} ->
+            L3T = Timings#pcl_timings.found3_time + Timer,
+            L3C = Timings#pcl_timings.found3_count + 1,
+            Timings0#pcl_timings{found3_time = L3T, found3_count = L3C};
         _ ->
             LLT = Timings#pcl_timings.foundlower_time + Timer,
             LLC = Timings#pcl_timings.foundlower_count + 1,
@@ -1956,7 +1977,7 @@ foldwithimm_simple_test() ->
 create_file_test() ->
     {RP, Filename} = {"../test/", "new_file.sst"},
     ok = file:write_file(filename:join(RP, Filename), term_to_binary("hello")),
-    KVL = lists:usort(generate_randomkeys({10000, 0})),
+    KVL = lists:usort(generate_randomkeys({50000, 0})),
     Tree = leveled_tree:from_orderedlist(KVL, ?CACHE_TYPE),
     FetchFun = fun(Slot) -> lists:nth(Slot, [Tree]) end,
     {ok,
@@ -1966,7 +1987,7 @@ create_file_test() ->
                                                 1,
                                                 FetchFun,
                                                 undefined,
-                                                10000,
+                                                50000,
                                                 native),
     lists:foreach(fun(X) ->
                         case checkready(SP) of
@@ -2007,7 +2028,8 @@ timings_test() ->
     ?assertMatch(3, T2#pcl_timings.sample_count),
     ?assertMatch(true, T2#pcl_timings.foundlower_time > T2#pcl_timings.found2_time),
     ?assertMatch(1, T2#pcl_timings.found2_count),
-    ?assertMatch(2, T2#pcl_timings.foundlower_count).    
+    ?assertMatch(1, T2#pcl_timings.found3_count),
+    ?assertMatch(1, T2#pcl_timings.foundlower_count).    
 
 
 coverage_cheat_test() ->

src/leveled_sst.erl

@@ -1517,10 +1517,13 @@ binaryslot_trimmedlist(FullBin, StartKey, EndKey, PressMethod) ->
 crc_check_slot(FullBin) ->
     <<CRC32PBL:32/integer, 
         PosBL:32/integer, 
-        CRC32H:32/integer, 
+        CRC32H:32/integer,
-        Header:PosBL/binary, 
+        Rest/binary>> = FullBin,
-        Blocks/binary>> = FullBin,
+    PosBL0 = min(PosBL, byte_size(FullBin) - 3), 
-    case {hmac(Header), hmac(PosBL)} of 
+        % If the position has been bit-flipped to beyond the maximum paossible
+        % length, use the maximum possible length
+    <<Header:PosBL0/binary, Blocks/binary>> = Rest,
+    case {hmac(Header), hmac(PosBL0)} of 
         {CRC32H, CRC32PBL} ->
             {Header, Blocks};
         _ ->