Revert "Disable compression L0 and L1"

This reverts commit b10c0cf895.
This commit is contained in:
Martin Sumner 2017-12-04 09:29:44 +00:00
parent b10c0cf895
commit 958d3f5e14

View file

@ -81,7 +81,6 @@
-define(CACHE_SIZE, 32).
-define(BLOCK_LENGTHS_LENGTH, 20).
-define(FLIPPER32, 4294967295).
-define(COMPRESS_AT_LEVEL, 2).
-include_lib("eunit/include/eunit.hrl").
@ -127,7 +126,7 @@
size :: integer(),
max_sqn :: integer()}).
-type press_methods() :: lz4|native|none.
-type press_methods() :: lz4|native.
%% yield_blockquery is used to detemrine if the work necessary to process a
%% range query beyond the fetching the slot should be managed from within
@ -205,8 +204,7 @@ sst_open(RootPath, Filename) ->
%% lists as merge_lists will not be called.
sst_new(RootPath, Filename, Level, KVList, MaxSQN, PressMethod) ->
{ok, Pid} = gen_fsm:start(?MODULE, [], []),
PressMethod0 = compress_level(Level, PressMethod),
{[], [], SlotList, FK} = merge_lists(KVList, PressMethod0),
{[], [], SlotList, FK} = merge_lists(KVList, PressMethod),
case gen_fsm:sync_send_event(Pid,
{sst_new,
RootPath,
@ -214,7 +212,7 @@ sst_new(RootPath, Filename, Level, KVList, MaxSQN, PressMethod) ->
Level,
{SlotList, FK},
MaxSQN,
PressMethod0},
PressMethod},
infinity) of
{ok, {SK, EK}, Bloom} ->
{ok, Pid, {SK, EK}, Bloom}
@ -237,9 +235,8 @@ sst_new(RootPath, Filename, Level, KVList, MaxSQN, PressMethod) ->
sst_new(RootPath, Filename,
KVL1, KVL2, IsBasement, Level,
MaxSQN, PressMethod) ->
PressMethod0 = compress_level(Level, PressMethod),
{Rem1, Rem2, SlotList, FK} =
merge_lists(KVL1, KVL2, {IsBasement, Level}, PressMethod0),
merge_lists(KVL1, KVL2, {IsBasement, Level}, PressMethod),
case SlotList of
[] ->
empty;
@ -252,7 +249,7 @@ sst_new(RootPath, Filename,
Level,
{SlotList, FK},
MaxSQN,
PressMethod0},
PressMethod},
infinity) of
{ok, {SK, EK}, Bloom} ->
{ok, Pid, {{Rem1, Rem2}, SK, EK}, Bloom}
@ -270,7 +267,6 @@ sst_new(RootPath, Filename,
sst_newlevelzero(RootPath, Filename,
Slots, FetchFun, Penciller,
MaxSQN, PressMethod) ->
PressMethod0 = compress_level(0, PressMethod),
{ok, Pid} = gen_fsm:start(?MODULE, [], []),
gen_fsm:send_event(Pid,
{sst_newlevelzero,
@ -280,7 +276,7 @@ sst_newlevelzero(RootPath, Filename,
FetchFun,
Penciller,
MaxSQN,
PressMethod0}),
PressMethod}),
{ok, Pid, noreply}.
-spec sst_get(pid(), tuple()) -> tuple()|not_present.
@ -791,13 +787,6 @@ fetch_range(StartKey, EndKey, ScanWidth, SegList, State) ->
State#state.compression_method),
{SlotsToFetchBinList, SlotsToPoint}.
-spec compress_level(integer(), press_methods()) -> press_methods().
%% @doc
%% disable compression at higher levels for improved performance
compress_level(Level, _PressMethod) when Level < ?COMPRESS_AT_LEVEL ->
none;
compress_level(_Level, PressMethod) ->
PressMethod.
write_file(RootPath, Filename, SummaryBin, SlotsBin, PressMethod) ->
SummaryLength = byte_size(SummaryBin),
@ -843,14 +832,10 @@ read_file(Filename, State) ->
Bloom}.
gen_fileversion(PressMethod) ->
% Native or none can be treated the same once written, as reader
% does not need to know as compression info will be in header of the
% block
Bit1 =
case PressMethod of
lz4 -> 1;
native -> 0;
none -> 0
native -> 0
end,
Bit1.
@ -960,10 +945,6 @@ serialise_block(Term, lz4) ->
serialise_block(Term, native) ->
Bin = term_to_binary(Term, ?BINARY_SETTINGS),
CRC32 = hmac(Bin),
<<Bin/binary, CRC32:32/integer>>;
serialise_block(Term, none) ->
Bin = term_to_binary(Term),
CRC32 = hmac(Bin),
<<Bin/binary, CRC32:32/integer>>.
@ -987,12 +968,10 @@ deserialise_block(Bin, PressMethod) ->
deserialise_checkedblock(Bin, lz4) ->
{ok, Bin0} = lz4:unpack(Bin),
binary_to_term(Bin0);
deserialise_checkedblock(Bin, _Other) ->
% native or none can be treated the same
deserialise_checkedblock(Bin, native) ->
binary_to_term(Bin).
-spec hmac(binary()|integer()) -> integer().
%% @doc
%% Perform a CRC check on an input