parent
b10c0cf895
commit
958d3f5e14
1 changed files with 8 additions and 29 deletions
|
@ -81,7 +81,6 @@
|
||||||
-define(CACHE_SIZE, 32).
|
-define(CACHE_SIZE, 32).
|
||||||
-define(BLOCK_LENGTHS_LENGTH, 20).
|
-define(BLOCK_LENGTHS_LENGTH, 20).
|
||||||
-define(FLIPPER32, 4294967295).
|
-define(FLIPPER32, 4294967295).
|
||||||
-define(COMPRESS_AT_LEVEL, 2).
|
|
||||||
|
|
||||||
-include_lib("eunit/include/eunit.hrl").
|
-include_lib("eunit/include/eunit.hrl").
|
||||||
|
|
||||||
|
@ -127,7 +126,7 @@
|
||||||
size :: integer(),
|
size :: integer(),
|
||||||
max_sqn :: integer()}).
|
max_sqn :: integer()}).
|
||||||
|
|
||||||
-type press_methods() :: lz4|native|none.
|
-type press_methods() :: lz4|native.
|
||||||
|
|
||||||
%% yield_blockquery is used to detemrine if the work necessary to process a
|
%% yield_blockquery is used to detemrine if the work necessary to process a
|
||||||
%% range query beyond the fetching the slot should be managed from within
|
%% range query beyond the fetching the slot should be managed from within
|
||||||
|
@ -205,8 +204,7 @@ sst_open(RootPath, Filename) ->
|
||||||
%% lists as merge_lists will not be called.
|
%% lists as merge_lists will not be called.
|
||||||
sst_new(RootPath, Filename, Level, KVList, MaxSQN, PressMethod) ->
|
sst_new(RootPath, Filename, Level, KVList, MaxSQN, PressMethod) ->
|
||||||
{ok, Pid} = gen_fsm:start(?MODULE, [], []),
|
{ok, Pid} = gen_fsm:start(?MODULE, [], []),
|
||||||
PressMethod0 = compress_level(Level, PressMethod),
|
{[], [], SlotList, FK} = merge_lists(KVList, PressMethod),
|
||||||
{[], [], SlotList, FK} = merge_lists(KVList, PressMethod0),
|
|
||||||
case gen_fsm:sync_send_event(Pid,
|
case gen_fsm:sync_send_event(Pid,
|
||||||
{sst_new,
|
{sst_new,
|
||||||
RootPath,
|
RootPath,
|
||||||
|
@ -214,7 +212,7 @@ sst_new(RootPath, Filename, Level, KVList, MaxSQN, PressMethod) ->
|
||||||
Level,
|
Level,
|
||||||
{SlotList, FK},
|
{SlotList, FK},
|
||||||
MaxSQN,
|
MaxSQN,
|
||||||
PressMethod0},
|
PressMethod},
|
||||||
infinity) of
|
infinity) of
|
||||||
{ok, {SK, EK}, Bloom} ->
|
{ok, {SK, EK}, Bloom} ->
|
||||||
{ok, Pid, {SK, EK}, Bloom}
|
{ok, Pid, {SK, EK}, Bloom}
|
||||||
|
@ -237,9 +235,8 @@ sst_new(RootPath, Filename, Level, KVList, MaxSQN, PressMethod) ->
|
||||||
sst_new(RootPath, Filename,
|
sst_new(RootPath, Filename,
|
||||||
KVL1, KVL2, IsBasement, Level,
|
KVL1, KVL2, IsBasement, Level,
|
||||||
MaxSQN, PressMethod) ->
|
MaxSQN, PressMethod) ->
|
||||||
PressMethod0 = compress_level(Level, PressMethod),
|
|
||||||
{Rem1, Rem2, SlotList, FK} =
|
{Rem1, Rem2, SlotList, FK} =
|
||||||
merge_lists(KVL1, KVL2, {IsBasement, Level}, PressMethod0),
|
merge_lists(KVL1, KVL2, {IsBasement, Level}, PressMethod),
|
||||||
case SlotList of
|
case SlotList of
|
||||||
[] ->
|
[] ->
|
||||||
empty;
|
empty;
|
||||||
|
@ -252,7 +249,7 @@ sst_new(RootPath, Filename,
|
||||||
Level,
|
Level,
|
||||||
{SlotList, FK},
|
{SlotList, FK},
|
||||||
MaxSQN,
|
MaxSQN,
|
||||||
PressMethod0},
|
PressMethod},
|
||||||
infinity) of
|
infinity) of
|
||||||
{ok, {SK, EK}, Bloom} ->
|
{ok, {SK, EK}, Bloom} ->
|
||||||
{ok, Pid, {{Rem1, Rem2}, SK, EK}, Bloom}
|
{ok, Pid, {{Rem1, Rem2}, SK, EK}, Bloom}
|
||||||
|
@ -270,7 +267,6 @@ sst_new(RootPath, Filename,
|
||||||
sst_newlevelzero(RootPath, Filename,
|
sst_newlevelzero(RootPath, Filename,
|
||||||
Slots, FetchFun, Penciller,
|
Slots, FetchFun, Penciller,
|
||||||
MaxSQN, PressMethod) ->
|
MaxSQN, PressMethod) ->
|
||||||
PressMethod0 = compress_level(0, PressMethod),
|
|
||||||
{ok, Pid} = gen_fsm:start(?MODULE, [], []),
|
{ok, Pid} = gen_fsm:start(?MODULE, [], []),
|
||||||
gen_fsm:send_event(Pid,
|
gen_fsm:send_event(Pid,
|
||||||
{sst_newlevelzero,
|
{sst_newlevelzero,
|
||||||
|
@ -280,7 +276,7 @@ sst_newlevelzero(RootPath, Filename,
|
||||||
FetchFun,
|
FetchFun,
|
||||||
Penciller,
|
Penciller,
|
||||||
MaxSQN,
|
MaxSQN,
|
||||||
PressMethod0}),
|
PressMethod}),
|
||||||
{ok, Pid, noreply}.
|
{ok, Pid, noreply}.
|
||||||
|
|
||||||
-spec sst_get(pid(), tuple()) -> tuple()|not_present.
|
-spec sst_get(pid(), tuple()) -> tuple()|not_present.
|
||||||
|
@ -791,13 +787,6 @@ fetch_range(StartKey, EndKey, ScanWidth, SegList, State) ->
|
||||||
State#state.compression_method),
|
State#state.compression_method),
|
||||||
{SlotsToFetchBinList, SlotsToPoint}.
|
{SlotsToFetchBinList, SlotsToPoint}.
|
||||||
|
|
||||||
-spec compress_level(integer(), press_methods()) -> press_methods().
|
|
||||||
%% @doc
|
|
||||||
%% disable compression at higher levels for improved performance
|
|
||||||
compress_level(Level, _PressMethod) when Level < ?COMPRESS_AT_LEVEL ->
|
|
||||||
none;
|
|
||||||
compress_level(_Level, PressMethod) ->
|
|
||||||
PressMethod.
|
|
||||||
|
|
||||||
write_file(RootPath, Filename, SummaryBin, SlotsBin, PressMethod) ->
|
write_file(RootPath, Filename, SummaryBin, SlotsBin, PressMethod) ->
|
||||||
SummaryLength = byte_size(SummaryBin),
|
SummaryLength = byte_size(SummaryBin),
|
||||||
|
@ -843,14 +832,10 @@ read_file(Filename, State) ->
|
||||||
Bloom}.
|
Bloom}.
|
||||||
|
|
||||||
gen_fileversion(PressMethod) ->
|
gen_fileversion(PressMethod) ->
|
||||||
% Native or none can be treated the same once written, as reader
|
|
||||||
% does not need to know as compression info will be in header of the
|
|
||||||
% block
|
|
||||||
Bit1 =
|
Bit1 =
|
||||||
case PressMethod of
|
case PressMethod of
|
||||||
lz4 -> 1;
|
lz4 -> 1;
|
||||||
native -> 0;
|
native -> 0
|
||||||
none -> 0
|
|
||||||
end,
|
end,
|
||||||
Bit1.
|
Bit1.
|
||||||
|
|
||||||
|
@ -960,10 +945,6 @@ serialise_block(Term, lz4) ->
|
||||||
serialise_block(Term, native) ->
|
serialise_block(Term, native) ->
|
||||||
Bin = term_to_binary(Term, ?BINARY_SETTINGS),
|
Bin = term_to_binary(Term, ?BINARY_SETTINGS),
|
||||||
CRC32 = hmac(Bin),
|
CRC32 = hmac(Bin),
|
||||||
<<Bin/binary, CRC32:32/integer>>;
|
|
||||||
serialise_block(Term, none) ->
|
|
||||||
Bin = term_to_binary(Term),
|
|
||||||
CRC32 = hmac(Bin),
|
|
||||||
<<Bin/binary, CRC32:32/integer>>.
|
<<Bin/binary, CRC32:32/integer>>.
|
||||||
|
|
||||||
|
|
||||||
|
@ -987,12 +968,10 @@ deserialise_block(Bin, PressMethod) ->
|
||||||
deserialise_checkedblock(Bin, lz4) ->
|
deserialise_checkedblock(Bin, lz4) ->
|
||||||
{ok, Bin0} = lz4:unpack(Bin),
|
{ok, Bin0} = lz4:unpack(Bin),
|
||||||
binary_to_term(Bin0);
|
binary_to_term(Bin0);
|
||||||
deserialise_checkedblock(Bin, _Other) ->
|
deserialise_checkedblock(Bin, native) ->
|
||||||
% native or none can be treated the same
|
|
||||||
binary_to_term(Bin).
|
binary_to_term(Bin).
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
-spec hmac(binary()|integer()) -> integer().
|
-spec hmac(binary()|integer()) -> integer().
|
||||||
%% @doc
|
%% @doc
|
||||||
%% Perform a CRC check on an input
|
%% Perform a CRC check on an input
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue