umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Remove SST blooms

Browse source
This commit is contained in:
martinsumner 2017-01-03 09:12:41 +00:00
parent 5b4c903d53
commit d28e5d639c
1 changed files with 41 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

106
src/leveled_sst.erl
View file

@ -72,7 +72,7 @@
-define(SLOT_SIZE, 128). % This is not configurable -define(SLOT_SIZE, 128). % This is not configurable
-define(COMPRESSION_LEVEL, 1). -define(COMPRESSION_LEVEL, 1).
-define(BINARY_SETTINGS, [{compressed, ?COMPRESSION_LEVEL}]). -define(BINARY_SETTINGS, [{compressed, ?COMPRESSION_LEVEL}]).
-define(LEVEL_BLOOM_BITS, [{0, 8}, {1, 10}, {2, 8}, {default, 6}]). % -define(LEVEL_BLOOM_BITS, [{0, 8}, {1, 10}, {2, 8}, {default, 6}]).
-define(MERGE_SCANWIDTH, 16). -define(MERGE_SCANWIDTH, 16).
-define(DISCARD_EXT, ".discarded"). -define(DISCARD_EXT, ".discarded").
-define(DELETE_TIMEOUT, 10000). -define(DELETE_TIMEOUT, 10000).
@ -117,8 +117,7 @@
last_key :: tuple(), last_key :: tuple(),
index :: list(), % leveled_skiplist index :: list(), % leveled_skiplist
size :: integer(), size :: integer(),
max_sqn :: integer(), max_sqn :: integer()}).
bloom}).
-record(state, {summary, -record(state, {summary,
handle :: file:fd(), handle :: file:fd(),
@ -417,58 +416,45 @@ code_change(_OldVsn, StateName, State, _Extra) ->
fetch(LedgerKey, Hash, State) -> fetch(LedgerKey, Hash, State) ->
Summary = State#state.summary, Summary = State#state.summary,
case leveled_tinybloom:check({hash, Hash}, Slot = lookup_slot(LedgerKey, Summary#summary.index),
Summary#summary.bloom) of SlotID = Slot#slot_index_value.slot_id,
false -> CachedBlockIdx = array:get(SlotID - 1,
{not_present, summary_bloom, null, State}; State#state.blockindex_cache),
true -> case CachedBlockIdx of
Slot = lookup_slot(LedgerKey, Summary#summary.index), none ->
SlotID = Slot#slot_index_value.slot_id, SlotBin = read_slot(State#state.handle, Slot),
CachedBlockIdx = array:get(SlotID - 1, {Result, BlockIdx} = binaryslot_get(SlotBin,
LedgerKey,
Hash,
none),
BlockIndexCache = array:set(SlotID - 1,
BlockIdx,
State#state.blockindex_cache), State#state.blockindex_cache),
case CachedBlockIdx of {Result,
none -> slot_fetch,
SlotBin = read_slot(State#state.handle, Slot), Slot#slot_index_value.slot_id,
{Result, BlockIdx} = binaryslot_get(SlotBin, State#state{blockindex_cache = BlockIndexCache}};
LedgerKey, _ ->
Hash, PosList = find_pos(CachedBlockIdx,
none), double_hash(Hash, LedgerKey),
BlockIndexCache = array:set(SlotID - 1, [],
BlockIdx, 0),
State#state.blockindex_cache), case PosList of
{Result, [] ->
slot_fetch, {not_present, slot_bloom, SlotID, State};
Slot#slot_index_value.slot_id,
State#state{blockindex_cache = BlockIndexCache}};
_ -> _ ->
PosList = find_pos(CachedBlockIdx, LastKV = array:get(SlotID - 1,
double_hash(Hash, LedgerKey), State#state.lastfetch_cache),
[], case LastKV of
0), {LedgerKey, _} ->
case PosList of {LastKV, slot_cache, SlotID, State};
[] ->
{not_present,
slot_bloom,
SlotID,
State};
_ -> _ ->
LastKV = array:get(SlotID - 1, SlotBin = read_slot(State#state.handle, Slot),
State#state.lastfetch_cache), Result = binaryslot_get(SlotBin,
case LastKV of LedgerKey,
{LedgerKey, _} -> Hash,
{LastKV, slot_cache, SlotID, State}; {true, PosList}),
_ -> {element(1, Result), slot_fetch, SlotID, State}
SlotBin = read_slot(State#state.handle,
Slot),
Result = binaryslot_get(SlotBin,
LedgerKey,
Hash,
{true, PosList}),
{element(1, Result),
slot_fetch,
SlotID,
State}
end
end end
end end
end. end.
@ -613,26 +599,12 @@ open_reader(Filename) ->
{ok, SummaryBin} = file:pread(Handle, SlotsLength + 8, SummaryLength), {ok, SummaryBin} = file:pread(Handle, SlotsLength + 8, SummaryLength),
{Handle, SummaryBin}. {Handle, SummaryBin}.
build_table_summary(SlotIndex, AllHashes, Level, FirstKey, L, MaxSQN) -> build_table_summary(SlotIndex, _AllHashes, _Level, FirstKey, L, MaxSQN) ->
BloomBits =
case lists:keyfind(Level, 1, ?LEVEL_BLOOM_BITS) of
{Level, N} ->
N;
false ->
element(2, lists:keyfind(default, 1, ?LEVEL_BLOOM_BITS))
end,
BloomSlots = max((length(AllHashes) * BloomBits) div 4096, 1),
BloomAddFun =
fun({H, _K}, Bloom) -> leveled_tinybloom:enter(H, Bloom) end,
Bloom = lists:foldr(BloomAddFun,
leveled_tinybloom:empty(BloomSlots),
AllHashes),
[{LastKey, _LastV}|_Rest] = SlotIndex, [{LastKey, _LastV}|_Rest] = SlotIndex,
Summary = #summary{first_key = FirstKey, Summary = #summary{first_key = FirstKey,
last_key = LastKey, last_key = LastKey,
size = L, size = L,
index = lists:reverse(SlotIndex), index = lists:reverse(SlotIndex),
bloom = Bloom,
max_sqn = MaxSQN}, max_sqn = MaxSQN},
SummBin = term_to_binary(Summary, ?BINARY_SETTINGS), SummBin = term_to_binary(Summary, ?BINARY_SETTINGS),
SummCRC = erlang:crc32(SummBin), SummCRC = erlang:crc32(SummBin),