Make tinybloom size configurable

Allow the bloom size to vary depending on how many fetchable keys there
are - so ther eis no large bloom held if most of the keys are index
entries for example
This commit is contained in:
martinsumner 2017-01-02 18:29:15 +00:00
parent 972aa85012
commit baa644383d
2 changed files with 6 additions and 5 deletions

View file

@ -72,7 +72,7 @@
-define(SLOT_SIZE, 128). % This is not configurable
-define(COMPRESSION_LEVEL, 1).
-define(BINARY_SETTINGS, [{compressed, ?COMPRESSION_LEVEL}]).
-define(LEVEL_BLOOM_SLOTS, [{0, 64}, {1, 48}, {default, 32}]).
-define(LEVEL_BLOOM_BITS, [{0, 12}, {1, 10}, {2, 8}, {default, 6}]).
-define(MERGE_SCANWIDTH, 16).
-define(DISCARD_EXT, ".discarded").
-define(DELETE_TIMEOUT, 10000).
@ -614,13 +614,14 @@ open_reader(Filename) ->
{Handle, SummaryBin}.
build_table_summary(SlotIndex, AllHashes, Level, FirstKey, L, MaxSQN) ->
BloomSlots =
case lists:keyfind(Level, 1, ?LEVEL_BLOOM_SLOTS) of
BloomBits =
case lists:keyfind(Level, 1, ?LEVEL_BLOOM_BITS) of
{Level, N} ->
N;
false ->
element(2, lists:keyfind(default, 1, ?LEVEL_BLOOM_SLOTS))
element(2, lists:keyfind(default, 1, ?LEVEL_BLOOM_BITS))
end,
BloomSlots = (length(AllHashes) * BloomBits) div 4096,
BloomAddFun =
fun({H, _K}, Bloom) -> leveled_tinybloom:enter(H, Bloom) end,
Bloom = lists:foldr(BloomAddFun,

View file

@ -133,7 +133,7 @@ getbit(Bit, BitArray, ArrayLength) ->
simple_test() ->
N = 4000,
W = 4,
W = 6,
KLin = lists:map(fun(X) -> "Key_" ++
integer_to_list(X) ++
integer_to_list(random:uniform(100)) ++