umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Switch segment and extra hash

Browse source 
More entropy by using the position index with the segment hash - so this would be a better filter to apply.

Also could increase the key count now, as extra hash can be larger.

As an aside - a leveled_iclerk unit test failure appeared - the range was just wrong.  Don't know why this strated happening
This commit is contained in:
Martin Sumner 2017-10-24 14:32:04 +01:00
parent 36264eb416
commit 26aa573ce1
3 changed files with 12 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/leveled_iclerk.erl
View file

@ -648,8 +648,8 @@ schedule_test_bycount(N) ->
?assertMatch(true, SecondsToCompaction0 < 5700), ?assertMatch(true, SecondsToCompaction0 < 5700),
SecondsToCompaction1 = schedule_compaction([14], N, CurrentTS), % tomorrow! SecondsToCompaction1 = schedule_compaction([14], N, CurrentTS), % tomorrow!
io:format("Seconds to compaction ~w~n", [SecondsToCompaction1]), io:format("Seconds to compaction ~w~n", [SecondsToCompaction1]),
?assertMatch(true, SecondsToCompaction1 > 81000), ?assertMatch(true, SecondsToCompaction1 >= 81180),
?assertMatch(true, SecondsToCompaction1 < 84300). ?assertMatch(true, SecondsToCompaction1 =< 84780).
simple_score_test() -> simple_score_test() ->

4
src/leveled_sst.erl
View file

@ -1186,8 +1186,8 @@ block_offsetandlength(BlockLengths, BlockID) ->
{BlocksPos, B1L + B2L + B3L + B4L, B5L} {BlocksPos, B1L + B2L + B3L + B4L, B5L}
end. end.
extra_hash({_SegHash, ExtraHash}) when is_integer(ExtraHash) -> extra_hash({SegHash, _ExtraHash}) when is_integer(SegHash) ->
ExtraHash band 32767; SegHash band 32767;
extra_hash(NotHash) -> extra_hash(NotHash) ->
NotHash. NotHash.

16
src/leveled_tinybloom.erl
View file

@ -48,7 +48,7 @@ create_bloom(HashList) ->
%% Check for the presence of a given hash within a bloom %% Check for the presence of a given hash within a bloom
check_hash(_Hash, <<>>) -> check_hash(_Hash, <<>>) ->
false; false;
check_hash({Hash, _ExtraHash}, BloomBin) -> check_hash({_SegHash, Hash}, BloomBin) ->
SlotSplit = (byte_size(BloomBin) div ?BITS_PER_KEY) - 1, SlotSplit = (byte_size(BloomBin) div ?BITS_PER_KEY) - 1,
{Slot, H0, H1} = split_hash(Hash, SlotSplit), {Slot, H0, H1} = split_hash(Hash, SlotSplit),
Mask = get_mask(H0, H1), Mask = get_mask(H0, H1),
@ -66,10 +66,10 @@ check_hash({Hash, _ExtraHash}, BloomBin) ->
%%% Internal Functions %%% Internal Functions
%%%============================================================================ %%%============================================================================
split_hash(SegHash, SlotSplit) -> split_hash(Hash, SlotSplit) ->
Slot = SegHash band SlotSplit, Slot = Hash band SlotSplit,
H0 = (SegHash bsr 4) band (?BAND_MASK), H0 = (Hash bsr 4) band (?BAND_MASK),
H1 = (SegHash bsr 10) band (?BAND_MASK), H1 = (Hash bsr 10) band (?BAND_MASK),
{Slot, H0, H1}. {Slot, H0, H1}.
get_mask(H0, H1) -> get_mask(H0, H1) ->
@ -87,7 +87,7 @@ get_mask(H0, H1) ->
add_hashlist([], _S, S0, S1) -> add_hashlist([], _S, S0, S1) ->
IntSize = ?INTEGER_SIZE, IntSize = ?INTEGER_SIZE,
<<S0:IntSize/integer, S1:IntSize/integer>>; <<S0:IntSize/integer, S1:IntSize/integer>>;
add_hashlist([{TopHash, _ExtraHash}|T], SlotSplit, S0, S1) -> add_hashlist([{_SegHash, TopHash}|T], SlotSplit, S0, S1) ->
{Slot, H0, H1} = split_hash(TopHash, SlotSplit), {Slot, H0, H1} = split_hash(TopHash, SlotSplit),
Mask = get_mask(H0, H1), Mask = get_mask(H0, H1),
case Slot of case Slot of
@ -101,7 +101,7 @@ add_hashlist([], _S, S0, S1, S2, S3) ->
IntSize = ?INTEGER_SIZE, IntSize = ?INTEGER_SIZE,
<<S0:IntSize/integer, S1:IntSize/integer, <<S0:IntSize/integer, S1:IntSize/integer,
S2:IntSize/integer, S3:IntSize/integer>>; S2:IntSize/integer, S3:IntSize/integer>>;
add_hashlist([{TopHash, _ExtraHash}|T], SlotSplit, S0, S1, S2, S3) -> add_hashlist([{_SegHash, TopHash}|T], SlotSplit, S0, S1, S2, S3) ->
{Slot, H0, H1} = split_hash(TopHash, SlotSplit), {Slot, H0, H1} = split_hash(TopHash, SlotSplit),
Mask = get_mask(H0, H1), Mask = get_mask(H0, H1),
case Slot of case Slot of
@ -126,7 +126,7 @@ add_hashlist([], _S, S0, S1, S2, S3, S4, S5, S6, S7, S8, S9,
SA:IntSize/integer, SB:IntSize/integer, SA:IntSize/integer, SB:IntSize/integer,
SC:IntSize/integer, SD:IntSize/integer, SC:IntSize/integer, SD:IntSize/integer,
SE:IntSize/integer, SF:IntSize/integer>>; SE:IntSize/integer, SF:IntSize/integer>>;
add_hashlist([{TopHash, _ExtraHash}|T], add_hashlist([{_SegHash, TopHash}|T],
SlotSplit, SlotSplit,
S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, S0, S1, S2, S3, S4, S5, S6, S7, S8, S9,
SA, SB, SC, SD, SE, SF) -> SA, SB, SC, SD, SE, SF) ->