umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Curtail trimmed slot crazyness

Browse source 
There was complicated and confusing code that achieved nothing for
effiency when trimming slots.  the expensive part (binary_to_term) was
still needed on every block, and it was hard to get code coverage and
make sense of what it was really trying to achieve.

This is now much simpler - and may set us up for potential further
indexing help.
This commit is contained in:
martinsumner 2017-03-19 21:47:22 +00:00
parent c203e2ee06
commit f20aba9c8b
1 changed files with 22 additions and 67 deletions
Download patch file Download diff file Expand all files Collapse all files

89
src/leveled_sst.erl
View file

@ -909,41 +909,34 @@ binaryslot_trimmedlist(FullBin, StartKey, EndKey) ->
LTrimFun = fun({K, _V}) -> K < StartKey end, LTrimFun = fun({K, _V}) -> K < StartKey end,
RTrimFun = fun({K, _V}) -> not leveled_codec:endkey_passed(EndKey, K) end, RTrimFun = fun({K, _V}) -> not leveled_codec:endkey_passed(EndKey, K) end,
BlockFetchFun = BlockFetchFun =
fun(Length, {Acc, Bin}) -> fun(Length, {Acc, Bin, Continue}) ->
case Length of case {Length, Continue} of
0 -> {0, _} ->
{Acc, Bin}; {Acc, Bin, false};
_ -> {_, true} ->
<<Block:Length/binary, Rest/binary>> = Bin, <<Block:Length/binary, Rest/binary>> = Bin,
BlockList = binary_to_term(Block), BlockList = binary_to_term(Block),
{FirstKey, _FV} = lists:nth(1, BlockList),
{LastKey, _LV} = lists:last(BlockList), {LastKey, _LV} = lists:last(BlockList),
TrimBools = trim_booleans(FirstKey, LastKey, case StartKey > LastKey of
StartKey, EndKey), true ->
case TrimBools of {Acc, Rest, true};
{true, _, _, _} -> false ->
{Acc, Rest};
{false, true, _, _} ->
{Acc ++ BlockList, Rest};
{false, false, true, false} ->
{_LDrop, RKeep} = lists:splitwith(LTrimFun, {_LDrop, RKeep} = lists:splitwith(LTrimFun,
BlockList), BlockList),
{Acc ++ RKeep, Rest}; case leveled_codec:endkey_passed(EndKey, LastKey) of
{false, false, false, true} -> true ->
{LKeep, _RDrop} = lists:splitwith(RTrimFun, {LKeep, _RDrop} = lists:splitwith(RTrimFun, RKeep),
BlockList), {Acc ++ LKeep, Rest, false};
{Acc ++ LKeep, Rest}; false ->
{false, false, true, true} -> {Acc ++ RKeep, Rest, true}
{_LDrop, RKeep} = lists:splitwith(LTrimFun, end
BlockList), end;
{LKeep, _RDrop} = lists:splitwith(RTrimFun, RKeep), {_ , false} ->
{Acc ++ LKeep, Rest} {Acc, Bin, false}
end
end end
end, end,
{Out, _Rem} = {Out, _Rem, _Continue} =
case crc_check_slot(FullBin) of case crc_check_slot(FullBin) of
{BlockLengths, RestBin} -> {BlockLengths, RestBin} ->
<<B1P:32/integer, <<B1P:32/integer,
@ -952,51 +945,13 @@ binaryslot_trimmedlist(FullBin, StartKey, EndKey) ->
B3L:32/integer, B3L:32/integer,
B4L:32/integer>> = BlockLengths, B4L:32/integer>> = BlockLengths,
<<_PosBinIndex:B1P/binary, Blocks/binary>> = RestBin, <<_PosBinIndex:B1P/binary, Blocks/binary>> = RestBin,
lists:foldl(BlockFetchFun, {[], Blocks}, [B1L, B2L, B3L, B4L]); lists:foldl(BlockFetchFun, {[], Blocks, true}, [B1L, B2L, B3L, B4L]);
crc_wonky -> crc_wonky ->
{[], <<>>} {[], <<>>, true}
end, end,
Out. Out.
trim_booleans(FirstKey, _LastKey, StartKey, all) ->
FirstKeyPassedStart = FirstKey > StartKey,
case FirstKeyPassedStart of
true ->
{false, true, false, false};
false ->
{false, false, true, false}
end;
trim_booleans(_FirstKey, LastKey, all, EndKey) ->
LastKeyPassedEnd = leveled_codec:endkey_passed(EndKey, LastKey),
case LastKeyPassedEnd of
true ->
{false, false, false, true};
false ->
{false, true, false, false}
end;
trim_booleans(FirstKey, LastKey, StartKey, EndKey) ->
FirstKeyPassedStart = FirstKey > StartKey,
PreRange = LastKey < StartKey,
PostRange = leveled_codec:endkey_passed(EndKey, FirstKey),
OutOfRange = PreRange or PostRange,
LastKeyPassedEnd = leveled_codec:endkey_passed(EndKey, LastKey),
case OutOfRange of
true ->
{true, false, false, false};
false ->
case {FirstKeyPassedStart, LastKeyPassedEnd} of
{true, false} ->
{false, true, false, false};
{false, false} ->
{false, false, true, false};
{true, true} ->
{false, false, false, true};
{false, true} ->
{false, false, true, true}
end
end.
crc_check_slot(FullBin) -> crc_check_slot(FullBin) ->
<<CRC32:32/integer, SlotBin/binary>> = FullBin, <<CRC32:32/integer, SlotBin/binary>> = FullBin,