umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Add cache population for non-yielding range fetches

Browse source 
More likely to require caching at lower levels.
This commit is contained in:
Martin Sumner 2018-05-17 16:38:52 +01:00
parent 779ccd9c2a
commit 989f23bca6
2 changed files with 83 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

139
src/leveled_sst.erl
View file

@ -145,7 +145,8 @@
:: {next, :: {next,
leveled_pmanifest:manifest_entry(), leveled_pmanifest:manifest_entry(),
leveled_codec:ledger_key()|all}. leveled_codec:ledger_key()|all}.
-type binaryslot_element()
:: {tuple(), tuple()}|{binary(), integer(), tuple(), tuple()}.
%% yield_blockquery is used to detemrine if the work necessary to process a %% yield_blockquery is used to detemrine if the work necessary to process a
%% range query beyond the fetching the slot should be managed from within %% range query beyond the fetching the slot should be managed from within
@ -375,8 +376,9 @@ sst_getfilteredrange(Pid, StartKey, EndKey, ScanWidth, SegList) ->
ScanWidth, SegList0}, ScanWidth, SegList0},
infinity) of infinity) of
{yield, SlotsToFetchBinList, SlotsToPoint, PressMethod} -> {yield, SlotsToFetchBinList, SlotsToPoint, PressMethod} ->
binaryslot_reader(SlotsToFetchBinList, PressMethod, SegList0) {L, _BIC} =
++ SlotsToPoint; binaryslot_reader(SlotsToFetchBinList, PressMethod, SegList0),
L ++ SlotsToPoint;
Reply -> Reply ->
Reply Reply
end. end.
@ -405,7 +407,8 @@ sst_getfilteredslots(Pid, SlotList, SegList) ->
SegL0 = tune_seglist(SegList), SegL0 = tune_seglist(SegList),
{SlotBins, PressMethod} = {SlotBins, PressMethod} =
gen_fsm:sync_send_event(Pid, {get_slots, SlotList, SegL0}, infinity), gen_fsm:sync_send_event(Pid, {get_slots, SlotList, SegL0}, infinity),
binaryslot_reader(SlotBins, PressMethod, SegL0). {L, _BIC} = binaryslot_reader(SlotBins, PressMethod, SegL0),
L.
-spec sst_getmaxsequencenumber(pid()) -> integer(). -spec sst_getmaxsequencenumber(pid()) -> integer().
%% @doc %% @doc
@ -588,11 +591,22 @@ reader({get_kvrange, StartKey, EndKey, ScanWidth, SegList}, _From, State) ->
reader, reader,
State}; State};
false -> false ->
{reply, {L, BIC} =
binaryslot_reader(SlotsToFetchBinList, PressMethod, SegList) binaryslot_reader(SlotsToFetchBinList, PressMethod, SegList),
++ SlotsToPoint, FoldFun =
reader, fun(CacheEntry, Cache) ->
State} case CacheEntry of
{_ID, none} ->
Cache;
{ID, Header} ->
array:set(ID - 1, Header, Cache)
end
end,
BlockIdxC0 = lists:foldl(FoldFun, State#state.blockindex_cache, BIC),
{reply,
L ++ SlotsToPoint,
reader,
State#state{blockindex_cache = BlockIdxC0}}
end; end;
reader({get_slots, SlotList, SegList}, _From, State) -> reader({get_slots, SlotList, SegList}, _From, State) ->
SlotBins = SlotBins =
@ -1346,15 +1360,16 @@ pointer_mapfun(Pointer) ->
%% Return a function that can pull individual slot binaries from a binary %% Return a function that can pull individual slot binaries from a binary
%% covering multiple slots %% covering multiple slots
binarysplit_mapfun(MultiSlotBin, StartPos) -> binarysplit_mapfun(MultiSlotBin, StartPos) ->
fun({SP, L, _ID, SK, EK}) -> fun({SP, L, ID, SK, EK}) ->
Start = SP - StartPos, Start = SP - StartPos,
<<_Pre:Start/binary, SlotBin:L/binary, _Post/binary>> = MultiSlotBin, <<_Pre:Start/binary, SlotBin:L/binary, _Post/binary>> = MultiSlotBin,
{SlotBin, SK, EK} {SlotBin, ID, SK, EK}
end. end.
-spec read_slots(file:io_device(), list(), -spec read_slots(file:io_device(), list(),
{false|list(), any()}, press_methods()) -> list(). {false|list(), any()}, press_methods())
-> list(binaryslot_element()).
%% @doc %% @doc
%% The reading of sots will return a list of either 2-tuples containing %% The reading of sots will return a list of either 2-tuples containing
%% {K, V} pairs - or 3-tuples containing {Binary, SK, EK}. The 3 tuples %% {K, V} pairs - or 3-tuples containing {Binary, SK, EK}. The 3 tuples
@ -1416,28 +1431,33 @@ read_slots(Handle, SlotList, {SegList, BlockIndexCache}, PressMethod) ->
lists:foldl(BinMapFun, [], SlotList). lists:foldl(BinMapFun, [], SlotList).
-spec binaryslot_reader(list({tuple(), tuple()}|{binary(), tuple(), tuple()}), -spec binaryslot_reader(list(binaryslot_element()),
native|lz4, native|lz4,
leveled_codec:segment_list()) leveled_codec:segment_list())
-> list({tuple(), tuple()}). -> {list({tuple(), tuple()}),
list({integer(), binary()})}.
%% @doc %% @doc
%% Read the binary slots converting them to {K, V} pairs if they were not %% Read the binary slots converting them to {K, V} pairs if they were not
%% already {K, V} pairs %% already {K, V} pairs
binaryslot_reader(SlotBinsToFetch, PressMethod, SegList) -> binaryslot_reader(SlotBinsToFetch, PressMethod, SegList) ->
binaryslot_reader(SlotBinsToFetch, PressMethod, SegList, []). binaryslot_reader(SlotBinsToFetch, PressMethod, SegList, [], []).
binaryslot_reader([], _PressMethod, _SegList, Acc) -> binaryslot_reader([], _PressMethod, _SegList, Acc, BIAcc) ->
Acc; {Acc, BIAcc};
binaryslot_reader([{SlotBin, SK, EK}|Tail], PressMethod, SegList, Acc) -> binaryslot_reader([{SlotBin, ID, SK, EK}|Tail],
PressMethod, SegList, Acc, BIAcc) ->
{TrimmedL, BICache} =
binaryslot_trimmedlist(SlotBin,
SK, EK,
PressMethod,
SegList),
binaryslot_reader(Tail, binaryslot_reader(Tail,
PressMethod, PressMethod,
SegList, SegList,
Acc ++ binaryslot_trimmedlist(SlotBin, Acc ++ TrimmedL,
SK, EK, [{ID, BICache}|BIAcc]);
PressMethod, binaryslot_reader([{K, V}|Tail], PressMethod, SegList, Acc, BIAcc) ->
SegList)); binaryslot_reader(Tail, PressMethod, SegList, Acc ++ [{K, V}], BIAcc).
binaryslot_reader([{K, V}|Tail], PressMethod, SegList, Acc) ->
binaryslot_reader(Tail, PressMethod, SegList, Acc ++ [{K, V}]).
read_length_list(Handle, LengthList) -> read_length_list(Handle, LengthList) ->
@ -1496,7 +1516,7 @@ binaryslot_tolist(FullBin, PressMethod) ->
binaryslot_trimmedlist(FullBin, all, all, PressMethod, false) -> binaryslot_trimmedlist(FullBin, all, all, PressMethod, false) ->
binaryslot_tolist(FullBin, PressMethod); {binaryslot_tolist(FullBin, PressMethod), none};
binaryslot_trimmedlist(FullBin, StartKey, EndKey, PressMethod, SegList) -> binaryslot_trimmedlist(FullBin, StartKey, EndKey, PressMethod, SegList) ->
LTrimFun = fun({K, _V}) -> K < StartKey end, LTrimFun = fun({K, _V}) -> K < StartKey end,
RTrimFun = fun({K, _V}) -> not leveled_codec:endkey_passed(EndKey, K) end, RTrimFun = fun({K, _V}) -> not leveled_codec:endkey_passed(EndKey, K) end,
@ -1519,9 +1539,11 @@ binaryslot_trimmedlist(FullBin, StartKey, EndKey, PressMethod, SegList) ->
{LastKey, _LV} -> {LastKey, _LV} ->
{_LDrop, RKeep} = lists:splitwith(LTrimFun, {_LDrop, RKeep} = lists:splitwith(LTrimFun,
BlockList), BlockList),
case leveled_codec:endkey_passed(EndKey, LastKey) of case leveled_codec:endkey_passed(EndKey,
LastKey) of
true -> true ->
{LKeep, _RDrop} = lists:splitwith(RTrimFun, RKeep), {LKeep, _RDrop}
= lists:splitwith(RTrimFun, RKeep),
{Acc ++ LKeep, false}; {Acc ++ LKeep, false};
false -> false ->
{Acc ++ RKeep, true} {Acc ++ RKeep, true}
@ -1584,20 +1606,21 @@ binaryslot_trimmedlist(FullBin, StartKey, EndKey, PressMethod, SegList) ->
end end
end, end,
{Acc, _Continue} = lists:foldl(BlockCheckFun, {[], true}, BlocksToCheck), {Acc, _Continue} = lists:foldl(BlockCheckFun, {[], true}, BlocksToCheck),
Acc; {Acc, none};
{{Header, _Blocks}, SegList} -> {{Header, _Blocks}, SegList} ->
BL = ?BLOCK_LENGTHS_LENGTH, BL = ?BLOCK_LENGTHS_LENGTH,
<<BlockLengths:BL/binary, BlockIdx/binary>> = Header, <<BlockLengths:BL/binary, BlockIdx/binary>> = Header,
PosList = find_pos(BlockIdx, SegList, [], 0), PosList = find_pos(BlockIdx, SegList, [], 0),
check_blocks(PosList, KVL = check_blocks(PosList,
FullBin, FullBin,
BlockLengths, BlockLengths,
byte_size(BlockIdx), byte_size(BlockIdx),
false, false,
PressMethod, PressMethod,
[]); []),
{KVL, Header};
{crc_wonky, _} -> {crc_wonky, _} ->
[] {[], none}
end. end.
@ -2293,8 +2316,9 @@ indexed_list_allindexkeys_test() ->
% "Indexed list flattened in ~w microseconds ~n", % "Indexed list flattened in ~w microseconds ~n",
% [timer:now_diff(os:timestamp(), SW)]), % [timer:now_diff(os:timestamp(), SW)]),
?assertMatch(Keys, BinToList), ?assertMatch(Keys, BinToList),
?assertMatch(Keys, ?assertMatch({Keys, none}, binaryslot_trimmedlist(FullBin,
binaryslot_trimmedlist(FullBin, all, all, native, false)). all, all,
native, false)).
indexed_list_allindexkeys_nolookup_test() -> indexed_list_allindexkeys_nolookup_test() ->
Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(1000)), Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(1000)),
@ -2308,8 +2332,9 @@ indexed_list_allindexkeys_nolookup_test() ->
% "Indexed list flattened in ~w microseconds ~n", % "Indexed list flattened in ~w microseconds ~n",
% [timer:now_diff(os:timestamp(), SW)]), % [timer:now_diff(os:timestamp(), SW)]),
?assertMatch(Keys, BinToList), ?assertMatch(Keys, BinToList),
?assertMatch(Keys, ?assertMatch({Keys, none}, binaryslot_trimmedlist(FullBin,
binaryslot_trimmedlist(FullBin, all, all, native, false)). all, all,
native, false)).
indexed_list_allindexkeys_trimmed_test() -> indexed_list_allindexkeys_trimmed_test() ->
Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(150)), Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(150)),
@ -2318,36 +2343,36 @@ indexed_list_allindexkeys_trimmed_test() ->
generate_binary_slot(lookup, Keys, native, no_timing), generate_binary_slot(lookup, Keys, native, no_timing),
EmptySlotSize = ?LOOK_SLOTSIZE - 1, EmptySlotSize = ?LOOK_SLOTSIZE - 1,
?assertMatch(<<_BL:20/binary, EmptySlotSize:8/integer>>, Header), ?assertMatch(<<_BL:20/binary, EmptySlotSize:8/integer>>, Header),
?assertMatch(Keys, binaryslot_trimmedlist(FullBin, ?assertMatch({Keys, none}, binaryslot_trimmedlist(FullBin,
{i, {i,
"Bucket", "Bucket",
{"t1_int", 0}, {"t1_int", 0},
null}, null},
{i, {i,
"Bucket", "Bucket",
{"t1_int", 99999}, {"t1_int", 99999},
null}, null},
native, native,
false)), false)),
{SK1, _} = lists:nth(10, Keys), {SK1, _} = lists:nth(10, Keys),
{EK1, _} = lists:nth(100, Keys), {EK1, _} = lists:nth(100, Keys),
R1 = lists:sublist(Keys, 10, 91), R1 = lists:sublist(Keys, 10, 91),
O1 = binaryslot_trimmedlist(FullBin, SK1, EK1, native, false), {O1, none} = binaryslot_trimmedlist(FullBin, SK1, EK1, native, false),
?assertMatch(91, length(O1)), ?assertMatch(91, length(O1)),
?assertMatch(R1, O1), ?assertMatch(R1, O1),
{SK2, _} = lists:nth(10, Keys), {SK2, _} = lists:nth(10, Keys),
{EK2, _} = lists:nth(20, Keys), {EK2, _} = lists:nth(20, Keys),
R2 = lists:sublist(Keys, 10, 11), R2 = lists:sublist(Keys, 10, 11),
O2 = binaryslot_trimmedlist(FullBin, SK2, EK2, native, false), {O2, none} = binaryslot_trimmedlist(FullBin, SK2, EK2, native, false),
?assertMatch(11, length(O2)), ?assertMatch(11, length(O2)),
?assertMatch(R2, O2), ?assertMatch(R2, O2),
{SK3, _} = lists:nth(?LOOK_SLOTSIZE - 1, Keys), {SK3, _} = lists:nth(?LOOK_SLOTSIZE - 1, Keys),
{EK3, _} = lists:nth(?LOOK_SLOTSIZE, Keys), {EK3, _} = lists:nth(?LOOK_SLOTSIZE, Keys),
R3 = lists:sublist(Keys, ?LOOK_SLOTSIZE - 1, 2), R3 = lists:sublist(Keys, ?LOOK_SLOTSIZE - 1, 2),
O3 = binaryslot_trimmedlist(FullBin, SK3, EK3, native, false), {O3, none} = binaryslot_trimmedlist(FullBin, SK3, EK3, native, false),
?assertMatch(2, length(O3)), ?assertMatch(2, length(O3)),
?assertMatch(R3, O3). ?assertMatch(R3, O3).
@ -2408,7 +2433,7 @@ indexed_list_mixedkeys_bitflip_test() ->
{SK1, _} = lists:nth(10, Keys), {SK1, _} = lists:nth(10, Keys),
{EK1, _} = lists:nth(20, Keys), {EK1, _} = lists:nth(20, Keys),
O1 = binaryslot_trimmedlist(SlotBin3, SK1, EK1, native, false), {O1, none} = binaryslot_trimmedlist(SlotBin3, SK1, EK1, native, false),
?assertMatch([], O1), ?assertMatch([], O1),
SlotBin4 = flip_byte(SlotBin, 0, 20), SlotBin4 = flip_byte(SlotBin, 0, 20),
@ -2420,8 +2445,8 @@ indexed_list_mixedkeys_bitflip_test() ->
ToList5 = binaryslot_tolist(SlotBin5, native), ToList5 = binaryslot_tolist(SlotBin5, native),
?assertMatch([], ToList4), ?assertMatch([], ToList4),
?assertMatch([], ToList5), ?assertMatch([], ToList5),
O4 = binaryslot_trimmedlist(SlotBin4, SK1, EK1, native, false), {O4, none} = binaryslot_trimmedlist(SlotBin4, SK1, EK1, native, false),
O5 = binaryslot_trimmedlist(SlotBin4, SK1, EK1, native, false), {O5, none} = binaryslot_trimmedlist(SlotBin4, SK1, EK1, native, false),
?assertMatch([], O4), ?assertMatch([], O4),
?assertMatch([], O5). ?assertMatch([], O5).

1
test/end_to_end/riak_SUITE.erl
View file

@ -83,6 +83,7 @@ crossbucket_aae(_Config) ->
ok = leveled_bookie:book_close(Bookie2), ok = leveled_bookie:book_close(Bookie2),
{ok, Bookie2A} = leveled_bookie:book_start(StartOpts2), {ok, Bookie2A} = leveled_bookie:book_start(StartOpts2),
test_segfilter_query(Bookie2A, CLs),
test_segfilter_query(Bookie2A, CLs), test_segfilter_query(Bookie2A, CLs),
test_singledelta_stores(Bookie2A, Bookie3, small, {B1, K1}), test_singledelta_stores(Bookie2A, Bookie3, small, {B1, K1}),