Add cache population for non-yielding range fetches
More likely to require caching at lower levels.
This commit is contained in:
Martin Sumner
parent
779ccd9c2a
commit
989f23bca6
2 changed files with 83 additions and 57 deletions
|
|
@ -145,7 +145,8 @@
|
|||
:: {next,
|
||||
leveled_pmanifest:manifest_entry(),
|
||||
leveled_codec:ledger_key()|all}.
|
||||
|
||||
-type binaryslot_element()
|
||||
:: {tuple(), tuple()}|{binary(), integer(), tuple(), tuple()}.
|
||||
|
||||
%% yield_blockquery is used to detemrine if the work necessary to process a
|
||||
%% range query beyond the fetching the slot should be managed from within
|
||||
|
|
@ -375,8 +376,9 @@ sst_getfilteredrange(Pid, StartKey, EndKey, ScanWidth, SegList) ->
|
|||
ScanWidth, SegList0},
|
||||
infinity) of
|
||||
{yield, SlotsToFetchBinList, SlotsToPoint, PressMethod} ->
|
||||
binaryslot_reader(SlotsToFetchBinList, PressMethod, SegList0)
|
||||
++ SlotsToPoint;
|
||||
{L, _BIC} =
|
||||
binaryslot_reader(SlotsToFetchBinList, PressMethod, SegList0),
|
||||
L ++ SlotsToPoint;
|
||||
Reply ->
|
||||
Reply
|
||||
end.
|
||||
|
|
@ -405,7 +407,8 @@ sst_getfilteredslots(Pid, SlotList, SegList) ->
|
|||
SegL0 = tune_seglist(SegList),
|
||||
{SlotBins, PressMethod} =
|
||||
gen_fsm:sync_send_event(Pid, {get_slots, SlotList, SegL0}, infinity),
|
||||
binaryslot_reader(SlotBins, PressMethod, SegL0).
|
||||
{L, _BIC} = binaryslot_reader(SlotBins, PressMethod, SegL0),
|
||||
L.
|
||||
|
||||
-spec sst_getmaxsequencenumber(pid()) -> integer().
|
||||
%% @doc
|
||||
|
|
@ -588,11 +591,22 @@ reader({get_kvrange, StartKey, EndKey, ScanWidth, SegList}, _From, State) ->
|
|||
reader,
|
||||
State};
|
||||
false ->
|
||||
{reply,
|
||||
binaryslot_reader(SlotsToFetchBinList, PressMethod, SegList)
|
||||
++ SlotsToPoint,
|
||||
reader,
|
||||
State}
|
||||
{L, BIC} =
|
||||
binaryslot_reader(SlotsToFetchBinList, PressMethod, SegList),
|
||||
FoldFun =
|
||||
fun(CacheEntry, Cache) ->
|
||||
case CacheEntry of
|
||||
{_ID, none} ->
|
||||
Cache;
|
||||
{ID, Header} ->
|
||||
array:set(ID - 1, Header, Cache)
|
||||
end
|
||||
end,
|
||||
BlockIdxC0 = lists:foldl(FoldFun, State#state.blockindex_cache, BIC),
|
||||
{reply,
|
||||
L ++ SlotsToPoint,
|
||||
reader,
|
||||
State#state{blockindex_cache = BlockIdxC0}}
|
||||
end;
|
||||
reader({get_slots, SlotList, SegList}, _From, State) ->
|
||||
SlotBins =
|
||||
|
|
@ -1346,15 +1360,16 @@ pointer_mapfun(Pointer) ->
|
|||
%% Return a function that can pull individual slot binaries from a binary
|
||||
%% covering multiple slots
|
||||
binarysplit_mapfun(MultiSlotBin, StartPos) ->
|
||||
fun({SP, L, _ID, SK, EK}) ->
|
||||
fun({SP, L, ID, SK, EK}) ->
|
||||
Start = SP - StartPos,
|
||||
<<_Pre:Start/binary, SlotBin:L/binary, _Post/binary>> = MultiSlotBin,
|
||||
{SlotBin, SK, EK}
|
||||
{SlotBin, ID, SK, EK}
|
||||
end.
|
||||
|
||||
|
||||
-spec read_slots(file:io_device(), list(),
|
||||
{false|list(), any()}, press_methods()) -> list().
|
||||
{false|list(), any()}, press_methods())
|
||||
-> list(binaryslot_element()).
|
||||
%% @doc
|
||||
%% The reading of sots will return a list of either 2-tuples containing
|
||||
%% {K, V} pairs - or 3-tuples containing {Binary, SK, EK}. The 3 tuples
|
||||
|
|
@ -1416,28 +1431,33 @@ read_slots(Handle, SlotList, {SegList, BlockIndexCache}, PressMethod) ->
|
|||
lists:foldl(BinMapFun, [], SlotList).
|
||||
|
||||
|
||||
-spec binaryslot_reader(list({tuple(), tuple()}|{binary(), tuple(), tuple()}),
|
||||
-spec binaryslot_reader(list(binaryslot_element()),
|
||||
native|lz4,
|
||||
leveled_codec:segment_list())
|
||||
-> list({tuple(), tuple()}).
|
||||
-> {list({tuple(), tuple()}),
|
||||
list({integer(), binary()})}.
|
||||
%% @doc
|
||||
%% Read the binary slots converting them to {K, V} pairs if they were not
|
||||
%% already {K, V} pairs
|
||||
binaryslot_reader(SlotBinsToFetch, PressMethod, SegList) ->
|
||||
binaryslot_reader(SlotBinsToFetch, PressMethod, SegList, []).
|
||||
binaryslot_reader(SlotBinsToFetch, PressMethod, SegList, [], []).
|
||||
|
||||
binaryslot_reader([], _PressMethod, _SegList, Acc) ->
|
||||
Acc;
|
||||
binaryslot_reader([{SlotBin, SK, EK}|Tail], PressMethod, SegList, Acc) ->
|
||||
binaryslot_reader([], _PressMethod, _SegList, Acc, BIAcc) ->
|
||||
{Acc, BIAcc};
|
||||
binaryslot_reader([{SlotBin, ID, SK, EK}|Tail],
|
||||
PressMethod, SegList, Acc, BIAcc) ->
|
||||
{TrimmedL, BICache} =
|
||||
binaryslot_trimmedlist(SlotBin,
|
||||
SK, EK,
|
||||
PressMethod,
|
||||
SegList),
|
||||
binaryslot_reader(Tail,
|
||||
PressMethod,
|
||||
SegList,
|
||||
Acc ++ binaryslot_trimmedlist(SlotBin,
|
||||
SK, EK,
|
||||
PressMethod,
|
||||
SegList));
|
||||
binaryslot_reader([{K, V}|Tail], PressMethod, SegList, Acc) ->
|
||||
binaryslot_reader(Tail, PressMethod, SegList, Acc ++ [{K, V}]).
|
||||
Acc ++ TrimmedL,
|
||||
[{ID, BICache}|BIAcc]);
|
||||
binaryslot_reader([{K, V}|Tail], PressMethod, SegList, Acc, BIAcc) ->
|
||||
binaryslot_reader(Tail, PressMethod, SegList, Acc ++ [{K, V}], BIAcc).
|
||||
|
||||
|
||||
read_length_list(Handle, LengthList) ->
|
||||
|
|
@ -1496,7 +1516,7 @@ binaryslot_tolist(FullBin, PressMethod) ->
|
|||
|
||||
|
||||
binaryslot_trimmedlist(FullBin, all, all, PressMethod, false) ->
|
||||
binaryslot_tolist(FullBin, PressMethod);
|
||||
{binaryslot_tolist(FullBin, PressMethod), none};
|
||||
binaryslot_trimmedlist(FullBin, StartKey, EndKey, PressMethod, SegList) ->
|
||||
LTrimFun = fun({K, _V}) -> K < StartKey end,
|
||||
RTrimFun = fun({K, _V}) -> not leveled_codec:endkey_passed(EndKey, K) end,
|
||||
|
|
@ -1519,9 +1539,11 @@ binaryslot_trimmedlist(FullBin, StartKey, EndKey, PressMethod, SegList) ->
|
|||
{LastKey, _LV} ->
|
||||
{_LDrop, RKeep} = lists:splitwith(LTrimFun,
|
||||
BlockList),
|
||||
case leveled_codec:endkey_passed(EndKey, LastKey) of
|
||||
case leveled_codec:endkey_passed(EndKey,
|
||||
LastKey) of
|
||||
true ->
|
||||
{LKeep, _RDrop} = lists:splitwith(RTrimFun, RKeep),
|
||||
{LKeep, _RDrop}
|
||||
= lists:splitwith(RTrimFun, RKeep),
|
||||
{Acc ++ LKeep, false};
|
||||
false ->
|
||||
{Acc ++ RKeep, true}
|
||||
|
|
@ -1584,20 +1606,21 @@ binaryslot_trimmedlist(FullBin, StartKey, EndKey, PressMethod, SegList) ->
|
|||
end
|
||||
end,
|
||||
{Acc, _Continue} = lists:foldl(BlockCheckFun, {[], true}, BlocksToCheck),
|
||||
Acc;
|
||||
{Acc, none};
|
||||
{{Header, _Blocks}, SegList} ->
|
||||
BL = ?BLOCK_LENGTHS_LENGTH,
|
||||
<<BlockLengths:BL/binary, BlockIdx/binary>> = Header,
|
||||
PosList = find_pos(BlockIdx, SegList, [], 0),
|
||||
check_blocks(PosList,
|
||||
FullBin,
|
||||
BlockLengths,
|
||||
byte_size(BlockIdx),
|
||||
false,
|
||||
PressMethod,
|
||||
[]);
|
||||
KVL = check_blocks(PosList,
|
||||
FullBin,
|
||||
BlockLengths,
|
||||
byte_size(BlockIdx),
|
||||
false,
|
||||
PressMethod,
|
||||
[]),
|
||||
{KVL, Header};
|
||||
{crc_wonky, _} ->
|
||||
[]
|
||||
{[], none}
|
||||
end.
|
||||
|
||||
|
||||
|
|
@ -2293,8 +2316,9 @@ indexed_list_allindexkeys_test() ->
|
|||
% "Indexed list flattened in ~w microseconds ~n",
|
||||
% [timer:now_diff(os:timestamp(), SW)]),
|
||||
?assertMatch(Keys, BinToList),
|
||||
?assertMatch(Keys,
|
||||
binaryslot_trimmedlist(FullBin, all, all, native, false)).
|
||||
?assertMatch({Keys, none}, binaryslot_trimmedlist(FullBin,
|
||||
all, all,
|
||||
native, false)).
|
||||
|
||||
indexed_list_allindexkeys_nolookup_test() ->
|
||||
Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(1000)),
|
||||
|
|
@ -2308,8 +2332,9 @@ indexed_list_allindexkeys_nolookup_test() ->
|
|||
% "Indexed list flattened in ~w microseconds ~n",
|
||||
% [timer:now_diff(os:timestamp(), SW)]),
|
||||
?assertMatch(Keys, BinToList),
|
||||
?assertMatch(Keys,
|
||||
binaryslot_trimmedlist(FullBin, all, all, native, false)).
|
||||
?assertMatch({Keys, none}, binaryslot_trimmedlist(FullBin,
|
||||
all, all,
|
||||
native, false)).
|
||||
|
||||
indexed_list_allindexkeys_trimmed_test() ->
|
||||
Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(150)),
|
||||
|
|
@ -2318,36 +2343,36 @@ indexed_list_allindexkeys_trimmed_test() ->
|
|||
generate_binary_slot(lookup, Keys, native, no_timing),
|
||||
EmptySlotSize = ?LOOK_SLOTSIZE - 1,
|
||||
?assertMatch(<<_BL:20/binary, EmptySlotSize:8/integer>>, Header),
|
||||
?assertMatch(Keys, binaryslot_trimmedlist(FullBin,
|
||||
{i,
|
||||
"Bucket",
|
||||
{"t1_int", 0},
|
||||
null},
|
||||
{i,
|
||||
"Bucket",
|
||||
{"t1_int", 99999},
|
||||
null},
|
||||
native,
|
||||
false)),
|
||||
?assertMatch({Keys, none}, binaryslot_trimmedlist(FullBin,
|
||||
{i,
|
||||
"Bucket",
|
||||
{"t1_int", 0},
|
||||
null},
|
||||
{i,
|
||||
"Bucket",
|
||||
{"t1_int", 99999},
|
||||
null},
|
||||
native,
|
||||
false)),
|
||||
|
||||
{SK1, _} = lists:nth(10, Keys),
|
||||
{EK1, _} = lists:nth(100, Keys),
|
||||
R1 = lists:sublist(Keys, 10, 91),
|
||||
O1 = binaryslot_trimmedlist(FullBin, SK1, EK1, native, false),
|
||||
{O1, none} = binaryslot_trimmedlist(FullBin, SK1, EK1, native, false),
|
||||
?assertMatch(91, length(O1)),
|
||||
?assertMatch(R1, O1),
|
||||
|
||||
{SK2, _} = lists:nth(10, Keys),
|
||||
{EK2, _} = lists:nth(20, Keys),
|
||||
R2 = lists:sublist(Keys, 10, 11),
|
||||
O2 = binaryslot_trimmedlist(FullBin, SK2, EK2, native, false),
|
||||
{O2, none} = binaryslot_trimmedlist(FullBin, SK2, EK2, native, false),
|
||||
?assertMatch(11, length(O2)),
|
||||
?assertMatch(R2, O2),
|
||||
|
||||
{SK3, _} = lists:nth(?LOOK_SLOTSIZE - 1, Keys),
|
||||
{EK3, _} = lists:nth(?LOOK_SLOTSIZE, Keys),
|
||||
R3 = lists:sublist(Keys, ?LOOK_SLOTSIZE - 1, 2),
|
||||
O3 = binaryslot_trimmedlist(FullBin, SK3, EK3, native, false),
|
||||
{O3, none} = binaryslot_trimmedlist(FullBin, SK3, EK3, native, false),
|
||||
?assertMatch(2, length(O3)),
|
||||
?assertMatch(R3, O3).
|
||||
|
||||
|
|
@ -2408,7 +2433,7 @@ indexed_list_mixedkeys_bitflip_test() ->
|
|||
|
||||
{SK1, _} = lists:nth(10, Keys),
|
||||
{EK1, _} = lists:nth(20, Keys),
|
||||
O1 = binaryslot_trimmedlist(SlotBin3, SK1, EK1, native, false),
|
||||
{O1, none} = binaryslot_trimmedlist(SlotBin3, SK1, EK1, native, false),
|
||||
?assertMatch([], O1),
|
||||
|
||||
SlotBin4 = flip_byte(SlotBin, 0, 20),
|
||||
|
|
@ -2420,8 +2445,8 @@ indexed_list_mixedkeys_bitflip_test() ->
|
|||
ToList5 = binaryslot_tolist(SlotBin5, native),
|
||||
?assertMatch([], ToList4),
|
||||
?assertMatch([], ToList5),
|
||||
O4 = binaryslot_trimmedlist(SlotBin4, SK1, EK1, native, false),
|
||||
O5 = binaryslot_trimmedlist(SlotBin4, SK1, EK1, native, false),
|
||||
{O4, none} = binaryslot_trimmedlist(SlotBin4, SK1, EK1, native, false),
|
||||
{O5, none} = binaryslot_trimmedlist(SlotBin4, SK1, EK1, native, false),
|
||||
?assertMatch([], O4),
|
||||
?assertMatch([], O5).
|
||||
|
||||
|
|
|
|||
|
|
@ -83,6 +83,7 @@ crossbucket_aae(_Config) ->
|
|||
ok = leveled_bookie:book_close(Bookie2),
|
||||
{ok, Bookie2A} = leveled_bookie:book_start(StartOpts2),
|
||||
|
||||
test_segfilter_query(Bookie2A, CLs),
|
||||
test_segfilter_query(Bookie2A, CLs),
|
||||
|
||||
test_singledelta_stores(Bookie2A, Bookie3, small, {B1, K1}),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue