umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Start adding in last_moified dates

Browse source 
With updated specs
This commit is contained in:
Martin Sumner 2018-10-29 21:50:32 +00:00
parent 14fd67e535
commit 8ba28700eb
2 changed files with 49 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

9
src/leveled_codec.erl
View file

@ -139,7 +139,8 @@
journal_keychanges/0,
index_specs/0,
segment_list/0,
maybe_lookup/0]).
maybe_lookup/0,
last_moddate/0]).
%%%============================================================================
@ -194,8 +195,10 @@ strip_to_seqonly({_, V}) -> element(1, V).
-spec strip_to_keyseqonly(ledger_kv()) -> {ledger_key(), integer()}.
strip_to_keyseqonly({LK, V}) -> {LK, element(1, V)}.
-spec strip_to_indexdetails(ledger_kv()) -> {integer(), segment_hash()}.
strip_to_indexdetails({_, V}) -> {element(1, V), element(3, V)}.
-spec strip_to_indexdetails(ledger_kv()) ->
{integer(), segment_hash(), last_moddate()}.
strip_to_indexdetails({_, V}) when tuple_size(V) == 4 ->
{element(1, V), element(3, V), undefined}.
-spec striphead_to_v1details(ledger_value()) -> ledger_value().
striphead_to_v1details(V) ->

52
src/leveled_sst.erl
View file

@ -1131,8 +1131,21 @@ lookup_slots(StartKey, EndKey, Tree) ->
%% based on a 17-bit hash (so 0.0039 fpr).
accumulate_positions({K, V}, {PosBinAcc, NoHashCount, HashAcc}) ->
{_SQN, H1} = leveled_codec:strip_to_indexdetails({K, V}),
-spec accumulate_positions(leveled_codec:ledger_kv(),
{binary(),
non_neg_integer(),
list(non_neg_integer()),
leveled_codec:last_moddate()}) ->
{binary(),
non_neg_integer(),
list(non_neg_integer()),
leveled_codec:last_moddate()}.
%% @doc
%% Fold function use to accumulate the position information needed to
%% populate the summary of the slot
accumulate_positions({K, V}, {PosBinAcc, NoHashCount, HashAcc, LMDAcc}) ->
{_SQN, H1, LMD} = leveled_codec:strip_to_indexdetails({K, V}),
LMDAcc0 = take_max_lastmoddate(LMD, LMDAcc),
PosH1 = extra_hash(H1),
case is_integer(PosH1) of
true ->
@ -1140,7 +1153,8 @@ accumulate_positions({K, V}, {PosBinAcc, NoHashCount, HashAcc}) ->
0 ->
{<<1:1/integer, PosH1:15/integer,PosBinAcc/binary>>,
0,
[H1|HashAcc]};
[H1|HashAcc],
LMDAcc0};
N ->
% The No Hash Count is an integer between 0 and 127
% and so at read time should count NHC + 1
@ -1151,12 +1165,27 @@ accumulate_positions({K, V}, {PosBinAcc, NoHashCount, HashAcc}) ->
NHC:7/integer,
PosBinAcc/binary>>,
0,
HashAcc}
HashAcc,
LMDAcc0}
end;
false ->
{PosBinAcc, NoHashCount + 1, HashAcc}
{PosBinAcc, NoHashCount + 1, HashAcc, LMDAcc0}
end.
-spec take_max_lastmoddate(leveled_codec:last_moddate(),
leveled_codec:last_moddate()) ->
leveled_codec:last_moddate().
%% @doc
%% Get the last modified date. If no Last Modified Date on any object, can't
%% add the accelerator and should check each object in turn
take_max_lastmoddate(_LMD, undefined) ->
undefined;
take_max_lastmoddate(undefined, _LMDAcc) ->
undefined;
take_max_lastmoddate(LMD, LMDAcc) ->
max(LMD, LMDAcc).
-spec generate_binary_slot(leveled_codec:lookup(),
list(leveled_codec:ledger_kv()),
press_method(),
@ -1176,8 +1205,8 @@ generate_binary_slot(Lookup, KVL, PressMethod, BuildTimings0) ->
{HashL, PosBinIndex} =
case Lookup of
lookup ->
InitAcc = {<<>>, 0, []},
{PosBinIndex0, NHC, HashL0} =
InitAcc = {<<>>, 0, [], 0},
{PosBinIndex0, NHC, HashL0, _LMD} =
lists:foldr(fun accumulate_positions/2, InitAcc, KVL),
PosBinIndex1 =
case NHC of
@ -1275,8 +1304,6 @@ generate_binary_slot(Lookup, KVL, PressMethod, BuildTimings0) ->
{{Header, SlotBin, HashL, LastKey}, BuildTimings3}.
-spec check_blocks(list(integer()),
binary()|{file:io_device(), integer()},
binary(),
@ -2878,5 +2905,12 @@ timings_test() ->
?assertMatch(true, T4#sst_timings.slot_fetch_time >
T3#sst_timings.slot_fetch_time).
take_max_lastmoddate_test() ->
% TODO: Remove this test
% Temporarily added to make dialyzer happy (until we've made use of last
% modified dates
?assertMatch(1, take_max_lastmoddate(0, 1)).
-endif.