diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index dbbb16e..1f9560c 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -200,7 +200,11 @@ strip_to_keyseqonly({LK, V}) -> {LK, element(1, V)}. -spec strip_to_indexdetails(ledger_kv()) -> {integer(), segment_hash(), last_moddate()}. strip_to_indexdetails({_, V}) when tuple_size(V) == 4 -> - {element(1, V), element(3, V), undefined}. + % A v1 value + {element(1, V), element(3, V), undefined}; +strip_to_indexdetails({_, V}) when tuple_size(V) > 4 -> + % A v2 value should have a fith element - Last Modified Date + {element(1, V), element(3, V), element(5, V)}. -spec striphead_to_v1details(ledger_value()) -> ledger_value(). striphead_to_v1details(V) -> @@ -612,9 +616,20 @@ generate_ledgerkv(PrimaryKey, SQN, Obj, Size, TS) -> Value = {SQN, Status, Hash, - MD}, + MD, + get_last_lastmodification(LastMods)}, {Bucket, Key, Value, {Hash, ObjHash}, LastMods}. +-spec get_last_lastmodification(list(erlang:timestamp())) -> non_neg_integer(). +%% @doc +%% Get the highest of the last modifications measured in seconds. This will be +%% stored as 4 bytes (unsigned) so will last for another 80 + years +get_last_lastmodification([]) -> + 0; +get_last_lastmodification(LastMods) -> + {Mega, Sec, _Micro} = lists:max(LastMods), + Mega * 1000000 + Sec. + extract_metadata(Obj, Size, ?RIAK_TAG) -> riak_extract_metadata(Obj, Size); @@ -623,7 +638,7 @@ extract_metadata(Obj, Size, ?STD_TAG) -> get_size(PK, Value) -> {Tag, _Bucket, _Key, _} = PK, - {_, _, _, MD} = Value, + MD = element(4, Value), case Tag of ?RIAK_TAG -> {_RMD, _VC, _Hash, Size} = MD, @@ -640,7 +655,7 @@ get_size(PK, Value) -> %% the sorted vclock) get_keyandobjhash(LK, Value) -> {Tag, Bucket, Key, _} = LK, - {_, _, _, MD} = Value, + MD = element(4, Value), case Tag of ?IDX_TAG -> from_ledgerkey(LK); % returns {Bucket, Key, IdxValue} diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index 0600005..4b0510b 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -88,9 +88,8 @@ -define(TIMING_SAMPLESIZE, 100). -define(CACHE_SIZE, 32). -define(BLOCK_LENGTHS_LENGTH, 20). --define(LMD_LENGTH, 6). +-define(LMD_LENGTH, 4). -define(FLIPPER32, 4294967295). --define(FLIPPER48, 281474976710655). -define(COMPRESS_AT_LEVEL, 1). -define(INDEX_MODDATE, true). @@ -881,7 +880,6 @@ fetch(LedgerKey, Hash, State, Timings0) -> Timings3}; {BlockLengths, _LMD, PosBin} -> PosList = find_pos(PosBin, extra_hash(Hash), [], 0), - io:format("Fetching referring to cache with PosList ~w~n", [PosList]), case PosList of [] -> {_SW3, Timings3} = @@ -1326,7 +1324,7 @@ accumulate_positions({K, V}, {PosBinAcc, NoHashCount, HashAcc, LMDAcc}) -> %% Get the last modified date. If no Last Modified Date on any object, can't %% add the accelerator and should check each object in turn take_max_lastmoddate(undefined, _LMDAcc) -> - ?FLIPPER48; + ?FLIPPER32; take_max_lastmoddate(LMD, LMDAcc) -> max(LMD, LMDAcc). @@ -1445,7 +1443,7 @@ generate_binary_slot(Lookup, KVL, PressMethod, IndexModDate, BuildTimings0) -> B3L:32/integer, B4L:32/integer, B5L:32/integer, - LMD:48/integer, + LMD:32/integer, PosBinIndex/binary>>; false -> < - io:format("Key matched~n"), {K, V}; _ -> case LedgerKeyToCheck of @@ -1520,10 +1514,10 @@ check_blocks([Pos|Rest], BlockPointer, BlockLengths, PosBinLength, -spec additional_offset(boolean()) -> pos_integer(). %% @doc -%% 4-byte CRC, 4-byte pos, 4-byte CRC, 5x4 byte lengths, 6 byte LMD +%% 4-byte CRC, 4-byte pos, 4-byte CRC, 5x4 byte lengths, 4 byte LMD %% LMD may not be present additional_offset(true) -> - ?BLOCK_LENGTHS_LENGTH + 4 + 4 + 4 + 6; + ?BLOCK_LENGTHS_LENGTH + 4 + 4 + 4 + ?LMD_LENGTH; additional_offset(false) -> ?BLOCK_LENGTHS_LENGTH + 4 + 4 + 4. @@ -1707,7 +1701,7 @@ extract_header(none, _IdxModDate) -> none; % used when the block cache has returned none extract_header(Header, true) -> BL = ?BLOCK_LENGTHS_LENGTH, - <> = Header, + <> = Header, {BlockLengths, LMD, PosBinIndex}; extract_header(Header, false) -> BL = ?BLOCK_LENGTHS_LENGTH, @@ -2538,8 +2532,8 @@ indexed_list_allindexkeys_test() -> {{HeaderF, FullBinF, _HL, _LK}, no_timing} = generate_binary_slot(lookup, Keys, native, false, no_timing), EmptySlotSize = ?LOOK_SLOTSIZE - 1, - LMD = ?FLIPPER48, - ?assertMatch(<<_BL:20/binary, LMD:48/integer, EmptySlotSize:8/integer>>, + LMD = ?FLIPPER32, + ?assertMatch(<<_BL:20/binary, LMD:32/integer, EmptySlotSize:8/integer>>, HeaderT), ?assertMatch(<<_BL:20/binary, EmptySlotSize:8/integer>>, HeaderF), @@ -2567,7 +2561,7 @@ indexed_list_allindexkeys_nolookup_test() -> ?NOLOOK_SLOTSIZE), {{Header, FullBin, _HL, _LK}, no_timing} = generate_binary_slot(no_lookup, Keys, native, ?INDEX_MODDATE,no_timing), - ?assertMatch(<<_BL:20/binary, _LMD:48/integer, 127:8/integer>>, Header), + ?assertMatch(<<_BL:20/binary, _LMD:32/integer, 127:8/integer>>, Header), % SW = os:timestamp(), BinToList = binaryslot_tolist(FullBin, native, ?INDEX_MODDATE), % io:format(user, @@ -2586,7 +2580,7 @@ indexed_list_allindexkeys_trimmed_test() -> {{Header, FullBin, _HL, _LK}, no_timing} = generate_binary_slot(lookup, Keys, native, ?INDEX_MODDATE,no_timing), EmptySlotSize = ?LOOK_SLOTSIZE - 1, - ?assertMatch(<<_BL:20/binary, _LMD:48/integer, EmptySlotSize:8/integer>>, + ?assertMatch(<<_BL:20/binary, _LMD:32/integer, EmptySlotSize:8/integer>>, Header), ?assertMatch({Keys, none}, binaryslot_trimmedlist(FullBin, {i, @@ -2640,7 +2634,7 @@ indexed_list_mixedkeys_bitflip_test() -> _B3L:32/integer, _B4L:32/integer, _B5L:32/integer, - _LMD:48/integer, + _LMD:32/integer, PosBin/binary>> = Header, TestKey1 = element(1, lists:nth(1, KVL1)),