diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index b450bcc..81a5a36 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -626,7 +626,6 @@ loadqueue_ledgercache(Cache) -> %% snapshot is to be used for one specific query only (this is much quicker to %% setup, assuming the range is a small subset of the overall key space). snapshot_store(LedgerCache0, Penciller, Inker, SnapType, Query) -> - SW = os:timestamp(), LedgerCache = readycache_forsnapshot(LedgerCache0, Query), BookiesMem = {LedgerCache#ledger_cache.loader, LedgerCache#ledger_cache.index, @@ -637,7 +636,6 @@ snapshot_store(LedgerCache0, Penciller, Inker, SnapType, Query) -> snapshot_query = Query, bookies_mem = BookiesMem}, {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), - leveled_log:log_randomtimer("B0004", [cache_size(LedgerCache)], SW, 0.02), case SnapType of store -> InkerOpts = #inker_options{start_snapshot=true, @@ -671,9 +669,6 @@ maybe_longrunning(SW, Aspect) -> ok end. -cache_size(LedgerCache) -> - ets:info(LedgerCache#ledger_cache.mem, size). - bucket_stats(State, Bucket, Tag) -> {ok, LedgerSnapshot, _JournalSnapshot} = snapshot_store(State, ledger, @@ -1188,8 +1183,9 @@ maybepush_ledgercache(MaxCacheSize, Cache, Penciller) -> case leveled_penciller:pcl_pushmem(Penciller, CacheToLoad) of ok -> Cache0 = #ledger_cache{}, - true = ets:delete_all_objects(Tab), - {ok, Cache0#ledger_cache{mem=Tab}}; + true = ets:delete(Tab), + NewTab = ets:new(mem, [ordered_set]), + {ok, Cache0#ledger_cache{mem=NewTab}}; returned -> {returned, Cache} end; diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 2485395..19d660e 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -364,6 +364,7 @@ rolling({return_hashtable, IndexList, HashTreeBin}, _From, State) -> file:close(Handle), ok = rename_for_read(State#state.filename, NewName), leveled_log:log("CDB03", [NewName]), + ets:delete(State#state.hashtree), {NewHandle, Index, LastKey} = open_for_readonly(NewName, State#state.last_key), case State#state.deferred_delete of diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index 665c6dc..484d927 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -433,7 +433,8 @@ riak_extract_metadata(ObjBin, Size) -> %% <>. -riak_metadata_to_binary(VclockBin, SibData) -> +riak_metadata_to_binary(Vclock, SibData) -> + VclockBin = term_to_binary(Vclock), VclockLen = byte_size(VclockBin), % <>. @@ -454,7 +455,7 @@ riak_metadata_from_binary(V1Binary) -> SC when is_integer(SC) -> get_metadata_from_siblings(SibsBin, SibCount, []) end, - {VclockBin, SibMetaBinList}. + {binary_to_term(VclockBin), SibMetaBinList}. % Fixes the value length for each sibling to be zero, and so includes no value slimbin_content(MetaBin) -> diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index cb5d71c..5fd85ca 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -65,13 +65,14 @@ -include("include/leveled.hrl"). -define(MAX_SLOTS, 256). --define(SLOT_SIZE, 128). % This is not configurable --define(NOLOOK_MULT, 2). % How much bigger is a slot/block with no lookups --define(NOLOOK_SLOTSIZE, ?SLOT_SIZE * ?NOLOOK_MULT). +-define(LOOK_SLOTSIZE, 128). % This is not configurable +-define(LOOK_BLOCKSIZE, {24, 32}). +-define(NOLOOK_SLOTSIZE, 256). +-define(NOLOOK_BLOCKSIZE, {56, 32}). -define(COMPRESSION_LEVEL, 1). -define(BINARY_SETTINGS, [{compressed, ?COMPRESSION_LEVEL}]). % -define(LEVEL_BLOOM_BITS, [{0, 8}, {1, 10}, {2, 8}, {default, 6}]). --define(MERGE_SCANWIDTH, 8). +-define(MERGE_SCANWIDTH, 16). -define(DISCARD_EXT, ".discarded"). -define(DELETE_TIMEOUT, 10000). -define(TREE_TYPE, idxt). @@ -120,12 +121,20 @@ size :: integer(), max_sqn :: integer()}). +%% yield_blockquery is used to detemrine if the work necessary to process a +%% range query beyond the fetching the slot should be managed from within +%% this process, or should be handled by the calling process. +%% Handling within the calling process may lead to extra binary heap garbage +%% see Issue 52. Handling within the SST process may lead to contention and +%% extra copying. Files at the top of the tree yield, those lower down don't. + -record(state, {summary, handle :: file:fd(), sst_timings :: tuple(), penciller :: pid(), root_path, filename, + yield_blockquery = false :: boolean(), blockindex_cache}). @@ -196,15 +205,18 @@ sst_get(Pid, LedgerKey, Hash) -> gen_fsm:sync_send_event(Pid, {get_kv, LedgerKey, Hash}, infinity). sst_getkvrange(Pid, StartKey, EndKey, ScanWidth) -> - Reply = gen_fsm:sync_send_event(Pid, + case gen_fsm:sync_send_event(Pid, {get_kvrange, StartKey, EndKey, ScanWidth}, - infinity), - FetchFun = - fun({SlotBin, SK, EK}, Acc) -> - Acc ++ binaryslot_trimmedlist(SlotBin, SK, EK) - end, - {SlotsToFetchBinList, SlotsToPoint} = Reply, - lists:foldl(FetchFun, [], SlotsToFetchBinList) ++ SlotsToPoint. + infinity) of + {yield, SlotsToFetchBinList, SlotsToPoint} -> + FetchFun = + fun({SlotBin, SK, EK}, Acc) -> + Acc ++ binaryslot_trimmedlist(SlotBin, SK, EK) + end, + lists:foldl(FetchFun, [], SlotsToFetchBinList) ++ SlotsToPoint; + Reply -> + Reply + end. sst_getslots(Pid, SlotList) -> SlotBins = gen_fsm:sync_send_event(Pid, {get_slots, SlotList}, infinity), @@ -267,7 +279,10 @@ starting({sst_new, RootPath, Filename, Level, {SlotList, FirstKey}, MaxSQN}, Length, MaxSQN), ActualFilename = write_file(RootPath, Filename, SummaryBin, SlotsBin), - UpdState = read_file(ActualFilename, State#state{root_path=RootPath}), + YBQ = Level =< 2, + UpdState = read_file(ActualFilename, + State#state{root_path=RootPath, + yield_blockquery=YBQ}), Summary = UpdState#state.summary, leveled_log:log_timer("SST08", [ActualFilename, Level, Summary#summary.max_sqn], @@ -292,7 +307,9 @@ starting({sst_newlevelzero, RootPath, Filename, SlotCount, MaxSQN), ActualFilename = write_file(RootPath, Filename, SummaryBin, SlotsBin), - UpdState = read_file(ActualFilename, State#state{root_path=RootPath}), + UpdState = read_file(ActualFilename, + State#state{root_path = RootPath, + yield_blockquery = true}), Summary = UpdState#state.summary, leveled_log:log_timer("SST08", [ActualFilename, 0, Summary#summary.max_sqn], @@ -315,10 +332,26 @@ reader({get_kv, LedgerKey, Hash}, _From, State) -> UpdTimings = leveled_log:sst_timing(State#state.sst_timings, SW, Stage), {reply, Result, reader, UpdState#state{sst_timings = UpdTimings}}; reader({get_kvrange, StartKey, EndKey, ScanWidth}, _From, State) -> - {reply, - fetch_range(StartKey, EndKey, ScanWidth, State), - reader, - State}; + {SlotsToFetchBinList, SlotsToPoint} = fetch_range(StartKey, + EndKey, + ScanWidth, + State), + case State#state.yield_blockquery of + true -> + {reply, + {yield, SlotsToFetchBinList, SlotsToPoint}, + reader, + State}; + false -> + FetchFun = + fun({SlotBin, SK, EK}, Acc) -> + Acc ++ binaryslot_trimmedlist(SlotBin, SK, EK) + end, + {reply, + lists:foldl(FetchFun, [], SlotsToFetchBinList) ++ SlotsToPoint, + reader, + State} + end; reader({get_slots, SlotList}, _From, State) -> SlotBins = read_slots(State#state.handle, SlotList), {reply, SlotBins, reader, State}; @@ -353,8 +386,13 @@ delete_pending({get_kv, LedgerKey, Hash}, _From, State) -> {Result, _Stage, _SlotID, UpdState} = fetch(LedgerKey, Hash, State), {reply, Result, delete_pending, UpdState, ?DELETE_TIMEOUT}; delete_pending({get_kvrange, StartKey, EndKey, ScanWidth}, _From, State) -> + {SlotsToFetchBinList, SlotsToPoint} = fetch_range(StartKey, + EndKey, + ScanWidth, + State), + % Always yield as about to clear and de-reference {reply, - fetch_range(StartKey, EndKey, ScanWidth, State), + {yield, SlotsToFetchBinList, SlotsToPoint}, delete_pending, State, ?DELETE_TIMEOUT}; @@ -427,7 +465,7 @@ fetch(LedgerKey, Hash, State) -> slot_fetch, Slot#slot_index_value.slot_id, State#state{blockindex_cache = BlockIndexCache}}; - <> -> + <> -> PosList = find_pos(BlockIdx, double_hash(Hash, LedgerKey), [], @@ -730,43 +768,56 @@ generate_binary_slot(Lookup, KVL) -> {[], <<0:1/integer, 127:7/integer>>} end, - BlockSize = + {SideBlockSize, MidBlockSize} = case Lookup of lookup -> - ?SLOT_SIZE div 4; + ?LOOK_BLOCKSIZE; no_lookup -> - ?NOLOOK_SLOTSIZE div 4 + ?NOLOOK_BLOCKSIZE end, - - {B1, B2, B3, B4} = + {B1, B2, B3, B4, B5} = case length(KVL) of - L when L =< BlockSize -> + L when L =< SideBlockSize -> {term_to_binary(KVL, ?BINARY_SETTINGS), <<0:0>>, <<0:0>>, + <<0:0>>, <<0:0>>}; - L when L =< 2 * BlockSize -> - {KVLA, KVLB} = lists:split(BlockSize, KVL), + L when L =< 2 * SideBlockSize -> + {KVLA, KVLB} = lists:split(SideBlockSize, KVL), {term_to_binary(KVLA, ?BINARY_SETTINGS), term_to_binary(KVLB, ?BINARY_SETTINGS), <<0:0>>, + <<0:0>>, <<0:0>>}; - L when L =< 3 * BlockSize -> - {KVLA, KVLB_Rest} = lists:split(BlockSize, KVL), - {KVLB, KVLC} = lists:split(BlockSize, KVLB_Rest), + L when L =< (2 * SideBlockSize + MidBlockSize) -> + {KVLA, KVLB_Rest} = lists:split(SideBlockSize, KVL), + {KVLB, KVLC} = lists:split(SideBlockSize, KVLB_Rest), {term_to_binary(KVLA, ?BINARY_SETTINGS), term_to_binary(KVLB, ?BINARY_SETTINGS), term_to_binary(KVLC, ?BINARY_SETTINGS), + <<0:0>>, <<0:0>>}; - L when L =< 4 * BlockSize -> - {KVLA, KVLB_Rest} = lists:split(BlockSize, KVL), - {KVLB, KVLC_Rest} = lists:split(BlockSize, KVLB_Rest), - {KVLC, KVLD} = lists:split(BlockSize, KVLC_Rest), + L when L =< (3 * SideBlockSize + MidBlockSize) -> + {KVLA, KVLB_Rest} = lists:split(SideBlockSize, KVL), + {KVLB, KVLC_Rest} = lists:split(SideBlockSize, KVLB_Rest), + {KVLC, KVLD} = lists:split(MidBlockSize, KVLC_Rest), {term_to_binary(KVLA, ?BINARY_SETTINGS), term_to_binary(KVLB, ?BINARY_SETTINGS), term_to_binary(KVLC, ?BINARY_SETTINGS), - term_to_binary(KVLD, ?BINARY_SETTINGS)} + term_to_binary(KVLD, ?BINARY_SETTINGS), + <<0:0>>}; + L when L =< (4 * SideBlockSize + MidBlockSize) -> + {KVLA, KVLB_Rest} = lists:split(SideBlockSize, KVL), + {KVLB, KVLC_Rest} = lists:split(SideBlockSize, KVLB_Rest), + {KVLC, KVLD_Rest} = lists:split(MidBlockSize, KVLC_Rest), + {KVLD, KVLE} = lists:split(SideBlockSize, KVLD_Rest), + {term_to_binary(KVLA, ?BINARY_SETTINGS), + term_to_binary(KVLB, ?BINARY_SETTINGS), + term_to_binary(KVLC, ?BINARY_SETTINGS), + term_to_binary(KVLD, ?BINARY_SETTINGS), + term_to_binary(KVLE, ?BINARY_SETTINGS)} end, B1P = byte_size(PosBinIndex), @@ -774,14 +825,16 @@ generate_binary_slot(Lookup, KVL) -> B2L = byte_size(B2), B3L = byte_size(B3), B4L = byte_size(B4), + B5L = byte_size(B5), Lengths = <>, + B4L:32/integer, + B5L:32/integer>>, SlotBin = <>, + B1/binary, B2/binary, B3/binary, B4/binary, B5/binary>>, CRC32 = erlang:crc32(SlotBin), FullBin = <>, @@ -811,8 +864,8 @@ read_block(Handle, Slot, BlockLengths, BlockID) -> Slot#slot_index_value.start_position + BlockPos + Offset - + 24, - % 4-byte CRC, 4 byte pos, 4x4 byte lengths + + 28, + % 4-byte CRC, 4 byte pos, 5x4 byte lengths Length), BlockBin. @@ -894,9 +947,12 @@ binaryslot_tolist(FullBin) -> B1L:32/integer, B2L:32/integer, B3L:32/integer, - B4L:32/integer>> = BlockLengths, + B4L:32/integer, + B5L:32/integer>> = BlockLengths, <<_PosBinIndex:B1P/binary, Blocks/binary>> = RestBin, - lists:foldl(BlockFetchFun, {[], Blocks}, [B1L, B2L, B3L, B4L]); + lists:foldl(BlockFetchFun, + {[], Blocks}, + [B1L, B2L, B3L, B4L, B5L]); crc_wonky -> {[], <<>>} end, @@ -908,56 +964,103 @@ binaryslot_trimmedlist(FullBin, all, all) -> binaryslot_trimmedlist(FullBin, StartKey, EndKey) -> LTrimFun = fun({K, _V}) -> K < StartKey end, RTrimFun = fun({K, _V}) -> not leveled_codec:endkey_passed(EndKey, K) end, - BlockFetchFun = - fun(Length, {Acc, Bin, Continue}) -> - case {Length, Continue} of - {0, _} -> - {Acc, Bin, false}; - {_, true} -> - <> = Bin, - BlockList = binary_to_term(Block), - {LastKey, _LV} = lists:last(BlockList), - case StartKey > LastKey of - true -> - {Acc, Rest, true}; - false -> - {_LDrop, RKeep} = lists:splitwith(LTrimFun, - BlockList), - case leveled_codec:endkey_passed(EndKey, LastKey) of - true -> - {LKeep, _RDrop} = lists:splitwith(RTrimFun, RKeep), - {Acc ++ LKeep, Rest, false}; - false -> - {Acc ++ RKeep, Rest, true} - end - end; - {_ , false} -> - {Acc, Bin, false} - end - end, - - {Out, _Rem, _Continue} = + + % It will be more effecient to check a subset of blocks. To work out + % the best subset we always look in the middle block of 5, and based on + % the first and last keys of that middle block when compared to the Start + % and EndKey of the query determines a subset of blocks + % + % This isn't perfectly efficient, esepcially if the query overlaps Block2 + % and Block3 (as Block 1 will also be checked), but finessing this last + % scenario is hard to do in concise code + BlocksToCheck = case crc_check_slot(FullBin) of {BlockLengths, RestBin} -> <> = BlockLengths, - <<_PosBinIndex:B1P/binary, Blocks/binary>> = RestBin, - lists:foldl(BlockFetchFun, {[], Blocks, true}, [B1L, B2L, B3L, B4L]); + B4L:32/integer, + B5L:32/integer>> = BlockLengths, + <<_PosBinIndex:B1P/binary, + Block1:B1L/binary, Block2:B2L/binary, + MidBlock:B3L/binary, + Block4:B4L/binary, Block5:B5L/binary>> = RestBin, + case B3L of + 0 -> + [Block1, Block2]; + _ -> + MidBlockList = binary_to_term(MidBlock), + {MidFirst, _} = lists:nth(1, MidBlockList), + {MidLast, _} = lists:last(MidBlockList), + Split = {StartKey > MidLast, + StartKey >= MidFirst, + leveled_codec:endkey_passed(EndKey, + MidFirst), + leveled_codec:endkey_passed(EndKey, + MidLast)}, + case Split of + {true, _, _, _} -> + [Block4, Block5]; + {false, true, false, true} -> + [MidBlockList]; + {false, true, false, false} -> + [MidBlockList, Block4, Block5]; + {false, false, true, true} -> + [Block1, Block2]; + {false, false, false, true} -> + [Block1, Block2, MidBlockList]; + _ -> + [Block1, Block2, MidBlockList, Block4, Block5] + end + end; crc_wonky -> - {[], <<>>, true} + [] end, - Out. + + BlockCheckFun = + fun(Block, {Acc, Continue}) -> + case {Block, Continue} of + {<<>>, _} -> + {Acc, false}; + {_, true} -> + BlockList = + case is_binary(Block) of + true -> + binary_to_term(Block); + false -> + Block + end, + {LastKey, _LV} = lists:last(BlockList), + case StartKey > LastKey of + true -> + {Acc, true}; + false -> + {_LDrop, RKeep} = lists:splitwith(LTrimFun, + BlockList), + case leveled_codec:endkey_passed(EndKey, LastKey) of + true -> + {LKeep, _RDrop} = lists:splitwith(RTrimFun, RKeep), + {Acc ++ LKeep, false}; + false -> + {Acc ++ RKeep, true} + end + end; + {_ , false} -> + {Acc, false} + end + end, + + {Acc, _Continue} = lists:foldl(BlockCheckFun, {[], true}, BlocksToCheck), + Acc. crc_check_slot(FullBin) -> <> = FullBin, case erlang:crc32(SlotBin) of CRC32 -> - <> = SlotBin, + <> = SlotBin, {BlockLengths, Rest}; _ -> leveled_log:log("SST09", []), @@ -965,7 +1068,7 @@ crc_check_slot(FullBin) -> end. block_offsetandlength(BlockLengths, BlockID) -> - <> = BlockLengths, + <> = BlockLengths, case BlockID of 1 -> <> = BlockLengths0, @@ -983,8 +1086,17 @@ block_offsetandlength(BlockLengths, BlockID) -> <> = BlockLengths0, - {BlocksPos, B1L + B2L + B3L, B4L} + B4L:32/integer, + _BR/binary>> = BlockLengths0, + {BlocksPos, B1L + B2L + B3L, B4L}; + 5 -> + <> = BlockLengths0, + {BlocksPos, B1L + B2L + B3L + B4L, B5L} end. double_hash(Hash, Key) -> @@ -1004,15 +1116,26 @@ fetch_value([Pos|Rest], BlockLengths, Blocks, Key) -> case K of Key -> {K, V}; - _ -> + _ -> fetch_value(Rest, BlockLengths, Blocks, Key) end. revert_position(Pos) -> - BlockNumber = (Pos div 32) + 1, - BlockPos = (Pos rem 32) + 1, - {BlockNumber, BlockPos}. + {SideBlockSize, MidBlockSize} = ?LOOK_BLOCKSIZE, + case Pos < 2 * SideBlockSize of + true -> + {(Pos div SideBlockSize) + 1, (Pos rem SideBlockSize) + 1}; + false -> + case Pos < (2 * SideBlockSize + MidBlockSize) of + true -> + {3, ((Pos - 2 * SideBlockSize) rem MidBlockSize) + 1}; + false -> + TailPos = Pos - 2 * SideBlockSize - MidBlockSize, + {(TailPos div SideBlockSize) + 4, + (TailPos rem SideBlockSize) + 1} + end + end. find_pos(<<>>, _Hash, PosList, _Count) -> PosList; @@ -1059,7 +1182,7 @@ find_pos(<<0:1/integer, NHC:7/integer, T/binary>>, Hash, PosList, Count) -> %% any lower sequence numbers should be compacted out of existence merge_lists(KVList1) -> - SlotCount = length(KVList1) div ?SLOT_SIZE, + SlotCount = length(KVList1) div ?LOOK_SLOTSIZE, {[], [], split_lists(KVList1, [], SlotCount), @@ -1071,7 +1194,7 @@ split_lists(LastPuff, SlotLists, 0) -> SlotD = generate_binary_slot(lookup, LastPuff), lists:reverse([SlotD|SlotLists]); split_lists(KVList1, SlotLists, N) -> - {Slot, KVListRem} = lists:split(?SLOT_SIZE, KVList1), + {Slot, KVListRem} = lists:split(?LOOK_SLOTSIZE, KVList1), SlotD = generate_binary_slot(lookup, Slot), split_lists(KVListRem, [SlotD|SlotLists], N - 1). @@ -1105,7 +1228,8 @@ merge_lists(KVList1, KVList2, LI, SlotList, FirstKey, SlotCount) -> form_slot([], [], _LI, Type, _Size, Slot, FK) -> {[], [], {Type, lists:reverse(Slot)}, FK}; -form_slot(KVList1, KVList2, _LI, lookup, ?SLOT_SIZE, Slot, FK) -> + +form_slot(KVList1, KVList2, _LI, lookup, ?LOOK_SLOTSIZE, Slot, FK) -> {KVList1, KVList2, {lookup, lists:reverse(Slot)}, FK}; form_slot(KVList1, KVList2, _LI, no_lookup, ?NOLOOK_SLOTSIZE, Slot, FK) -> {KVList1, KVList2, {no_lookup, lists:reverse(Slot)}, FK}; @@ -1142,7 +1266,7 @@ form_slot(KVList1, KVList2, {IsBasement, TS}, no_lookup, Size, Slot, FK) -> [{TopK, TopV}|Slot], FK0); lookup -> - case Size >= ?SLOT_SIZE of + case Size >= ?LOOK_SLOTSIZE of true -> {KVList1, KVList2, @@ -1310,7 +1434,7 @@ form_slot_test() -> R1 = form_slot([SkippingKV], [], {true, 99999999}, no_lookup, - ?SLOT_SIZE + 1, + ?LOOK_SLOTSIZE + 1, Slot, {o, "B1", "K5", null}), ?assertMatch({[], [], {no_lookup, Slot}, {o, "B1", "K5", null}}, R1). @@ -1399,7 +1523,7 @@ indexed_list_mixedkeys2_test() -> indexed_list_allindexkeys_test() -> Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(150)), 128), {PosBinIndex1, FullBin, _HL, _LK} = generate_binary_slot(lookup, Keys), - ?assertMatch(<<_BL:20/binary, 127:8/integer>>, PosBinIndex1), + ?assertMatch(<<_BL:24/binary, 127:8/integer>>, PosBinIndex1), % SW = os:timestamp(), BinToList = binaryslot_tolist(FullBin), % io:format(user, @@ -1410,9 +1534,9 @@ indexed_list_allindexkeys_test() -> indexed_list_allindexkeys_nolookup_test() -> Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(1000)), - 128 * ?NOLOOK_MULT), + ?NOLOOK_SLOTSIZE), {PosBinIndex1, FullBin, _HL, _LK} = generate_binary_slot(no_lookup, Keys), - ?assertMatch(<<_BL:20/binary, 127:8/integer>>, PosBinIndex1), + ?assertMatch(<<_BL:24/binary, 127:8/integer>>, PosBinIndex1), % SW = os:timestamp(), BinToList = binaryslot_tolist(FullBin), % io:format(user, @@ -1424,7 +1548,7 @@ indexed_list_allindexkeys_nolookup_test() -> indexed_list_allindexkeys_trimmed_test() -> Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(150)), 128), {PosBinIndex1, FullBin, _HL, _LK} = generate_binary_slot(lookup, Keys), - ?assertMatch(<<_BL:20/binary, 127:8/integer>>, PosBinIndex1), + ?assertMatch(<<_BL:24/binary, 127:8/integer>>, PosBinIndex1), ?assertMatch(Keys, binaryslot_trimmedlist(FullBin, {i, "Bucket", @@ -1546,7 +1670,7 @@ merge_test() -> simple_persisted_range_test() -> {RP, Filename} = {"../test/", "simple_test"}, - KVList0 = generate_randomkeys(1, ?SLOT_SIZE * 16, 1, 20), + KVList0 = generate_randomkeys(1, ?LOOK_SLOTSIZE * 16, 1, 20), KVList1 = lists:ukeysort(1, KVList0), [{FirstKey, _FV}|_Rest] = KVList1, {LastKey, _LV} = lists:last(KVList1), @@ -1659,8 +1783,9 @@ additional_range_test() -> simple_persisted_slotsize_test() -> {RP, Filename} = {"../test/", "simple_slotsize_test"}, - KVList0 = generate_randomkeys(1, ?SLOT_SIZE * 2, 1, 20), - KVList1 = lists:sublist(lists:ukeysort(1, KVList0), ?SLOT_SIZE), + KVList0 = generate_randomkeys(1, ?LOOK_SLOTSIZE * 2, 1, 20), + KVList1 = lists:sublist(lists:ukeysort(1, KVList0), + ?LOOK_SLOTSIZE), [{FirstKey, _FV}|_Rest] = KVList1, {LastKey, _LV} = lists:last(KVList1), {ok, Pid, {FirstKey, LastKey}} = sst_new(RP, @@ -1677,7 +1802,7 @@ simple_persisted_slotsize_test() -> simple_persisted_test() -> {RP, Filename} = {"../test/", "simple_test"}, - KVList0 = generate_randomkeys(1, ?SLOT_SIZE * 32, 1, 20), + KVList0 = generate_randomkeys(1, ?LOOK_SLOTSIZE * 32, 1, 20), KVList1 = lists:ukeysort(1, KVList0), [{FirstKey, _FV}|_Rest] = KVList1, {LastKey, _LV} = lists:last(KVList1), @@ -1706,7 +1831,7 @@ simple_persisted_test() -> ++ "microseconds~n", [length(KVList1), timer:now_diff(os:timestamp(), SW1)]), ok = sst_printtimings(Pid), - KVList2 = generate_randomkeys(1, ?SLOT_SIZE * 32, 1, 20), + KVList2 = generate_randomkeys(1, ?LOOK_SLOTSIZE * 32, 1, 20), MapFun = fun({K, V}, Acc) -> In = lists:keymember(K, 1, KVList1), @@ -1731,8 +1856,6 @@ simple_persisted_test() -> FoldFun = fun(X, Acc) -> case X of {pointer, P, S, SK, EK} -> - io:format("Get slot ~w with Acc at ~w~n", - [S, length(Acc)]), Acc ++ sst_getslots(P, [{pointer, P, S, SK, EK}]); _ -> Acc ++ [X] diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index 9e7f758..e53eb4f 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -262,7 +262,7 @@ check_forobject(Bookie, TestObject) -> Vclock, _Hash, size} = leveled_codec:riak_extract_metadata(HeadBinary, size), - true = binary_to_term(Vclock) == TestObject#r_object.vclock. + true = Vclock == TestObject#r_object.vclock. check_formissingobject(Bookie, Bucket, Key) -> not_found = book_riakget(Bookie, Bucket, Key),