From 1e40648c83fed054908a2edf53cd88a21c85e821 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 15 May 2018 10:11:37 +0100 Subject: [PATCH 01/12] Add initial log --- src/leveled_penciller.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index f007084..b6d03f8 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -677,6 +677,7 @@ handle_call({fetch_keys, [State#state.levelzero_size], SW, 0.01), + io:format("Fetch keys with segment_list of ~w~n", [SegmentList]), SetupFoldFun = fun(Level, Acc) -> Pointers = leveled_pmanifest:range_lookup(State#state.manifest, From 6160db3377e502f89e16fdac80bb5ab281e25113 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 15 May 2018 10:30:09 +0100 Subject: [PATCH 02/12] Further log --- src/leveled_sst.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index fd0638e..3956e51 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -403,6 +403,7 @@ sst_getslots(Pid, SlotList) -> %% false as a SegList to not filter sst_getfilteredslots(Pid, SlotList, SegList) -> SegL0 = tune_seglist(SegList), + io:format("Tuned seglist of ~w~n", [SegL0]), {SlotBins, PressMethod} = gen_fsm:sync_send_event(Pid, {get_slots, SlotList, SegL0}, infinity), binaryslot_reader(SlotBins, PressMethod). From 15a5db505548f5dd209b3807af4965983ee12d3f Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 15 May 2018 10:36:07 +0100 Subject: [PATCH 03/12] Sort tuned seg list --- src/leveled_sst.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index 3956e51..cbb9396 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -1630,7 +1630,7 @@ tune_hash(SegHash) -> tune_seglist(SegList) -> case is_list(SegList) of true -> - lists:map(fun tune_hash/1, SegList); + lists:usort(lists:map(fun tune_hash/1, SegList)); false -> SegList end. From 3d1c085af3b6c5581d4158b40e0910aafeb1d9ee Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 15 May 2018 10:41:56 +0100 Subject: [PATCH 04/12] Revert log additions --- src/leveled_penciller.erl | 1 - src/leveled_sst.erl | 1 - 2 files changed, 2 deletions(-) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index b6d03f8..f007084 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -677,7 +677,6 @@ handle_call({fetch_keys, [State#state.levelzero_size], SW, 0.01), - io:format("Fetch keys with segment_list of ~w~n", [SegmentList]), SetupFoldFun = fun(Level, Acc) -> Pointers = leveled_pmanifest:range_lookup(State#state.manifest, diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index cbb9396..22fbd0a 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -403,7 +403,6 @@ sst_getslots(Pid, SlotList) -> %% false as a SegList to not filter sst_getfilteredslots(Pid, SlotList, SegList) -> SegL0 = tune_seglist(SegList), - io:format("Tuned seglist of ~w~n", [SegL0]), {SlotBins, PressMethod} = gen_fsm:sync_send_event(Pid, {get_slots, SlotList, SegL0}, infinity), binaryslot_reader(SlotBins, PressMethod). From 06fd9856b54f5c25c3a79fe9146600ae1503ad85 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 16 May 2018 10:34:47 +0100 Subject: [PATCH 05/12] Add debug log --- src/leveled_runner.erl | 2 +- src/leveled_sst.erl | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/leveled_runner.erl b/src/leveled_runner.erl index c9dc03a..37f7163 100644 --- a/src/leveled_runner.erl +++ b/src/leveled_runner.erl @@ -478,7 +478,7 @@ foldobjects(SnapFun, Tag, KeyRanges, FoldObjFun, DeferredFetch, SegmentList) -> % initial accumulator FoldObjFun; false -> - % no initial accumulatr passed, and so should be just a list + % no initial accumulator passed, and so should be just a list {FoldObjFun, []} end, diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index 22fbd0a..53f1bd0 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -1364,6 +1364,7 @@ read_slots(Handle, SlotList, {SegList, BlockIndexCache}, PressMethod) -> BL = ?BLOCK_LENGTHS_LENGTH, case array:get(ID - 1, BlockIndexCache) of none -> + io:format("BlockIndex cache not available for fetch_range~n"), % If there is an attempt to use the seg list query and the % index block cache isn't cached for any part this may be % slower as each slot will be read in turn @@ -1378,6 +1379,7 @@ read_slots(Handle, SlotList, {SegList, BlockIndexCache}, PressMethod) -> % present without lifting the slot off disk. Also the % fact that we know position can be used to filter out % other keys + io:format("BlockIndex cache used in fetch_range~n"), case find_pos(BlockIdx, SegList, [], 0) of [] -> Acc; From 8b4adcccaf193f2802830d398f46e0cdbcebf9c6 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 16 May 2018 10:51:59 +0100 Subject: [PATCH 06/12] Debug logs --- src/leveled_sst.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index 53f1bd0..74258a7 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -369,6 +369,7 @@ sst_getkvrange(Pid, StartKey, EndKey, ScanWidth) -> %% leveled_tictac sst_getfilteredrange(Pid, StartKey, EndKey, ScanWidth, SegList) -> SegList0 = tune_seglist(SegList), + io:format("Using tuned seglist ~w~n", [SegList0]), case gen_fsm:sync_send_event(Pid, {get_kvrange, StartKey, EndKey, @@ -1364,7 +1365,6 @@ read_slots(Handle, SlotList, {SegList, BlockIndexCache}, PressMethod) -> BL = ?BLOCK_LENGTHS_LENGTH, case array:get(ID - 1, BlockIndexCache) of none -> - io:format("BlockIndex cache not available for fetch_range~n"), % If there is an attempt to use the seg list query and the % index block cache isn't cached for any part this may be % slower as each slot will be read in turn @@ -1379,7 +1379,6 @@ read_slots(Handle, SlotList, {SegList, BlockIndexCache}, PressMethod) -> % present without lifting the slot off disk. Also the % fact that we know position can be used to filter out % other keys - io:format("BlockIndex cache used in fetch_range~n"), case find_pos(BlockIdx, SegList, [], 0) of [] -> Acc; @@ -1683,6 +1682,7 @@ find_pos(<<1:1/integer, PotentialHit:15/integer, T/binary>>, HashList, PosList, Count) when is_list(HashList) -> case lists:member(PotentialHit, HashList) of true -> + io:format("Found pos based on ~w~n", [PotentialHit]), find_pos(T, HashList, PosList ++ [Count], Count + 1); false -> find_pos(T, HashList, PosList, Count + 1) From a7cda7213f7cf5475bb9993b8d311c898a0a8004 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 16 May 2018 11:06:51 +0100 Subject: [PATCH 07/12] Debug logs --- src/leveled_sst.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index 74258a7..09de726 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -1381,8 +1381,11 @@ read_slots(Handle, SlotList, {SegList, BlockIndexCache}, PressMethod) -> % other keys case find_pos(BlockIdx, SegList, [], 0) of [] -> + io:format("Empty postion list for slot~n"), Acc; PositionList -> + io:format("~w positions found for slot~n", + [length(PositionList)]), Acc ++ check_blocks(PositionList, Handle, SP, From 2cc088fc646e2627b43e738915f1c0c145aa11f7 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 16 May 2018 11:24:57 +0100 Subject: [PATCH 08/12] More debug --- src/leveled_sst.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index 09de726..9ad4cc7 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -1278,6 +1278,7 @@ check_blocks([Pos|Rest], Handle, StartPos, BlockLengths, PosBinLength, _ -> case LedgerKeyToCheck of false -> + io:format("{K, V} found in block of ~w~n", [{K, V}]), Acc ++ [{K, V}]; _ -> check_blocks(Rest, Handle, StartPos, From 5414e18047f3a6c064c2944109ea1c7caf1c0612 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 16 May 2018 11:38:26 +0100 Subject: [PATCH 09/12] Accumulate keys in check_blocks Previously couldn't accumulate keys using check-blocks - so if a key was found in the first position, and there were other positions to check for other keys, those other positions wouldn't be checked. --- src/leveled_sst.erl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index 9ad4cc7..0ac2dec 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -369,7 +369,6 @@ sst_getkvrange(Pid, StartKey, EndKey, ScanWidth) -> %% leveled_tictac sst_getfilteredrange(Pid, StartKey, EndKey, ScanWidth, SegList) -> SegList0 = tune_seglist(SegList), - io:format("Using tuned seglist ~w~n", [SegList0]), case gen_fsm:sync_send_event(Pid, {get_kvrange, StartKey, EndKey, @@ -1258,9 +1257,12 @@ generate_binary_slot(Lookup, KVL, PressMethod, BuildTimings0) -> % Acc should start as not_present if LedgerKey is a key, and a list if % LedgerKey is false +check_blocks([], _Handle, _StartPos, _BlockLengths, _PosBinLength, + _LedgerKeyToCheck, _PressMethod, Acc) when is_list(Acc) -> + lists:reverse(Acc); check_blocks([], _Handle, _StartPos, _BlockLengths, _PosBinLength, _LedgerKeyToCheck, _PressMethod, Acc) -> - Acc; + Acc; % Should be not_present only? check_blocks([Pos|Rest], Handle, StartPos, BlockLengths, PosBinLength, LedgerKeyToCheck, PressMethod, Acc) -> {BlockNumber, BlockPos} = revert_position(Pos), @@ -1278,8 +1280,10 @@ check_blocks([Pos|Rest], Handle, StartPos, BlockLengths, PosBinLength, _ -> case LedgerKeyToCheck of false -> - io:format("{K, V} found in block of ~w~n", [{K, V}]), - Acc ++ [{K, V}]; + check_blocks(Rest, Handle, StartPos, + BlockLengths, PosBinLength, + LedgerKeyToCheck, PressMethod, + [{K, V}|Acc]); _ -> check_blocks(Rest, Handle, StartPos, BlockLengths, PosBinLength, @@ -1382,11 +1386,8 @@ read_slots(Handle, SlotList, {SegList, BlockIndexCache}, PressMethod) -> % other keys case find_pos(BlockIdx, SegList, [], 0) of [] -> - io:format("Empty postion list for slot~n"), Acc; PositionList -> - io:format("~w positions found for slot~n", - [length(PositionList)]), Acc ++ check_blocks(PositionList, Handle, SP, @@ -1686,7 +1687,6 @@ find_pos(<<1:1/integer, PotentialHit:15/integer, T/binary>>, HashList, PosList, Count) when is_list(HashList) -> case lists:member(PotentialHit, HashList) of true -> - io:format("Found pos based on ~w~n", [PotentialHit]), find_pos(T, HashList, PosList ++ [Count], Count + 1); false -> find_pos(T, HashList, PosList, Count + 1) From 9cc27469c10131cf8ea0c66198db3992f9972b33 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 16 May 2018 15:11:08 +0100 Subject: [PATCH 10/12] Add test to invoke segment filter issue there is an issue if you have a segment filter, and there is more than one match in the slot. --- test/end_to_end/riak_SUITE.erl | 48 ++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/test/end_to_end/riak_SUITE.erl b/test/end_to_end/riak_SUITE.erl index 1891412..fff62fe 100644 --- a/test/end_to_end/riak_SUITE.erl +++ b/test/end_to_end/riak_SUITE.erl @@ -12,8 +12,8 @@ all() -> [ crossbucket_aae, handoff, - dollar_bucket_index, - dollar_key_index + dollar_bucket_index, + dollar_key_index ]. -define(MAGIC, 53). % riak_kv -> riak_object @@ -80,6 +80,50 @@ crossbucket_aae(_Config) -> {ok, Bookie2A} = leveled_bookie:book_start(StartOpts2), test_singledelta_stores(Bookie2A, Bookie3, small, {B1, K1}), + SW0 = os:timestamp(), + SliceSize = 20, + + CL1 = lists:sublist(lists:nth(1, CLs), 100, SliceSize), + CL2 = lists:sublist(lists:nth(2, CLs), 100, SliceSize), + CL3 = lists:sublist(lists:nth(3, CLs), 100, SliceSize), + CL4 = lists:sublist(lists:nth(4, CLs), 100, SliceSize), + + SegMapFun = + fun({_RN, RiakObject, _Spc}) -> + B = RiakObject#r_object.bucket, + K = RiakObject#r_object.key, + leveled_tictac:keyto_segment32(<>) + end, + SL1 = lists:map(SegMapFun, CL1), + SL2 = lists:map(SegMapFun, CL2), + SL3 = lists:map(SegMapFun, CL3), + SL4 = lists:map(SegMapFun, CL4), + + HeadSegmentFolderGen = + fun(SegL) -> + {foldheads_allkeys, + ?RIAK_TAG, + {fun(_B, _K, _PO, Acc) -> Acc + 1 end, 0}, + false, true, SegL} + end, + + {async, SL1Folder} = + leveled_bookie:book_returnfolder(Bookie3, HeadSegmentFolderGen(SL1)), + {async, SL2Folder} = + leveled_bookie:book_returnfolder(Bookie3, HeadSegmentFolderGen(SL2)), + {async, SL3Folder} = + leveled_bookie:book_returnfolder(Bookie3, HeadSegmentFolderGen(SL3)), + {async, SL4Folder} = + leveled_bookie:book_returnfolder(Bookie3, HeadSegmentFolderGen(SL4)), + + Results = [SL1Folder(), SL2Folder(), SL3Folder(), SL4Folder()], + lists:foreach(fun(R) -> true = R >= SliceSize end, Results), + + io:format("SegList folders returned results of ~w " ++ + "for SliceSize ~w in ~w ms~n", + [Results, SliceSize, + timer:now_diff(os:timestamp(), SW0)/1000]), + ok = leveled_bookie:book_close(Bookie2A), ok = leveled_bookie:book_close(Bookie3). From cbf6e26fc8ec123f4434957ef8a5565e2ec0f817 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 16 May 2018 15:11:18 +0100 Subject: [PATCH 11/12] Revert "Accumulate keys in check_blocks" This reverts commit 5414e18047f3a6c064c2944109ea1c7caf1c0612. --- src/leveled_sst.erl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index 0ac2dec..9ad4cc7 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -369,6 +369,7 @@ sst_getkvrange(Pid, StartKey, EndKey, ScanWidth) -> %% leveled_tictac sst_getfilteredrange(Pid, StartKey, EndKey, ScanWidth, SegList) -> SegList0 = tune_seglist(SegList), + io:format("Using tuned seglist ~w~n", [SegList0]), case gen_fsm:sync_send_event(Pid, {get_kvrange, StartKey, EndKey, @@ -1257,12 +1258,9 @@ generate_binary_slot(Lookup, KVL, PressMethod, BuildTimings0) -> % Acc should start as not_present if LedgerKey is a key, and a list if % LedgerKey is false -check_blocks([], _Handle, _StartPos, _BlockLengths, _PosBinLength, - _LedgerKeyToCheck, _PressMethod, Acc) when is_list(Acc) -> - lists:reverse(Acc); check_blocks([], _Handle, _StartPos, _BlockLengths, _PosBinLength, _LedgerKeyToCheck, _PressMethod, Acc) -> - Acc; % Should be not_present only? + Acc; check_blocks([Pos|Rest], Handle, StartPos, BlockLengths, PosBinLength, LedgerKeyToCheck, PressMethod, Acc) -> {BlockNumber, BlockPos} = revert_position(Pos), @@ -1280,10 +1278,8 @@ check_blocks([Pos|Rest], Handle, StartPos, BlockLengths, PosBinLength, _ -> case LedgerKeyToCheck of false -> - check_blocks(Rest, Handle, StartPos, - BlockLengths, PosBinLength, - LedgerKeyToCheck, PressMethod, - [{K, V}|Acc]); + io:format("{K, V} found in block of ~w~n", [{K, V}]), + Acc ++ [{K, V}]; _ -> check_blocks(Rest, Handle, StartPos, BlockLengths, PosBinLength, @@ -1386,8 +1382,11 @@ read_slots(Handle, SlotList, {SegList, BlockIndexCache}, PressMethod) -> % other keys case find_pos(BlockIdx, SegList, [], 0) of [] -> + io:format("Empty postion list for slot~n"), Acc; PositionList -> + io:format("~w positions found for slot~n", + [length(PositionList)]), Acc ++ check_blocks(PositionList, Handle, SP, @@ -1687,6 +1686,7 @@ find_pos(<<1:1/integer, PotentialHit:15/integer, T/binary>>, HashList, PosList, Count) when is_list(HashList) -> case lists:member(PotentialHit, HashList) of true -> + io:format("Found pos based on ~w~n", [PotentialHit]), find_pos(T, HashList, PosList ++ [Count], Count + 1); false -> find_pos(T, HashList, PosList, Count + 1) From 18aabb49babdf94b732a2a756b2deb08702ba753 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 16 May 2018 17:24:23 +0100 Subject: [PATCH 12/12] Segment filter and multiple keys in slot An issue was spotted. If we use a segment filter in a query, and there are multiple matches within a given slot - only the first match is returned. Tests didn't detect this. Now they do, and the issue is resolved. --- src/leveled_sst.erl | 18 ++++---- test/end_to_end/riak_SUITE.erl | 84 +++++++++++++++++++++++----------- 2 files changed, 67 insertions(+), 35 deletions(-) diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index 9ad4cc7..ba6ec2e 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -369,7 +369,6 @@ sst_getkvrange(Pid, StartKey, EndKey, ScanWidth) -> %% leveled_tictac sst_getfilteredrange(Pid, StartKey, EndKey, ScanWidth, SegList) -> SegList0 = tune_seglist(SegList), - io:format("Using tuned seglist ~w~n", [SegList0]), case gen_fsm:sync_send_event(Pid, {get_kvrange, StartKey, EndKey, @@ -1260,7 +1259,12 @@ generate_binary_slot(Lookup, KVL, PressMethod, BuildTimings0) -> check_blocks([], _Handle, _StartPos, _BlockLengths, _PosBinLength, _LedgerKeyToCheck, _PressMethod, Acc) -> - Acc; + case is_list(Acc) of + true -> + lists:reverse(Acc); + false -> + Acc + end; check_blocks([Pos|Rest], Handle, StartPos, BlockLengths, PosBinLength, LedgerKeyToCheck, PressMethod, Acc) -> {BlockNumber, BlockPos} = revert_position(Pos), @@ -1278,8 +1282,10 @@ check_blocks([Pos|Rest], Handle, StartPos, BlockLengths, PosBinLength, _ -> case LedgerKeyToCheck of false -> - io:format("{K, V} found in block of ~w~n", [{K, V}]), - Acc ++ [{K, V}]; + check_blocks(Rest, Handle, StartPos, + BlockLengths, PosBinLength, + LedgerKeyToCheck, PressMethod, + [{K, V}|Acc]); _ -> check_blocks(Rest, Handle, StartPos, BlockLengths, PosBinLength, @@ -1382,11 +1388,8 @@ read_slots(Handle, SlotList, {SegList, BlockIndexCache}, PressMethod) -> % other keys case find_pos(BlockIdx, SegList, [], 0) of [] -> - io:format("Empty postion list for slot~n"), Acc; PositionList -> - io:format("~w positions found for slot~n", - [length(PositionList)]), Acc ++ check_blocks(PositionList, Handle, SP, @@ -1686,7 +1689,6 @@ find_pos(<<1:1/integer, PotentialHit:15/integer, T/binary>>, HashList, PosList, Count) when is_list(HashList) -> case lists:member(PotentialHit, HashList) of true -> - io:format("Found pos based on ~w~n", [PotentialHit]), find_pos(T, HashList, PosList ++ [Count], Count + 1); false -> find_pos(T, HashList, PosList, Count + 1) diff --git a/test/end_to_end/riak_SUITE.erl b/test/end_to_end/riak_SUITE.erl index fff62fe..b2bbdc1 100644 --- a/test/end_to_end/riak_SUITE.erl +++ b/test/end_to_end/riak_SUITE.erl @@ -61,25 +61,11 @@ crossbucket_aae(_Config) -> %% Check all the objects are found - used to trigger HEAD performance log ok = testutil:checkhead_forlist(Bookie2, lists:nth(1, CLs)), - % Start a new store, and load the same objects (except fot the original - % test object) into this store - - StartOpts3 = [{root_path, RootPathB}, - {max_journalsize, 200000000}, - {max_pencillercachesize, 16000}, - {sync_strategy, testutil:sync_strategy()}], - {ok, Bookie3} = leveled_bookie:book_start(StartOpts3), - lists:foreach(fun(ObjL) -> testutil:riakload(Bookie3, ObjL) end, CLs), - test_singledelta_stores(Bookie2, Bookie3, small, {B1, K1}), - test_singledelta_stores(Bookie2, Bookie3, medium, {B1, K1}), - test_singledelta_stores(Bookie2, Bookie3, xsmall, {B1, K1}), - test_singledelta_stores(Bookie2, Bookie3, xxsmall, {B1, K1}), - - % Test with a newly opend book (i.e with no blovk indexes cached) - ok = leveled_bookie:book_close(Bookie2), - {ok, Bookie2A} = leveled_bookie:book_start(StartOpts2), - test_singledelta_stores(Bookie2A, Bookie3, small, {B1, K1}), - + % This part of the test tests an issue with accelerating folds by segment + % list, when there is more than one key with a matching segment in the + % slot. Previously this was not handled correctly - and this test part + % of the test detects this, by finding slices of keys which are probably + % in the same slot SW0 = os:timestamp(), SliceSize = 20, @@ -94,35 +80,78 @@ crossbucket_aae(_Config) -> K = RiakObject#r_object.key, leveled_tictac:keyto_segment32(<>) end, + BKMapFun = + fun({_RN, RiakObject, _Spc}) -> + B = RiakObject#r_object.bucket, + K = RiakObject#r_object.key, + {B, K} + end, + SL1 = lists:map(SegMapFun, CL1), SL2 = lists:map(SegMapFun, CL2), SL3 = lists:map(SegMapFun, CL3), SL4 = lists:map(SegMapFun, CL4), + BK1 = lists:map(BKMapFun, CL1), + BK2 = lists:map(BKMapFun, CL2), + BK3 = lists:map(BKMapFun, CL3), + BK4 = lists:map(BKMapFun, CL4), + HeadSegmentFolderGen = - fun(SegL) -> + fun(SegL, BKL) -> {foldheads_allkeys, ?RIAK_TAG, - {fun(_B, _K, _PO, Acc) -> Acc + 1 end, 0}, + {fun(B, K, _PO, Acc) -> + case lists:member({B, K}, BKL) of + true -> + Acc + 1; + false -> + Acc + end + end, 0}, false, true, SegL} end, {async, SL1Folder} = - leveled_bookie:book_returnfolder(Bookie3, HeadSegmentFolderGen(SL1)), + leveled_bookie:book_returnfolder(Bookie2, + HeadSegmentFolderGen(SL1, BK1)), {async, SL2Folder} = - leveled_bookie:book_returnfolder(Bookie3, HeadSegmentFolderGen(SL2)), + leveled_bookie:book_returnfolder(Bookie2, + HeadSegmentFolderGen(SL2, BK2)), {async, SL3Folder} = - leveled_bookie:book_returnfolder(Bookie3, HeadSegmentFolderGen(SL3)), + leveled_bookie:book_returnfolder(Bookie2, + HeadSegmentFolderGen(SL3, BK3)), {async, SL4Folder} = - leveled_bookie:book_returnfolder(Bookie3, HeadSegmentFolderGen(SL4)), + leveled_bookie:book_returnfolder(Bookie2, + HeadSegmentFolderGen(SL4, BK4)), Results = [SL1Folder(), SL2Folder(), SL3Folder(), SL4Folder()], - lists:foreach(fun(R) -> true = R >= SliceSize end, Results), - io:format("SegList folders returned results of ~w " ++ "for SliceSize ~w in ~w ms~n", [Results, SliceSize, timer:now_diff(os:timestamp(), SW0)/1000]), + lists:foreach(fun(R) -> true = R == SliceSize end, Results), + + % Start a new store, and load the same objects (except fot the original + % test object) into this store + % + % This is now the comparison part of the test + + StartOpts3 = [{root_path, RootPathB}, + {max_journalsize, 200000000}, + {max_pencillercachesize, 16000}, + {sync_strategy, testutil:sync_strategy()}], + {ok, Bookie3} = leveled_bookie:book_start(StartOpts3), + lists:foreach(fun(ObjL) -> testutil:riakload(Bookie3, ObjL) end, CLs), + test_singledelta_stores(Bookie2, Bookie3, small, {B1, K1}), + test_singledelta_stores(Bookie2, Bookie3, medium, {B1, K1}), + test_singledelta_stores(Bookie2, Bookie3, xsmall, {B1, K1}), + test_singledelta_stores(Bookie2, Bookie3, xxsmall, {B1, K1}), + + % Test with a newly opend book (i.e with no block indexes cached) + ok = leveled_bookie:book_close(Bookie2), + {ok, Bookie2A} = leveled_bookie:book_start(StartOpts2), + test_singledelta_stores(Bookie2A, Bookie3, small, {B1, K1}), ok = leveled_bookie:book_close(Bookie2A), ok = leveled_bookie:book_close(Bookie3). @@ -551,3 +580,4 @@ dollar_bucket_index(_Config) -> ok = leveled_bookie:book_close(Bookie1), testutil:reset_filestructure(). +