diff --git a/src/leveled_runner.erl b/src/leveled_runner.erl index c9dc03a..37f7163 100644 --- a/src/leveled_runner.erl +++ b/src/leveled_runner.erl @@ -478,7 +478,7 @@ foldobjects(SnapFun, Tag, KeyRanges, FoldObjFun, DeferredFetch, SegmentList) -> % initial accumulator FoldObjFun; false -> - % no initial accumulatr passed, and so should be just a list + % no initial accumulator passed, and so should be just a list {FoldObjFun, []} end, diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index fd0638e..ba6ec2e 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -1259,7 +1259,12 @@ generate_binary_slot(Lookup, KVL, PressMethod, BuildTimings0) -> check_blocks([], _Handle, _StartPos, _BlockLengths, _PosBinLength, _LedgerKeyToCheck, _PressMethod, Acc) -> - Acc; + case is_list(Acc) of + true -> + lists:reverse(Acc); + false -> + Acc + end; check_blocks([Pos|Rest], Handle, StartPos, BlockLengths, PosBinLength, LedgerKeyToCheck, PressMethod, Acc) -> {BlockNumber, BlockPos} = revert_position(Pos), @@ -1277,7 +1282,10 @@ check_blocks([Pos|Rest], Handle, StartPos, BlockLengths, PosBinLength, _ -> case LedgerKeyToCheck of false -> - Acc ++ [{K, V}]; + check_blocks(Rest, Handle, StartPos, + BlockLengths, PosBinLength, + LedgerKeyToCheck, PressMethod, + [{K, V}|Acc]); _ -> check_blocks(Rest, Handle, StartPos, BlockLengths, PosBinLength, @@ -1629,7 +1637,7 @@ tune_hash(SegHash) -> tune_seglist(SegList) -> case is_list(SegList) of true -> - lists:map(fun tune_hash/1, SegList); + lists:usort(lists:map(fun tune_hash/1, SegList)); false -> SegList end. diff --git a/test/end_to_end/riak_SUITE.erl b/test/end_to_end/riak_SUITE.erl index 1891412..b2bbdc1 100644 --- a/test/end_to_end/riak_SUITE.erl +++ b/test/end_to_end/riak_SUITE.erl @@ -12,8 +12,8 @@ all() -> [ crossbucket_aae, handoff, - dollar_bucket_index, - dollar_key_index + dollar_bucket_index, + dollar_key_index ]. -define(MAGIC, 53). % riak_kv -> riak_object @@ -61,8 +61,81 @@ crossbucket_aae(_Config) -> %% Check all the objects are found - used to trigger HEAD performance log ok = testutil:checkhead_forlist(Bookie2, lists:nth(1, CLs)), + % This part of the test tests an issue with accelerating folds by segment + % list, when there is more than one key with a matching segment in the + % slot. Previously this was not handled correctly - and this test part + % of the test detects this, by finding slices of keys which are probably + % in the same slot + SW0 = os:timestamp(), + SliceSize = 20, + + CL1 = lists:sublist(lists:nth(1, CLs), 100, SliceSize), + CL2 = lists:sublist(lists:nth(2, CLs), 100, SliceSize), + CL3 = lists:sublist(lists:nth(3, CLs), 100, SliceSize), + CL4 = lists:sublist(lists:nth(4, CLs), 100, SliceSize), + + SegMapFun = + fun({_RN, RiakObject, _Spc}) -> + B = RiakObject#r_object.bucket, + K = RiakObject#r_object.key, + leveled_tictac:keyto_segment32(<>) + end, + BKMapFun = + fun({_RN, RiakObject, _Spc}) -> + B = RiakObject#r_object.bucket, + K = RiakObject#r_object.key, + {B, K} + end, + + SL1 = lists:map(SegMapFun, CL1), + SL2 = lists:map(SegMapFun, CL2), + SL3 = lists:map(SegMapFun, CL3), + SL4 = lists:map(SegMapFun, CL4), + + BK1 = lists:map(BKMapFun, CL1), + BK2 = lists:map(BKMapFun, CL2), + BK3 = lists:map(BKMapFun, CL3), + BK4 = lists:map(BKMapFun, CL4), + + HeadSegmentFolderGen = + fun(SegL, BKL) -> + {foldheads_allkeys, + ?RIAK_TAG, + {fun(B, K, _PO, Acc) -> + case lists:member({B, K}, BKL) of + true -> + Acc + 1; + false -> + Acc + end + end, 0}, + false, true, SegL} + end, + + {async, SL1Folder} = + leveled_bookie:book_returnfolder(Bookie2, + HeadSegmentFolderGen(SL1, BK1)), + {async, SL2Folder} = + leveled_bookie:book_returnfolder(Bookie2, + HeadSegmentFolderGen(SL2, BK2)), + {async, SL3Folder} = + leveled_bookie:book_returnfolder(Bookie2, + HeadSegmentFolderGen(SL3, BK3)), + {async, SL4Folder} = + leveled_bookie:book_returnfolder(Bookie2, + HeadSegmentFolderGen(SL4, BK4)), + + Results = [SL1Folder(), SL2Folder(), SL3Folder(), SL4Folder()], + io:format("SegList folders returned results of ~w " ++ + "for SliceSize ~w in ~w ms~n", + [Results, SliceSize, + timer:now_diff(os:timestamp(), SW0)/1000]), + lists:foreach(fun(R) -> true = R == SliceSize end, Results), + % Start a new store, and load the same objects (except fot the original % test object) into this store + % + % This is now the comparison part of the test StartOpts3 = [{root_path, RootPathB}, {max_journalsize, 200000000}, @@ -74,12 +147,12 @@ crossbucket_aae(_Config) -> test_singledelta_stores(Bookie2, Bookie3, medium, {B1, K1}), test_singledelta_stores(Bookie2, Bookie3, xsmall, {B1, K1}), test_singledelta_stores(Bookie2, Bookie3, xxsmall, {B1, K1}), - - % Test with a newly opend book (i.e with no blovk indexes cached) + + % Test with a newly opend book (i.e with no block indexes cached) ok = leveled_bookie:book_close(Bookie2), {ok, Bookie2A} = leveled_bookie:book_start(StartOpts2), test_singledelta_stores(Bookie2A, Bookie3, small, {B1, K1}), - + ok = leveled_bookie:book_close(Bookie2A), ok = leveled_bookie:book_close(Bookie3). @@ -507,3 +580,4 @@ dollar_bucket_index(_Config) -> ok = leveled_bookie:book_close(Bookie1), testutil:reset_filestructure(). +