diff --git a/rebar.config b/rebar.config index 7a46b31..a6c6659 100644 --- a/rebar.config +++ b/rebar.config @@ -1,8 +1,10 @@ {erl_opts, [warnings_as_errors, {platform_define, "^2[0-1]{1}", fsm_deprecated}, {platform_define, "^1[7-8]{1}", old_rand}, + {platform_define, "^17", no_log2}, {platform_define, "^R", no_sync}, {platform_define, "^R", old_rand}, + {platform_define, "^R", no_log2}, {platform_define, "^R", slow_test}]}. {xref_checks, [undefined_function_calls,undefined_functions]}. diff --git a/src/leveled_math.erl b/src/leveled_math.erl new file mode 100644 index 0000000..f88ed4f --- /dev/null +++ b/src/leveled_math.erl @@ -0,0 +1,38 @@ +%% Handle missing log2 prior to OTP18 + +-module(leveled_math). + +%% API +-export([ + log2/1 + ]). + + +-include_lib("eunit/include/eunit.hrl"). + +%%%=================================================================== +%%% Use log2 +%%%=================================================================== +-ifndef(no_log2). + +log2(X) -> + math:log2(X). + +-else. +%%%=================================================================== +%%% Old (r18) random style functions +%%%=================================================================== + +log2(X) -> + math:log(X) / 0.6931471805599453. + +-endif. + + +-ifdef(TEST). + +log2_test() -> + ?assertMatch(8, round(log2(256))), + ?assertMatch(16, round(log2(65536))). + +-endif. diff --git a/src/leveled_tictac.erl b/src/leveled_tictac.erl index 9b1a813..76d77da 100644 --- a/src/leveled_tictac.erl +++ b/src/leveled_tictac.erl @@ -73,6 +73,7 @@ keyto_segment32/1, keyto_segment48/1, generate_segmentfilter_list/2, + adjust_segmentmatch_list/3, merge_binaries/2, join_segment/2, match_segment/2 @@ -383,6 +384,49 @@ generate_segmentfilter_list(SegmentList, Size) -> SegmentList end. +-spec adjust_segmentmatch_list(list(integer()), tree_size(), tree_size()) + -> list(integer()). +%% @doc +%% If we have dirty segments discovered by comparing trees of size CompareSize, +%% and we want to see if it matches a segment for a key which was created for a +%% tree of size Store Size, then we need to alter the segment list +%% +%% See timing_test/0 when considering using this or match_segment/2 +%% +%% Check with KeyCount=10000 SegCount=4 TreeSizes small large: +%% adjust_segmentmatch_list check took 1.256 ms match_segment took 5.229 ms +%% +%% Check with KeyCount=10000 SegCount=8 TreeSizes small large: +%% adjust_segmentmatch_list check took 2.065 ms match_segment took 8.637 ms +%% +%% Check with KeyCount=10000 SegCount=4 TreeSizes medium large: +%% adjust_segmentmatch_list check took 0.453 ms match_segment took 4.843 ms +%% +%% Check with KeyCount=10000 SegCount=4 TreeSizes small medium: +%% adjust_segmentmatch_list check took 0.451 ms match_segment took 5.528 ms +%% +%% Check with KeyCount=100000 SegCount=4 TreeSizes small large: +%% adjust_segmentmatch_list check took 11.986 ms match_segment took 56.522 ms +%% +adjust_segmentmatch_list(SegmentList, CompareSize, StoreSize) -> + CompareSizeI = get_size(CompareSize), + StoreSizeI = get_size(StoreSize), + if CompareSizeI =< StoreSizeI -> + ExpItems = StoreSizeI div CompareSizeI - 1, + ShiftFactor = round(leveled_math:log2(CompareSizeI * ?L2_CHUNKSIZE)), + ExpList = + lists:map(fun(X) -> X bsl ShiftFactor end, lists:seq(1, ExpItems)), + UpdSegmentList = + lists:foldl(fun(S, Acc) -> + L = lists:map(fun(F) -> F + S end, ExpList), + L ++ Acc + end, + [], + SegmentList), + lists:usort(UpdSegmentList ++ SegmentList) + end. + + -spec match_segment({integer(), tree_size()}, {integer(), tree_size()}) -> boolean(). %% @doc @@ -696,15 +740,102 @@ compare_trees_maxonedelta(Tree0, Tree1) -> end. segment_match_test() -> - segment_match_tester(small, large), - segment_match_tester(xlarge, medium). + segment_match_tester(small, large, <<"K0">>), + segment_match_tester(xlarge, medium, <<"K1">>), + expand_membershiplist_tester(small, large, <<"K0">>), + expand_membershiplist_tester(xsmall, large, <<"K1">>), + expand_membershiplist_tester(large, xlarge, <<"K2">>). -segment_match_tester(Size1, Size2) -> - HashKey = keyto_segment32(<<"K0">>), +segment_match_tester(Size1, Size2, Key) -> + HashKey = keyto_segment32(Key), Segment1 = get_segment(HashKey, Size1), Segment2 = get_segment(HashKey, Size2), ?assertMatch(true, match_segment({Segment1, Size1}, {Segment2, Size2})). +expand_membershiplist_tester(SmallSize, LargeSize, Key) -> + HashKey = keyto_segment32(Key), + Segment1 = get_segment(HashKey, SmallSize), + Segment2 = get_segment(HashKey, LargeSize), + AdjList = adjust_segmentmatch_list([Segment1], SmallSize, LargeSize), + ?assertMatch(true, lists:member(Segment2, AdjList)). + + +segment_expandsimple_test() -> + AdjList = adjust_segmentmatch_list([1, 100], small, medium), + io:format("List adjusted to ~w~n", [AdjList]), + ?assertMatch(true, lists:member(1, AdjList)), + ?assertMatch(true, lists:member(100, AdjList)), + ?assertMatch(true, lists:member(65537, AdjList)), + ?assertMatch(true, lists:member(131073, AdjList)), + ?assertMatch(true, lists:member(196609, AdjList)), + ?assertMatch(true, lists:member(65636, AdjList)), + ?assertMatch(true, lists:member(131172, AdjList)), + ?assertMatch(true, lists:member(196708, AdjList)), + ?assertMatch(8, length(AdjList)), + OrigList = adjust_segmentmatch_list([1, 100], medium, medium), + ?assertMatch([1, 100], OrigList). + + +timing_test() -> + timing_tester(10000, 4, small, large), + timing_tester(10000, 8, small, large), + timing_tester(10000, 4, medium, large), + timing_tester(10000, 4, small, medium), + timing_tester(100000, 4, small, large). + + +timing_tester(KeyCount, SegCount, SmallSize, LargeSize) -> + SegList = + lists:map(fun(_C) -> + leveled_rand:uniform(get_size(SmallSize) * ?L2_CHUNKSIZE - 1) + end, + lists:seq(1, SegCount)), + KeyToSegFun = + fun(I) -> + HK = keyto_segment32(integer_to_binary(I)), + {I, get_segment(HK, LargeSize)} + end, + + MatchList = lists:map(KeyToSegFun, lists:seq(1, KeyCount)), + + {T0, Out0} = + adjustsegmentlist_check(SegList, MatchList, SmallSize, LargeSize), + {T1, Out1} = + matchbysegment_check(SegList, MatchList, SmallSize, LargeSize), + ?assertMatch(true, Out0 == Out1), + io:format(user, "~nCheck with KeyCount=~w SegCount=~w TreeSizes ~w ~w:~n", + [KeyCount, SegCount, SmallSize, LargeSize]), + io:format(user, + "adjust_segmentmatch_list check took ~w ms " ++ + "match_segment took ~w ms~n", + [T0, T1]). + + +adjustsegmentlist_check(SegList, MatchList, SmallSize, LargeSize) -> + SW = os:timestamp(), + AdjList = adjust_segmentmatch_list(SegList, SmallSize, LargeSize), + PredFun = + fun({_I, S}) -> + lists:member(S, AdjList) + end, + OL = lists:filter(PredFun, MatchList), + {timer:now_diff(os:timestamp(), SW)/1000, OL}. + +matchbysegment_check(SegList, MatchList, SmallSize, LargeSize) -> + SW = os:timestamp(), + PredFun = + fun({_I, S}) -> + FoldFun = + fun(_SM, true) -> + true; + (SM, false) -> + match_segment({SM, SmallSize}, {S, LargeSize}) + end, + lists:foldl(FoldFun, false, SegList) + end, + OL = lists:filter(PredFun, MatchList), + {timer:now_diff(os:timestamp(), SW)/1000, OL}. + -endif.