From e9fb893ea0c01af034ca54343b7434164b2897f8 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Mon, 5 Nov 2018 10:31:15 +0000 Subject: [PATCH] Check segment is as expected with tuplebuckets In head_only mode --- src/leveled_codec.erl | 12 ++-- test/end_to_end/riak_SUITE.erl | 11 +--- test/end_to_end/testutil.erl | 14 ++++- test/end_to_end/tictac_SUITE.erl | 100 ++++++++++++++++++++++++++++++- 4 files changed, 118 insertions(+), 19 deletions(-) diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index 196672a..e9328b8 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -187,12 +187,12 @@ segment_hash({?HEAD_TAG, Bucket, Key, SubK}) segment_hash({?HEAD_TAG, Bucket, Key, _SubK}) when is_binary(Bucket), is_binary(Key) -> segment_hash(<>); -% segment_hash({?HEAD_TAG, {BucketType, Bucket}, Key, SubKey}) -% when is_binary(BucketType), is_binary(Bucket) -> -% segment_hash({?HEAD_TAG, -% <>, -% Key, -% SubKey}); +segment_hash({?HEAD_TAG, {BucketType, Bucket}, Key, SubKey}) + when is_binary(BucketType), is_binary(Bucket) -> + segment_hash({?HEAD_TAG, + <>, + Key, + SubKey}); segment_hash(Key) -> segment_hash(term_to_binary(Key)). diff --git a/test/end_to_end/riak_SUITE.erl b/test/end_to_end/riak_SUITE.erl index e7de788..d41af08 100644 --- a/test/end_to_end/riak_SUITE.erl +++ b/test/end_to_end/riak_SUITE.erl @@ -199,7 +199,7 @@ basic_riak_tester(Bucket, KeyCount) -> {_I2L, Obj2L, _Spc2L} = lists:last(ObjList2), SegList = - lists:map(fun(Obj) -> get_aae_segment(Obj) end, + lists:map(fun(Obj) -> testutil:get_aae_segment(Obj) end, [Obj1, Obj2, Obj3, Obj4, Obj5, Obj2L]), BKList = lists:map(fun(Obj) -> @@ -223,15 +223,6 @@ basic_riak_tester(Bucket, KeyCount) -> ok = leveled_bookie:book_destroy(Bookie2). -get_aae_segment(Obj) -> - get_aae_segment(testutil:get_bucket(Obj), testutil:get_key(Obj)). - -get_aae_segment({Type, Bucket}, Key) -> - leveled_tictac:keyto_segment32(<>); -get_aae_segment(Bucket, Key) -> - leveled_tictac:keyto_segment32(<>). - - fetchclocks_modifiedbetween(_Config) -> RootPathA = testutil:reset_filestructure("fetchClockA"), RootPathB = testutil:reset_filestructure("fetchClockB"), diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index 0f0d83b..a4235a0 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -33,6 +33,8 @@ get_compressiblevalue/0, get_compressiblevalue_andinteger/0, get_randomindexes_generator/1, + get_aae_segment/1, + get_aae_segment/2, name_list/0, load_objects/5, load_objects/6, @@ -799,4 +801,14 @@ find_journals(RootPath) -> CDBFiles. convert_to_seconds({MegaSec, Seconds, _MicroSec}) -> - MegaSec * 1000000 + Seconds. \ No newline at end of file + MegaSec * 1000000 + Seconds. + + + +get_aae_segment(Obj) -> + get_aae_segment(testutil:get_bucket(Obj), testutil:get_key(Obj)). + +get_aae_segment({Type, Bucket}, Key) -> + leveled_tictac:keyto_segment32(<>); +get_aae_segment(Bucket, Key) -> + leveled_tictac:keyto_segment32(<>). \ No newline at end of file diff --git a/test/end_to_end/tictac_SUITE.erl b/test/end_to_end/tictac_SUITE.erl index 20c748c..e0f5f01 100644 --- a/test/end_to_end/tictac_SUITE.erl +++ b/test/end_to_end/tictac_SUITE.erl @@ -5,13 +5,15 @@ -export([ many_put_compare/1, index_compare/1, - basic_headonly/1 + basic_headonly/1, + tuplebuckets_headonly/1 ]). all() -> [ many_put_compare, index_compare, - basic_headonly + basic_headonly, + tuplebuckets_headonly ]. -define(LMD_FORMAT, "~4..0w~2..0w~2..0w~2..0w~2..0w"). @@ -540,6 +542,100 @@ index_compare(_Config) -> ok = leveled_bookie:book_close(Book2D). +tuplebuckets_headonly(_Config) -> + ObjectCount = 60000, + + RootPathHO = testutil:reset_filestructure("testTBHO"), + StartOpts1 = [{root_path, RootPathHO}, + {max_pencillercachesize, 16000}, + {sync_strategy, none}, + {head_only, with_lookup}, + {max_journalsize, 500000}], + {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), + + ObjectSpecFun = + fun(Op) -> + fun(N) -> + Bucket = {<<"BucketType">>, <<"B", 0:4/integer, N:4/integer>>}, + Key = <<"K", N:32/integer>>, + <> = + crypto:hash(md5, <>), + {Op, Bucket, Key, null, Hash} + end + end, + + ObjectSpecL = lists:map(ObjectSpecFun(add), lists:seq(1, ObjectCount)), + + SW0 = os:timestamp(), + ok = load_objectspecs(ObjectSpecL, 32, Bookie1), + io:format("Loaded an object count of ~w in ~w ms~n", + [ObjectCount, timer:now_diff(os:timestamp(), SW0)/1000]), + + CheckHeadFun = + fun({add, B, K, null, H}) -> + {ok, H} = + leveled_bookie:book_headonly(Bookie1, B, K, null) + end, + lists:foreach(CheckHeadFun, ObjectSpecL), + + BucketList = + lists:map(fun(I) -> + {<<"BucketType">>, <<"B", 0:4/integer, I:4/integer>>} + end, + lists:seq(0, 15)), + + FoldHeadFun = + fun(B, {K, null}, V, Acc) -> + [{add, B, K, null, V}|Acc] + end, + SW1 = os:timestamp(), + + {async, HeadRunner1} = + leveled_bookie:book_headfold(Bookie1, + ?HEAD_TAG, + {bucket_list, BucketList}, + {FoldHeadFun, []}, + false, false, + false), + ReturnedObjSpecL1 = lists:reverse(HeadRunner1()), + [FirstItem|_Rest] = ReturnedObjSpecL1, + LastItem = lists:last(ReturnedObjSpecL1), + + io:format("Returned ~w objects with first ~w and last ~w in ~w ms~n", + [length(ReturnedObjSpecL1), + FirstItem, LastItem, + timer:now_diff(os:timestamp(), SW1)/1000]), + + true = ReturnedObjSpecL1 == lists:sort(ObjectSpecL), + + {add, {TB, B1}, K1, null, _H1} = FirstItem, + {add, {TB, BL}, KL, null, _HL} = LastItem, + SegList = [testutil:get_aae_segment({TB, B1}, K1), + testutil:get_aae_segment({TB, BL}, KL)], + + SW2 = os:timestamp(), + {async, HeadRunner2} = + leveled_bookie:book_headfold(Bookie1, + ?HEAD_TAG, + {bucket_list, BucketList}, + {FoldHeadFun, []}, + false, false, + SegList), + ReturnedObjSpecL2 = lists:reverse(HeadRunner2()), + + io:format("Returned ~w objects using seglist in ~w ms~n", + [length(ReturnedObjSpecL2), + timer:now_diff(os:timestamp(), SW2)/1000]), + + true = length(ReturnedObjSpecL2) < (ObjectCount/1000 + 2), + % Not too many false positives + true = lists:member(FirstItem, ReturnedObjSpecL2), + true = lists:member(LastItem, ReturnedObjSpecL2), + + leveled_bookie:book_destroy(Bookie1). + + + basic_headonly(_Config) -> ObjectCount = 200000, RemoveCount = 100,