From 9d92ca0773020d2103d8366fb482a0e40913dbc8 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Mon, 16 Mar 2020 12:51:14 +0000 Subject: [PATCH] Add tests for appDefined functions --- src/leveled_bookie.erl | 4 + src/leveled_codec.erl | 10 +- src/leveled_head.erl | 9 ++ test/end_to_end/appdefined_SUITE.erl | 134 ++++++++++++++++++++++++++- test/end_to_end/recovery_SUITE.erl | 81 +++++++++++++++- test/end_to_end/testutil.erl | 77 ++++++++++++--- 6 files changed, 289 insertions(+), 26 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index bcbf4ef..27c158d 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -3276,6 +3276,10 @@ sqnorder_mutatefold_test() -> ok = book_destroy(Bookie1). +search_test() -> + ?assertMatch({value, 5}, search(fun(X) -> X == 5 end, lists:seq(1, 10))), + ?assertMatch(false, search(fun(X) -> X == 55 end, lists:seq(1, 10))). + check_notfound_test() -> ProbablyFun = fun() -> probably end, MissingFun = fun() -> missing end, diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index 95e894e..35ab5d1 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -366,14 +366,12 @@ endkey_passed(EndKey, CheckingKey) -> %% Take the default strategy for compaction, and override the approach for any %% tags passed in inker_reload_strategy(AltList) -> - ReloadStrategy0 = + DefaultList = lists:map(fun leveled_head:default_reload_strategy/1, leveled_head:defined_objecttags()), - lists:foldl(fun({X, Y}, SList) -> - lists:keyreplace(X, 1, SList, {X, Y}) - end, - ReloadStrategy0, - AltList). + lists:ukeymerge(1, + lists:ukeysort(1, AltList), + lists:ukeysort(1, DefaultList)). -spec get_tagstrategy(ledger_key()|tag()|dummy, compaction_strategy()) diff --git a/src/leveled_head.erl b/src/leveled_head.erl index 0669ec5..ad4bbfa 100644 --- a/src/leveled_head.erl +++ b/src/leveled_head.erl @@ -472,4 +472,13 @@ diff_index_test() -> ?assertMatch([{add, <<"idx1_bin">>, <<"20840930001702Zoe">>}, {remove, <<"idx1_bin">>,<<"20231126131808Madison">>}], IdxSpecs). +decode_test() -> + Bin = <<"999">>, + BinTerm = term_to_binary("999"), + ?assertMatch("999", binary_to_list( + decode_maybe_binary(<<1:8/integer, Bin/binary>>))), + ?assertMatch("999", decode_maybe_binary(<<0:8/integer, BinTerm/binary>>)), + ?assertMatch("999", binary_to_list( + decode_maybe_binary(<<2:8/integer, Bin/binary>>))). + -endif. \ No newline at end of file diff --git a/test/end_to_end/appdefined_SUITE.erl b/test/end_to_end/appdefined_SUITE.erl index 79301de..c041844 100644 --- a/test/end_to_end/appdefined_SUITE.erl +++ b/test/end_to_end/appdefined_SUITE.erl @@ -2,11 +2,14 @@ -include_lib("common_test/include/ct.hrl"). -include("include/leveled.hrl"). -export([all/0]). --export([application_defined_tag/1 +-export([ + application_defined_tag/1, + bespoketag_recalc/1 ]). all() -> [ - application_defined_tag + % application_defined_tag, + bespoketag_recalc ]. @@ -62,6 +65,8 @@ application_defined_tag_tester(KeyCount, Tag, Functions, ExpectMD) -> StartOpts1 = [{root_path, RootPath}, {sync_strategy, testutil:sync_strategy()}, {log_level, warn}, + {reload_strategy, + [{bespoke_tag1, retain}, {bespoke_tag2, retain}]}, {override_functions, Functions}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), Value = leveled_rand:rand_bytes(512), @@ -107,8 +112,6 @@ application_defined_tag_tester(KeyCount, Tag, Functions, ExpectMD) -> ok = leveled_bookie:book_close(Bookie2). - - object_generator(Count, V) -> Hash = erlang:phash2({count, V}), @@ -118,4 +121,125 @@ object_generator(Count, V) -> {Bucket, Key, [{hash, Hash}, {shard, Count rem 10}, - {random, Random}, {value, V}]}. \ No newline at end of file + {random, Random}, {value, V}]}. + + + +bespoketag_recalc(_Config) -> + %% Get a sensible behaviour using the recalc compaction strategy with a + %% bespoke tag + + RootPath = testutil:reset_filestructure(), + B0 = <<"B0">>, + KeyCount = 7000, + + ExtractMDFun = + fun(bespoke_tag, Size, Obj) -> + [{index, IL}, {value, _V}] = Obj, + {{erlang:phash2(term_to_binary(Obj)), + Size, + {index, IL}}, + [os:timestamp()]} + end, + CalcIndexFun = + fun(bespoke_tag, UpdMeta, PrvMeta) -> + % io:format("UpdMeta ~w PrvMeta ~w~n", [UpdMeta, PrvMeta]), + {index, UpdIndexes} = element(3, UpdMeta), + IndexDeltas = + case PrvMeta of + not_present -> + UpdIndexes; + PrvMeta when is_tuple(PrvMeta) -> + {index, PrvIndexes} = element(3, PrvMeta), + lists:subtract(UpdIndexes, PrvIndexes) + end, + lists:map(fun(I) -> {add, <<"temp_int">>, I} end, IndexDeltas) + end, + + BookOpts = [{root_path, RootPath}, + {cache_size, 1000}, + {max_journalobjectcount, 6000}, + {max_pencillercachesize, 8000}, + {sync_strategy, testutil:sync_strategy()}, + {reload_strategy, [{bespoke_tag, recalc}]}, + {override_functions, + [{extract_metadata, ExtractMDFun}, + {diff_indexspecs, CalcIndexFun}]}], + + {ok, Book1} = leveled_bookie:book_start(BookOpts), + LoadFun = + fun(Book, MustFind) -> + fun(I) -> + testutil:stdload_object(Book, + B0, list_to_binary(["A"|integer_to_list(I rem KeyCount)]), + I, erlang:phash2({value, I}), + infinity, bespoke_tag, false, MustFind) + end + end, + lists:foreach(LoadFun(Book1, false), lists:seq(1, KeyCount)), + lists:foreach(LoadFun(Book1, true), lists:seq(KeyCount + 1, KeyCount * 2)), + + FoldFun = + fun(_B0, {IV0, _K0}, Acc) -> + case IV0 - 1 of + Acc -> + Acc + 1; + _Unexpected -> + % io:format("Eh? - ~w ~w~n", [Unexpected, Acc]), + Acc + 1 + end + end, + + CountFold = + fun(Book, CurrentCount) -> + leveled_bookie:book_indexfold(Book, + B0, + {FoldFun, 0}, + {<<"temp_int">>, 0, CurrentCount}, + {true, undefined}) + end, + + {async, FolderA} = CountFold(Book1, 2 * KeyCount), + CountA = FolderA(), + io:format("Counted double index entries ~w - everything loaded OK~n", + [CountA]), + true = 2 * KeyCount == CountA, + + io:format("Before close looking for Key 999 ~w~n", + [leveled_bookie:book_head(Book1, B0, <<"A999">>, bespoke_tag)]), + + ok = leveled_bookie:book_close(Book1), + + {ok, Book2} = leveled_bookie:book_start(BookOpts), + io:format("After opening looking for Key 999 ~w~n", + [leveled_bookie:book_head(Book2, B0, <<"A999">>, bespoke_tag)]), + + lists:foreach(LoadFun(Book2, true), lists:seq(KeyCount * 2 + 1, KeyCount * 3)), + + io:format("After fresh load looking for Key 999 ~w~n", + [leveled_bookie:book_head(Book2, B0, <<"A999">>, bespoke_tag)]), + + {async, FolderB} = CountFold(Book2, 3 * KeyCount), + CountB = FolderB(), + io:format("Counted triple index entries ~w - everything re-loaded~n", + [CountB]), + true = 3 * KeyCount == CountB, + + testutil:compact_and_wait(Book2), + ok = leveled_bookie:book_close(Book2), + + io:format("Restart from blank ledger~n"), + + leveled_penciller:clean_testdir(proplists:get_value(root_path, BookOpts) ++ + "/ledger"), + {ok, Book3} = leveled_bookie:book_start(BookOpts), + + {async, FolderC} = CountFold(Book3, 3 * KeyCount), + CountC = FolderC(), + io:format("All index entries ~w present - recalc ok~n", + [CountC]), + true = 3 * KeyCount == CountC, + + ok = leveled_bookie:book_close(Book3), + + testutil:reset_filestructure(). \ No newline at end of file diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index 3538cea..44d9418 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -10,6 +10,7 @@ recalc_strategy/1, recalc_transition_strategy/1, recovr_strategy/1, + stdtag_recalc/1, aae_missingjournal/1, aae_bustedjournal/1, journal_compaction_bustedjournal/1, @@ -31,7 +32,8 @@ all() -> [ journal_compaction_bustedjournal, close_duringcompaction, allkeydelta_journal_multicompact, - recompact_keydeltas + recompact_keydeltas, + stdtag_recalc ]. @@ -149,8 +151,6 @@ recovery_with_samekeyupdates(_Config) -> testutil:reset_filestructure(). - - hot_backup_simple(_Config) -> % The journal may have a hot backup. This allows for an online Bookie % to be sent a message to prepare a backup function, which an asynchronous @@ -331,6 +331,81 @@ rotate_wipe_compact(Strategy1, Strategy2) -> testutil:reset_filestructure(). +stdtag_recalc(_Config) -> + %% Setting the ?STD_TAG to do recalc, should result in the ?STD_TAG + %% behaving like recovr - as no recalc is done for ?STD_TAG + + %% NOTE -This is a test to confirm bad things happen! + + RootPath = testutil:reset_filestructure(), + B0 = <<"B0">>, + KeyCount = 7000, + BookOpts = [{root_path, RootPath}, + {cache_size, 1000}, + {max_journalobjectcount, 5000}, + {max_pencillercachesize, 10000}, + {sync_strategy, testutil:sync_strategy()}, + {reload_strategy, [{?STD_TAG, recalc}]}], + {ok, Book1} = leveled_bookie:book_start(BookOpts), + LoadFun = + fun(Book) -> + fun(I) -> + testutil:stdload_object(Book, + B0, erlang:phash2(I rem KeyCount), + I, erlang:phash2({value, I}), + infinity, ?STD_TAG, false, false) + end + end, + lists:foreach(LoadFun(Book1), lists:seq(1, KeyCount)), + lists:foreach(LoadFun(Book1), lists:seq(KeyCount + 1, KeyCount * 2)), + + CountFold = + fun(Book, CurrentCount) -> + leveled_bookie:book_indexfold(Book, + B0, + {fun(_BF, _KT, Acc) -> Acc + 1 end, + 0}, + {<<"temp_int">>, 0, CurrentCount}, + {true, undefined}) + end, + + {async, FolderA} = CountFold(Book1, 2 * KeyCount), + CountA = FolderA(), + io:format("Counted double index entries ~w - everything loaded OK~n", + [CountA]), + true = 2 * KeyCount == CountA, + + ok = leveled_bookie:book_close(Book1), + + {ok, Book2} = leveled_bookie:book_start(BookOpts), + lists:foreach(LoadFun(Book2), lists:seq(KeyCount * 2 + 1, KeyCount * 3)), + + {async, FolderB} = CountFold(Book2, 3 * KeyCount), + CountB = FolderB(), + io:format("Maybe counted less index entries ~w - everything not loaded~n", + [CountB]), + true = 3 * KeyCount >= CountB, + + compact_and_wait(Book2), + ok = leveled_bookie:book_close(Book2), + + io:format("Restart from blank ledger"), + + leveled_penciller:clean_testdir(proplists:get_value(root_path, BookOpts) ++ + "/ledger"), + {ok, Book3} = leveled_bookie:book_start(BookOpts), + + {async, FolderC} = CountFold(Book3, 3 * KeyCount), + CountC = FolderC(), + io:format("Missing index entries ~w - recalc not supported on ?STD_TAG~n", + [CountC]), + true = 3 * KeyCount > CountC, + + ok = leveled_bookie:book_close(Book3), + + testutil:reset_filestructure(). + + recovr_strategy(_Config) -> RootPath = testutil:reset_filestructure(), BookOpts = [{root_path, RootPath}, diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index 52db533..b84d757 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -10,6 +10,7 @@ stdload/2, stdload_expiring/3, stdload_object/6, + stdload_object/9, reset_filestructure/0, reset_filestructure/1, check_bucket_stats/2, @@ -59,7 +60,8 @@ get_value_from_objectlistitem/1, numbered_key/1, fixed_bin_key/1, - convert_to_seconds/1]). + convert_to_seconds/1, + compact_and_wait/1]). -define(RETURN_TERMS, {true, undefined}). -define(SLOWOFFER_DELAY, 5). @@ -241,17 +243,46 @@ stdload_expiring(Book, KeyCount, TTL, V, Acc) -> stdload_expiring(Book, KeyCount - 1, TTL, V, [{I, B, K}|Acc]). stdload_object(Book, B, K, I, V, TTL) -> - Obj = [{index, I}, {value, V}], - IdxSpecs = - case leveled_bookie:book_get(Book, B, K) of - {ok, PrevObj} -> - {index, OldI} = lists:keyfind(index, 1, PrevObj), - io:format("Remove index ~w for ~w~n", [OldI, I]), - [{remove, <<"temp_int">>, OldI}, {add, <<"temp_int">>, I}]; - not_found -> - [{add, <<"temp_int">>, I}] + stdload_object(Book, B, K, I, V, TTL, ?STD_TAG, true, false). + +stdload_object(Book, B, K, I, V, TTL, Tag, RemovePrev2i, MustFind) -> + Obj = [{index, [I]}, {value, V}], + {IdxSpecs, Obj0} = + case {leveled_bookie:book_get(Book, B, K, Tag), MustFind} of + {{ok, PrevObj}, _} -> + {index, PrevIs} = lists:keyfind(index, 1, PrevObj), + case RemovePrev2i of + true -> + MapFun = + fun(OldI) -> {remove, <<"temp_int">>, OldI} end, + {[{add, <<"temp_int">>, I}|lists:map(MapFun, PrevIs)], + Obj}; + false -> + {[{add, <<"temp_int">>, I}], + [{index, [I|PrevIs]}, {value, V}]} + end; + {not_found, false} -> + {[{add, <<"temp_int">>, I}], Obj}; + {not_found, true} -> + HR = leveled_bookie:book_head(Book, B, K, Tag), + io:format("Unexpected not_found for key=~w I=~w HR=~w~n ", + [K, I, HR]), + {[{add, <<"temp_int">>, I}], Obj} end, - R = leveled_bookie:book_tempput(Book, B, K, Obj, IdxSpecs, ?STD_TAG, TTL), + R = + case TTL of + infinity -> + leveled_bookie:book_put(Book, B, K, Obj0, IdxSpecs, Tag); + TTL when is_integer(TTL) -> + leveled_bookie:book_tempput(Book, B, K, Obj0, + IdxSpecs, Tag, TTL) + end, + case K of + <<57, 57, 57>> -> + io:format("K ~w I ~w R ~w~n", [K, I, R]); + _ -> + ok + end, case R of ok -> ok; @@ -262,6 +293,7 @@ stdload_object(Book, B, K, I, V, TTL) -> + reset_filestructure() -> reset_filestructure(0, ?ROOT_PATH). @@ -883,4 +915,25 @@ get_aae_segment(Obj) -> get_aae_segment({Type, Bucket}, Key) -> leveled_tictac:keyto_segment32(<>); get_aae_segment(Bucket, Key) -> - leveled_tictac:keyto_segment32(<>). \ No newline at end of file + leveled_tictac:keyto_segment32(<>). + +compact_and_wait(Book) -> + compact_and_wait(Book, 20000). + +compact_and_wait(Book, WaitForDelete) -> + ok = leveled_bookie:book_compactjournal(Book, 30000), + F = fun leveled_bookie:book_islastcompactionpending/1, + lists:foldl(fun(X, Pending) -> + case Pending of + false -> + false; + true -> + io:format("Loop ~w waiting for journal " + ++ "compaction to complete~n", [X]), + timer:sleep(20000), + F(Book) + end end, + true, + lists:seq(1, 15)), + io:format("Waiting for journal deletes~n"), + timer:sleep(WaitForDelete).