Add tests for appDefined functions

src/leveled_bookie.erl

@@ -3276,6 +3276,10 @@ sqnorder_mutatefold_test() ->
     
     ok = book_destroy(Bookie1).
 
+search_test() ->
+    ?assertMatch({value, 5}, search(fun(X) -> X == 5 end, lists:seq(1, 10))),
+    ?assertMatch(false, search(fun(X) -> X == 55 end, lists:seq(1, 10))).
+
 check_notfound_test() ->
     ProbablyFun = fun() -> probably end,
     MissingFun = fun() -> missing end,

src/leveled_codec.erl

@@ -366,14 +366,12 @@ endkey_passed(EndKey, CheckingKey) ->
 %% Take the default strategy for compaction, and override the approach for any 
 %% tags passed in
 inker_reload_strategy(AltList) ->
-    ReloadStrategy0 = 
+    DefaultList = 
         lists:map(fun leveled_head:default_reload_strategy/1,
                     leveled_head:defined_objecttags()),
-    lists:foldl(fun({X, Y}, SList) ->
+    lists:ukeymerge(1,
-                        lists:keyreplace(X, 1, SList, {X, Y})
+                    lists:ukeysort(1, AltList),
-                        end,
+                    lists:ukeysort(1, DefaultList)).
-                    ReloadStrategy0,
-                    AltList).
 
 
 -spec get_tagstrategy(ledger_key()|tag()|dummy, compaction_strategy()) 

src/leveled_head.erl

@@ -472,4 +472,13 @@ diff_index_test() ->
     ?assertMatch([{add, <<"idx1_bin">>, <<"20840930001702Zoe">>},
                     {remove, <<"idx1_bin">>,<<"20231126131808Madison">>}], IdxSpecs).
 
+decode_test() ->
+    Bin = <<"999">>,
+    BinTerm = term_to_binary("999"),
+    ?assertMatch("999", binary_to_list(
+                            decode_maybe_binary(<<1:8/integer, Bin/binary>>))),
+    ?assertMatch("999", decode_maybe_binary(<<0:8/integer, BinTerm/binary>>)),
+    ?assertMatch("999", binary_to_list(
+                            decode_maybe_binary(<<2:8/integer, Bin/binary>>))).
+
 -endif.

test/end_to_end/appdefined_SUITE.erl

@@ -2,11 +2,14 @@
 -include_lib("common_test/include/ct.hrl").
 -include("include/leveled.hrl").
 -export([all/0]).
--export([application_defined_tag/1
+-export([
+            application_defined_tag/1,
+            bespoketag_recalc/1
             ]).
 
 all() -> [
-            application_defined_tag
+            % application_defined_tag,
+            bespoketag_recalc
             ].
 
 
@@ -62,6 +65,8 @@ application_defined_tag_tester(KeyCount, Tag, Functions, ExpectMD) ->
     StartOpts1 = [{root_path, RootPath},
                     {sync_strategy, testutil:sync_strategy()},
                     {log_level, warn},
+                    {reload_strategy,
+                        [{bespoke_tag1, retain}, {bespoke_tag2, retain}]},
                     {override_functions, Functions}],
     {ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
     Value = leveled_rand:rand_bytes(512),
@@ -107,8 +112,6 @@ application_defined_tag_tester(KeyCount, Tag, Functions, ExpectMD) ->
 
     ok = leveled_bookie:book_close(Bookie2).
 
-    
-
 
 object_generator(Count, V) ->
     Hash = erlang:phash2({count, V}),
@@ -118,4 +121,125 @@ object_generator(Count, V) ->
     {Bucket, 
         Key,
         [{hash, Hash}, {shard, Count rem 10},
-            {random, Random}, {value, V}]}.
+            {random, Random}, {value, V}]}.
+
+
+
+bespoketag_recalc(_Config) ->
+    %% Get a sensible behaviour using the recalc compaction strategy with a
+    %% bespoke tag
+
+    RootPath = testutil:reset_filestructure(),
+    B0 = <<"B0">>,
+    KeyCount = 7000,
+
+    ExtractMDFun = 
+        fun(bespoke_tag, Size, Obj) ->
+            [{index, IL}, {value, _V}] = Obj,
+            {{erlang:phash2(term_to_binary(Obj)), 
+                    Size,
+                    {index, IL}},
+                [os:timestamp()]}
+        end,
+    CalcIndexFun =
+        fun(bespoke_tag, UpdMeta, PrvMeta) ->
+            % io:format("UpdMeta ~w PrvMeta ~w~n", [UpdMeta, PrvMeta]),
+            {index, UpdIndexes} = element(3, UpdMeta),
+            IndexDeltas =
+                case PrvMeta of
+                    not_present ->
+                        UpdIndexes;
+                    PrvMeta when is_tuple(PrvMeta) ->
+                        {index, PrvIndexes} = element(3, PrvMeta),
+                        lists:subtract(UpdIndexes, PrvIndexes)
+                end,
+            lists:map(fun(I) -> {add, <<"temp_int">>, I} end, IndexDeltas)
+        end,
+
+    BookOpts = [{root_path, RootPath},
+                    {cache_size, 1000},
+                    {max_journalobjectcount, 6000},
+                    {max_pencillercachesize, 8000},
+                    {sync_strategy, testutil:sync_strategy()},
+                    {reload_strategy, [{bespoke_tag, recalc}]},
+                    {override_functions,
+                        [{extract_metadata, ExtractMDFun},
+                            {diff_indexspecs, CalcIndexFun}]}],
+    
+    {ok, Book1} = leveled_bookie:book_start(BookOpts),
+    LoadFun =
+        fun(Book, MustFind) ->
+            fun(I) ->
+                testutil:stdload_object(Book,
+                                        B0, list_to_binary(["A"|integer_to_list(I rem KeyCount)]),
+                                        I, erlang:phash2({value, I}),
+                                        infinity, bespoke_tag, false, MustFind)
+            end
+        end,
+    lists:foreach(LoadFun(Book1, false), lists:seq(1, KeyCount)),
+    lists:foreach(LoadFun(Book1, true), lists:seq(KeyCount + 1, KeyCount * 2)),
+
+    FoldFun =
+        fun(_B0, {IV0, _K0}, Acc) -> 
+            case IV0 - 1 of
+                Acc ->
+                    Acc + 1;
+                _Unexpected ->
+                    % io:format("Eh? - ~w ~w~n", [Unexpected, Acc]),
+                    Acc + 1
+            end
+        end,
+
+    CountFold =
+        fun(Book, CurrentCount) ->
+            leveled_bookie:book_indexfold(Book,
+                                            B0,
+                                            {FoldFun, 0},
+                                            {<<"temp_int">>, 0, CurrentCount},
+                                            {true, undefined})
+        end,
+
+    {async, FolderA} = CountFold(Book1, 2 * KeyCount),
+    CountA = FolderA(),
+    io:format("Counted double index entries ~w - everything loaded OK~n",
+                [CountA]),
+    true = 2 * KeyCount == CountA,
+
+    io:format("Before close looking for Key 999 ~w~n",
+                [leveled_bookie:book_head(Book1, B0, <<"A999">>, bespoke_tag)]),
+
+    ok = leveled_bookie:book_close(Book1),
+
+    {ok, Book2} = leveled_bookie:book_start(BookOpts),
+    io:format("After opening looking for Key 999 ~w~n",
+                [leveled_bookie:book_head(Book2, B0, <<"A999">>, bespoke_tag)]),
+
+    lists:foreach(LoadFun(Book2, true), lists:seq(KeyCount * 2 + 1, KeyCount * 3)),
+
+    io:format("After fresh load looking for Key 999 ~w~n",
+                [leveled_bookie:book_head(Book2, B0, <<"A999">>, bespoke_tag)]),
+
+    {async, FolderB} = CountFold(Book2, 3 * KeyCount),
+    CountB = FolderB(),
+    io:format("Counted triple index entries ~w - everything re-loaded~n",
+                [CountB]),
+    true = 3 * KeyCount == CountB,
+
+    testutil:compact_and_wait(Book2),
+    ok = leveled_bookie:book_close(Book2),
+
+    io:format("Restart from blank ledger~n"),
+
+    leveled_penciller:clean_testdir(proplists:get_value(root_path, BookOpts) ++
+                                    "/ledger"),
+    {ok, Book3} = leveled_bookie:book_start(BookOpts),
+
+    {async, FolderC} = CountFold(Book3, 3 * KeyCount),
+    CountC = FolderC(),
+    io:format("All index entries ~w present - recalc ok~n",
+                [CountC]),
+    true = 3 * KeyCount == CountC,
+
+    ok = leveled_bookie:book_close(Book3),
+    
+    testutil:reset_filestructure().

test/end_to_end/recovery_SUITE.erl

@@ -10,6 +10,7 @@
             recalc_strategy/1,
             recalc_transition_strategy/1,
             recovr_strategy/1,
+            stdtag_recalc/1,
             aae_missingjournal/1,
             aae_bustedjournal/1,
             journal_compaction_bustedjournal/1,
@@ -31,7 +32,8 @@ all() -> [
             journal_compaction_bustedjournal,
             close_duringcompaction,
             allkeydelta_journal_multicompact,
-            recompact_keydeltas
+            recompact_keydeltas,
+            stdtag_recalc
             ].
 
 
@@ -149,8 +151,6 @@ recovery_with_samekeyupdates(_Config) ->
     testutil:reset_filestructure().
 
 
-
-
 hot_backup_simple(_Config) ->
     % The journal may have a hot backup.  This allows for an online Bookie
     % to be sent a message to prepare a backup function, which an asynchronous
@@ -331,6 +331,81 @@ rotate_wipe_compact(Strategy1, Strategy2) ->
     testutil:reset_filestructure().
 
 
+stdtag_recalc(_Config) ->
+    %% Setting the ?STD_TAG to do recalc, should result in the ?STD_TAG
+    %% behaving like recovr - as no recalc is done for ?STD_TAG
+
+    %% NOTE -This is a test to confirm bad things happen!
+
+    RootPath = testutil:reset_filestructure(),
+    B0 = <<"B0">>,
+    KeyCount = 7000,
+    BookOpts = [{root_path, RootPath},
+                    {cache_size, 1000},
+                    {max_journalobjectcount, 5000},
+                    {max_pencillercachesize, 10000},
+                    {sync_strategy, testutil:sync_strategy()},
+                    {reload_strategy, [{?STD_TAG, recalc}]}],
+    {ok, Book1} = leveled_bookie:book_start(BookOpts),
+    LoadFun =
+        fun(Book) ->
+            fun(I) ->
+                testutil:stdload_object(Book,
+                                        B0, erlang:phash2(I rem KeyCount),
+                                        I, erlang:phash2({value, I}),
+                                        infinity, ?STD_TAG, false, false)
+            end
+        end,
+    lists:foreach(LoadFun(Book1), lists:seq(1, KeyCount)),
+    lists:foreach(LoadFun(Book1), lists:seq(KeyCount + 1, KeyCount * 2)),
+
+    CountFold =
+        fun(Book, CurrentCount) ->
+            leveled_bookie:book_indexfold(Book,
+                                            B0,
+                                            {fun(_BF, _KT, Acc) -> Acc + 1 end,
+                                                0},
+                                            {<<"temp_int">>, 0, CurrentCount},
+                                            {true, undefined})
+        end,
+
+    {async, FolderA} = CountFold(Book1, 2 * KeyCount),
+    CountA = FolderA(),
+    io:format("Counted double index entries ~w - everything loaded OK~n",
+                [CountA]),
+    true = 2 * KeyCount == CountA,
+
+    ok = leveled_bookie:book_close(Book1),
+
+    {ok, Book2} = leveled_bookie:book_start(BookOpts),
+    lists:foreach(LoadFun(Book2), lists:seq(KeyCount * 2 + 1, KeyCount * 3)),
+
+    {async, FolderB} = CountFold(Book2, 3 * KeyCount),
+    CountB = FolderB(),
+    io:format("Maybe counted less index entries ~w - everything not loaded~n",
+                [CountB]),
+    true = 3 * KeyCount >= CountB,
+
+    compact_and_wait(Book2),
+    ok = leveled_bookie:book_close(Book2),
+
+    io:format("Restart from blank ledger"),
+
+    leveled_penciller:clean_testdir(proplists:get_value(root_path, BookOpts) ++
+                                    "/ledger"),
+    {ok, Book3} = leveled_bookie:book_start(BookOpts),
+
+    {async, FolderC} = CountFold(Book3, 3 * KeyCount),
+    CountC = FolderC(),
+    io:format("Missing index entries ~w - recalc not supported on ?STD_TAG~n",
+                [CountC]),
+    true = 3 * KeyCount > CountC,
+
+    ok = leveled_bookie:book_close(Book3),
+    
+    testutil:reset_filestructure().
+
+
 recovr_strategy(_Config) ->
     RootPath = testutil:reset_filestructure(),
     BookOpts = [{root_path, RootPath},

test/end_to_end/testutil.erl

@@ -10,6 +10,7 @@
             stdload/2,
             stdload_expiring/3,
             stdload_object/6,
+            stdload_object/9,
             reset_filestructure/0,
             reset_filestructure/1,
             check_bucket_stats/2,
@@ -59,7 +60,8 @@
             get_value_from_objectlistitem/1,
             numbered_key/1,
             fixed_bin_key/1,
-            convert_to_seconds/1]).
+            convert_to_seconds/1,
+            compact_and_wait/1]).
 
 -define(RETURN_TERMS, {true, undefined}).
 -define(SLOWOFFER_DELAY, 5).
@@ -241,17 +243,46 @@ stdload_expiring(Book, KeyCount, TTL, V, Acc) ->
     stdload_expiring(Book, KeyCount - 1, TTL, V, [{I, B, K}|Acc]).
 
 stdload_object(Book, B, K, I, V, TTL) ->
-    Obj = [{index, I}, {value, V}],
+    stdload_object(Book, B, K, I, V, TTL, ?STD_TAG, true, false).
-    IdxSpecs = 
+
-        case leveled_bookie:book_get(Book, B, K) of
+stdload_object(Book, B, K, I, V, TTL, Tag, RemovePrev2i, MustFind) ->
-            {ok, PrevObj} ->
+    Obj = [{index, [I]}, {value, V}],
-                {index, OldI} = lists:keyfind(index, 1, PrevObj),
+    {IdxSpecs, Obj0} = 
-                io:format("Remove index ~w for ~w~n", [OldI, I]),
+        case {leveled_bookie:book_get(Book, B, K, Tag), MustFind} of
-                [{remove, <<"temp_int">>, OldI}, {add, <<"temp_int">>, I}];
+            {{ok, PrevObj}, _} ->
-            not_found ->
+                {index, PrevIs} = lists:keyfind(index, 1, PrevObj),
-                [{add, <<"temp_int">>, I}]
+                case RemovePrev2i of
+                    true ->
+                        MapFun =
+                            fun(OldI) -> {remove, <<"temp_int">>, OldI} end,
+                        {[{add, <<"temp_int">>, I}|lists:map(MapFun, PrevIs)],
+                            Obj};
+                    false ->
+                        {[{add, <<"temp_int">>, I}],
+                            [{index, [I|PrevIs]}, {value, V}]}
+                end;
+            {not_found, false} ->
+                {[{add, <<"temp_int">>, I}], Obj};
+            {not_found, true} ->
+                HR = leveled_bookie:book_head(Book, B, K, Tag),
+                io:format("Unexpected not_found for key=~w I=~w HR=~w~n ",
+                            [K, I, HR]), 
+                {[{add, <<"temp_int">>, I}], Obj}
         end,
-    R = leveled_bookie:book_tempput(Book, B, K, Obj, IdxSpecs, ?STD_TAG, TTL),
+    R =
+        case TTL of
+            infinity ->
+                leveled_bookie:book_put(Book, B, K, Obj0, IdxSpecs, Tag);
+            TTL when is_integer(TTL) ->
+                leveled_bookie:book_tempput(Book, B, K, Obj0,
+                                            IdxSpecs, Tag, TTL)
+        end,
+    case K of
+        <<57, 57, 57>> ->
+            io:format("K ~w I ~w R ~w~n", [K, I, R]);
+        _ ->
+            ok
+    end,
     case R of
         ok -> 
             ok;
@@ -262,6 +293,7 @@ stdload_object(Book, B, K, I, V, TTL) ->
 
 
 
+
 reset_filestructure() ->
     reset_filestructure(0, ?ROOT_PATH).
     
@@ -883,4 +915,25 @@ get_aae_segment(Obj) ->
 get_aae_segment({Type, Bucket}, Key) ->
     leveled_tictac:keyto_segment32(<<Type/binary, Bucket/binary, Key/binary>>);
 get_aae_segment(Bucket, Key) ->
-    leveled_tictac:keyto_segment32(<<Bucket/binary, Key/binary>>).
+    leveled_tictac:keyto_segment32(<<Bucket/binary, Key/binary>>).
+
+compact_and_wait(Book) ->
+    compact_and_wait(Book, 20000).
+
+compact_and_wait(Book, WaitForDelete) ->
+    ok = leveled_bookie:book_compactjournal(Book, 30000),
+    F = fun leveled_bookie:book_islastcompactionpending/1,
+    lists:foldl(fun(X, Pending) ->
+                        case Pending of
+                            false ->
+                                false;
+                            true ->
+                                io:format("Loop ~w waiting for journal "
+                                    ++ "compaction to complete~n", [X]),
+                                timer:sleep(20000),
+                                F(Book)
+                        end end,
+                    true,
+                    lists:seq(1, 15)),
+    io:format("Waiting for journal deletes~n"),
+    timer:sleep(WaitForDelete).