Add test of index expiry

To show how this works, and prove that it does work thta way.

Test may require adjusting if tested on a slow node (e.g. reduce KeyCount or increase TTL)

test/end_to_end/iterator_SUITE.erl

@@ -6,7 +6,8 @@
 -define(KEY_ONLY, {false, undefined}).
 
 -export([all/0]).
--export([breaking_folds/1,
+-export([expiring_indexes/1,
+            breaking_folds/1,
             single_object_with2i/1,
             small_load_with2i/1,
             query_count/1,
@@ -15,6 +16,7 @@
             rotating_objects/1]).
 
 all() -> [
+            expiring_indexes,
             breaking_folds,
             single_object_with2i,
             small_load_with2i,
@@ -25,6 +27,95 @@ all() -> [
             ].
 
 
+expiring_indexes(_Config) ->
+    % Add objects to ths tore with index entries, where the objects (and hence
+    % the indexes have an expiry time.  Confirm that the indexes and the
+    % objects are no longer present after the expiry time (and are present
+    % before).  Confirm that replacing an object has the expected outcome, if
+    % the IndexSpecs are updated as part of the request.
+    KeyCount = 50000,
+    Future = 120,
+        % 2 minutes - if running tests on a slow machine, may need to increase
+        % this value
+    RootPath = testutil:reset_filestructure(),
+    StartOpts1 = [{root_path, RootPath},
+                    {max_journalsize, 100000000},
+                    {sync_strategy, testutil:sync_strategy()}],
+    {ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
+    
+    SW1 = os:timestamp(),
+    IBKL1 = testutil:stdload_expiring(Bookie1, KeyCount, Future),
+    LoadTime = timer:now_diff(os:timestamp(), SW1)/1000000,
+    io:format("Load of ~w std objects in ~w seconds~n",  [KeyCount, LoadTime]),
+
+
+    FilterFun = fun({I, _B, _K}) -> lists:member(I, [5, 6, 7, 8]) end,
+    LoadedEntriesInRange = lists:sort(lists:filter(FilterFun, IBKL1)),
+
+    true = LoadTime < (Future - 30), 
+        % need 30 seconds spare to run query
+        % and add replacements
+
+    {_I0, B0, K0} = hd(IBKL1),
+    false = FilterFun(hd(IBKL1)), 
+        % The head entry should not have index between 5 and 8
+
+    FoldFun = fun(BF, {IdxV, KeyF}, Acc) -> [{IdxV, BF, KeyF}|Acc] end,
+    InitAcc = [],
+    IndexFold = 
+        fun() ->
+            leveled_bookie:book_indexfold(Bookie1,
+                                            B0,
+                                            {FoldFun, InitAcc},
+                                            {<<"temp_int">>, 5, 8},
+                                            {true, undefined})
+        end,
+
+    {async, Folder1} = IndexFold(),
+    QR1 = Folder1(),
+    true = lists:sort(QR1) == LoadedEntriesInRange,
+    % Replace object with one with an index value of 6
+    testutil:stdload_object(Bookie1, B0, K0, 6, <<"value">>,
+                            leveled_util:integer_now() + 600),
+    % Now replace again, shortening the timeout to 15s,
+    % this time index value of 6
+    testutil:stdload_object(Bookie1, B0, K0, 5, <<"value">>,
+                            leveled_util:integer_now() + 15),
+    {async, Folder2} = IndexFold(),
+        leveled_bookie:book_indexfold(Bookie1,
+                                        B0,
+                                        {FoldFun, InitAcc},
+                                        {<<"temp_int">>, 5, 8},
+                                        {true, undefined}),
+    QR2 = Folder2(),
+    io:format("Query with additional entry length ~w~n", [length(QR2)]),
+    true = lists:sort(QR2) == lists:sort([{5, B0, K0}|LoadedEntriesInRange]),
+    % Wait for a 15s timeout + lus a second to be sure
+    timer:sleep(15000 + 1000),
+    {async, Folder3} = IndexFold(),
+    QR3 = Folder3(),
+    % Now the entry should be missing (and the 600s TTL entry should not have
+    % resurfaced)
+    io:format("Query results length ~w following sleep~n", [length(QR3)]),
+    true = lists:sort(QR3) == LoadedEntriesInRange,
+
+    FoldTime = timer:now_diff(os:timestamp(), SW1)/1000000 - LoadTime,
+    io:format("Query returned ~w entries in ~w seconds - 3 queries + 15s wait~n",
+                [length(QR1), FoldTime]),
+    true = (LoadTime + FoldTime) < Future,
+    SleepTime = round((Future - (LoadTime + FoldTime)) * 1000 + 1000), % add a second
+    io:format("Sleeping ~w s for all to expire~n", [SleepTime/1000]),
+    timer:sleep(SleepTime),
+
+    % Index entries should now have expired
+    {async, Folder4} = IndexFold(),
+    QR4 = Folder4(),
+    true = QR4 == [],
+
+    ok = leveled_bookie:book_close(Bookie1),
+    testutil:reset_filestructure().
+
+
 breaking_folds(_Config) ->
     % Run various iterators and show that they can be broken by throwing an
     % exception from within the fold