Correct tombstone handling

Prepare SFT files for handling tombstones correctly (without expiry
dates).

Also some work as it can be seen from tests that some SFT files ar enot
be cleared out correctly.  Pausing before trying t clear out the fles to
experiment and trial the possibility that there is a timing issue.

src/leveled_sft.erl

@@ -1101,6 +1101,8 @@ maybe_reap_expiredkey({_, infinity}, _) ->
     false; % key is not set to expire
 maybe_reap_expiredkey({_, TS}, {basement, CurrTS}) when CurrTS > TS ->
     true; % basement and ready to expire
+maybe_reap_expiredkey(tomb, {basement, _CurrTS}) ->
+    true; % always expire in basement
 maybe_reap_expiredkey(_, _) ->
     false.
 
@@ -1760,6 +1762,7 @@ key_dominates_test() ->
     KV4 = {{o, "Bucket", "Key4", null}, {7, {active, infinity}, []}},
     KV5 = {{o, "Bucket", "Key1", null}, {4, {active, infinity}, []}},
     KV6 = {{o, "Bucket", "Key1", null}, {99, {tomb, 999}, []}},
+    KV7 = {{o, "Bucket", "Key1", null}, {99, tomb, []}},
     KL1 = [KV1, KV2],
     KL2 = [KV3, KV4],
     ?assertMatch({{next_key, KV1}, [KV2], KL2},
@@ -1789,7 +1792,19 @@ key_dominates_test() ->
     ?assertMatch({{next_key, KV6}, [], []},
                     key_dominates([KV6], [], {basement, 1})),
     ?assertMatch({{next_key, KV6}, [], []},
-                    key_dominates([], [KV6], {basement, 1})).
+                    key_dominates([], [KV6], {basement, 1})),
+    ?assertMatch({skipped_key, [], []},
+                    key_dominates([KV7], [], {basement, 1})),
+    ?assertMatch({skipped_key, [], []},
+                    key_dominates([], [KV7], {basement, 1})),
+    ?assertMatch({skipped_key, [KV7|KL2], [KV2]},
+                    key_dominates([KV7|KL2], KL1, 1)),
+    ?assertMatch({{next_key, KV7}, KL2, [KV2]},
+                    key_dominates([KV7|KL2], [KV2], 1)),
+    ?assertMatch({skipped_key, [KV7|KL2], [KV2]},
+                    key_dominates([KV7|KL2], KL1, {basement, 1})),
+    ?assertMatch({skipped_key, KL2, [KV2]},
+                    key_dominates([KV7|KL2], [KV2], {basement, 1})).
 
 
 big_iterator_test() ->

test/end_to_end/basic_SUITE.erl

@@ -11,12 +11,12 @@
             ]).
 
 all() -> [
-            % simple_put_fetch_head_delete,
-            % many_put_fetch_head,
-            % journal_compaction,
-            fetchput_snapshot %,
-            % load_and_count,
-            % load_and_count_withdelete
+            simple_put_fetch_head_delete,
+            many_put_fetch_head,
+            journal_compaction,
+            fetchput_snapshot,
+            load_and_count,
+            load_and_count_withdelete
             ].
 
 

test/end_to_end/testutil.erl

@@ -26,6 +26,8 @@
 
 
 reset_filestructure() ->
+    io:format("Waiting 2s to give a chance for all file closes to complete~n"),
+    timer:sleep(2000),
     RootPath  = "test",
     filelib:ensure_dir(RootPath ++ "/journal/"),
     filelib:ensure_dir(RootPath ++ "/ledger/"),