From b2089baa1e413db6750729349cf20a211190f2ed Mon Sep 17 00:00:00 2001
From: martinsumner <martin.sumner@adaptip.co.uk>
Date: Fri, 21 Oct 2016 15:21:37 +0100
Subject: [PATCH] Correct tombstone handling

Prepare SFT files for handling tombstones correctly (without expiry
dates).

Also some work as it can be seen from tests that some SFT files ar enot
be cleared out correctly.  Pausing before trying t clear out the fles to
experiment and trial the possibility that there is a timing issue.
---
 src/leveled_sft.erl             | 17 ++++++++++++++++-
 test/end_to_end/basic_SUITE.erl | 12 ++++++------
 test/end_to_end/testutil.erl    |  2 ++
 3 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl
index 93d22d8..14ba7df 100644
--- a/src/leveled_sft.erl
+++ b/src/leveled_sft.erl
@@ -1101,6 +1101,8 @@ maybe_reap_expiredkey({_, infinity}, _) ->
     false; % key is not set to expire
 maybe_reap_expiredkey({_, TS}, {basement, CurrTS}) when CurrTS > TS ->
     true; % basement and ready to expire
+maybe_reap_expiredkey(tomb, {basement, _CurrTS}) ->
+    true; % always expire in basement
 maybe_reap_expiredkey(_, _) ->
     false.
 
@@ -1760,6 +1762,7 @@ key_dominates_test() ->
     KV4 = {{o, "Bucket", "Key4", null}, {7, {active, infinity}, []}},
     KV5 = {{o, "Bucket", "Key1", null}, {4, {active, infinity}, []}},
     KV6 = {{o, "Bucket", "Key1", null}, {99, {tomb, 999}, []}},
+    KV7 = {{o, "Bucket", "Key1", null}, {99, tomb, []}},
     KL1 = [KV1, KV2],
     KL2 = [KV3, KV4],
     ?assertMatch({{next_key, KV1}, [KV2], KL2},
@@ -1789,7 +1792,19 @@ key_dominates_test() ->
     ?assertMatch({{next_key, KV6}, [], []},
                     key_dominates([KV6], [], {basement, 1})),
     ?assertMatch({{next_key, KV6}, [], []},
-                    key_dominates([], [KV6], {basement, 1})).
+                    key_dominates([], [KV6], {basement, 1})),
+    ?assertMatch({skipped_key, [], []},
+                    key_dominates([KV7], [], {basement, 1})),
+    ?assertMatch({skipped_key, [], []},
+                    key_dominates([], [KV7], {basement, 1})),
+    ?assertMatch({skipped_key, [KV7|KL2], [KV2]},
+                    key_dominates([KV7|KL2], KL1, 1)),
+    ?assertMatch({{next_key, KV7}, KL2, [KV2]},
+                    key_dominates([KV7|KL2], [KV2], 1)),
+    ?assertMatch({skipped_key, [KV7|KL2], [KV2]},
+                    key_dominates([KV7|KL2], KL1, {basement, 1})),
+    ?assertMatch({skipped_key, KL2, [KV2]},
+                    key_dominates([KV7|KL2], [KV2], {basement, 1})).
 
 
 big_iterator_test() ->
diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl
index cfc4b0c..718587c 100644
--- a/test/end_to_end/basic_SUITE.erl
+++ b/test/end_to_end/basic_SUITE.erl
@@ -11,12 +11,12 @@
             ]).
 
 all() -> [
-            % simple_put_fetch_head_delete,
-            % many_put_fetch_head,
-            % journal_compaction,
-            fetchput_snapshot %,
-            % load_and_count,
-            % load_and_count_withdelete
+            simple_put_fetch_head_delete,
+            many_put_fetch_head,
+            journal_compaction,
+            fetchput_snapshot,
+            load_and_count,
+            load_and_count_withdelete
             ].
 
 
diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl
index e3bf77d..9043bf9 100644
--- a/test/end_to_end/testutil.erl
+++ b/test/end_to_end/testutil.erl
@@ -26,6 +26,8 @@
 
 
 reset_filestructure() ->
+    io:format("Waiting 2s to give a chance for all file closes to complete~n"),
+    timer:sleep(2000),
     RootPath  = "test",
     filelib:ensure_dir(RootPath ++ "/journal/"),
     filelib:ensure_dir(RootPath ++ "/ledger/"),