Handle missing journal files without data loss

Make sure data added after the journal has been lost will not be lost (even though lost data may stay lost)

src/leveled_bookie.erl

@@ -1419,6 +1419,7 @@ startup(InkerOpts, PencillerOpts, State) ->
                                     LedgerSQN + 1,
                                     get_loadfun(State),
                                     Penciller),
+    ok = leveled_inker:ink_checksqn(Inker, LedgerSQN),
     {Inker, Penciller}.
 
 

src/leveled_inker.erl

@@ -115,6 +115,7 @@
         ink_doom/1,
         ink_roll/1,
         ink_backup/2,
+        ink_checksqn/2,
         build_dummy_journal/0,
         clean_testdir/1,
         filepath/2,
@@ -432,6 +433,12 @@ ink_updatemanifest(Pid, ManifestSnippet, DeletedFiles) ->
 ink_printmanifest(Pid) ->
     gen_server:call(Pid, print_manifest, infinity).
 
+-spec ink_checksqn(pid(), integer()) -> ok.
+%% @doc
+%% Check that the Inker doesn't have a SQN behind that of the Ledger
+ink_checksqn(Pid, LedgerSQN) ->
+    gen_server:call(Pid, {check_sqn, LedgerSQN}).
+
 %%%============================================================================
 %%% gen_server callbacks
 %%%============================================================================
@@ -633,6 +640,14 @@ handle_call({backup, BackupPath}, _from, State)
                                 length(BackupManifest)], 
                             SW),
     {reply, ok, State};
+handle_call({check_sqn, LedgerSQN}, _From, State) ->
+    case State#state.journal_sqn of
+        JSQN when JSQN < LedgerSQN ->
+            leveled_log:log("I0025", [JSQN, LedgerSQN]),
+            {reply, ok, State#state{journal_sqn = LedgerSQN}};
+        _JSQN ->
+            {reply, ok, State}
+    end;
 handle_call(close, _From, State) ->
     case State#state.is_snapshot of
         true ->

src/leveled_log.erl

@@ -290,6 +290,9 @@
         {info, "Backup commencing into folder with ~w existing files"}},
     {"I0024",
         {info, "Prompted roll at NewSQN=~w"}},
+    {"I0025",
+        {warn, "Journal SQN of ~w is below Ledger SQN of ~w " ++
+                "anti-entropy will be required"}},
 
     {"IC001",
         {info, "Closed for reason ~w so maybe leaving garbage"}},

test/end_to_end/basic_SUITE.erl

@@ -17,17 +17,17 @@
             ]).
 
 all() -> [
-            % simple_put_fetch_head_delete,
+            simple_put_fetch_head_delete,
-            % many_put_fetch_head,
+            many_put_fetch_head,
-            % journal_compaction,
+            journal_compaction,
-            % fetchput_snapshot,
+            fetchput_snapshot,
-            % load_and_count,
+            load_and_count,
-            % load_and_count_withdelete,
+            load_and_count_withdelete,
-            % space_clear_ondelete,
+            space_clear_ondelete,
-            % is_empty_test,
+            is_empty_test,
-            % many_put_fetch_switchcompression,
+            many_put_fetch_switchcompression,
-            % bigjournal_littlejournal,
+            bigjournal_littlejournal,
-            % safereaderror_startup,
+            safereaderror_startup,
             remove_journal_test
             ].