Mas d31 i413 (#415)

* Allow snapshots to be reused in queries

Allow for a full bookie snapshot to be re-used for multiple queries, not just KV fetches.

* Reduce log noise

The internal dummy tag is expected so should not prompt a log on reload

* Snapshot should have same status of active db

wrt head_only and head_lookup

* Allow logging to specified on snapshots

* Shutdown snapshot bookie is primary goes down

Inker and Penciller already will shut down based on `erlang:monitor/2`

* Review feedback

Formatting and code readability fixes

test/end_to_end/tictac_SUITE.erl

@@ -738,11 +738,45 @@ basic_headonly_test(ObjectCount, RemoveCount, HeadOnly) ->
                                                 Bucket0, 
                                                 Key0),
             CheckHeadFun = 
-                fun({add, SegID, B, K, H}) ->
-                    {ok, H} = 
-                        leveled_bookie:book_headonly(Bookie1, SegID, B, K)
+                fun(DB) ->
+                    fun({add, SegID, B, K, H}) ->
+                        {ok, H} = 
+                            leveled_bookie:book_headonly(DB, SegID, B, K)
+                    end
                 end,
-            lists:foreach(CheckHeadFun, ObjectSpecL);
+            lists:foreach(CheckHeadFun(Bookie1), ObjectSpecL),
+            {ok, Snapshot} =
+                leveled_bookie:book_start([{snapshot_bookie, Bookie1}]),
+            ok = leveled_bookie:book_loglevel(Snapshot, warn),
+            ok =
+                leveled_bookie:book_addlogs(
+                    Snapshot, [b0001, b0002, b0003, i0027, p0007]
+                ),
+            ok =
+                leveled_bookie:book_removelogs(
+                    Snapshot, [b0019]
+                ),
+            io:format(
+                "Checking for ~w objects against Snapshot ~w~n",
+                [length(ObjectSpecL), Snapshot]),
+            lists:foreach(CheckHeadFun(Snapshot), ObjectSpecL),
+            io:format("Closing snapshot ~w~n", [Snapshot]),
+            ok = leveled_bookie:book_close(Snapshot),
+            {ok, AltSnapshot} =
+                leveled_bookie:book_start([{snapshot_bookie, Bookie1}]),
+            ok =
+                leveled_bookie:book_addlogs(
+                    AltSnapshot, [b0001, b0002, b0003, b0004, i0027, p0007]
+                ),
+            true = is_process_alive(AltSnapshot),
+            io:format(
+                "Closing actual store ~w with snapshot ~w open~n",
+                [Bookie1, AltSnapshot]
+            ),
+            ok = leveled_bookie:book_close(Bookie1),
+            % Sleep a beat so as not to race with the 'DOWN' message
+            timer:sleep(10),
+            false = is_process_alive(AltSnapshot);
         no_lookup ->
             {unsupported_message, head} = 
                 leveled_bookie:book_head(Bookie1, 
@@ -753,11 +787,11 @@ basic_headonly_test(ObjectCount, RemoveCount, HeadOnly) ->
                 leveled_bookie:book_headonly(Bookie1, 
                                                 SegmentID0, 
                                                 Bucket0, 
-                                                Key0)
+                                                Key0),
+            io:format("Closing actual store ~w~n", [Bookie1]),
+            ok = leveled_bookie:book_close(Bookie1)
     end,
-
-
-    ok = leveled_bookie:book_close(Bookie1),
+    
     {ok, FinalJournals} = file:list_dir(JFP),
     io:format("Trim has reduced journal count from " ++ 
                     "~w to ~w and ~w after restart~n",