Penciller Refactor

Removed o(100) lines of code by refactoring the Penciller to no longer
use ETS tables.  The code is less confusing, and probably not an awful
lot slower.

src/leveled_bookie.erl

@@ -268,7 +268,8 @@ init([Opts]) ->
             {ok,
                 {Penciller, LedgerCache},
                 Inker} = book_snapshotstore(Bookie, self(), ?SNAPSHOT_TIMEOUT),
-            ok = leveled_penciller:pcl_loadsnapshot(Penciller, []),
+            ok = leveled_penciller:pcl_loadsnapshot(Penciller,
+                                                    gb_trees:empty()),
             io:format("Snapshot starting with Pcl ~w Ink ~w~n",
                                                 [Penciller, Inker]),
             {ok, #state{penciller=Penciller,
@@ -431,11 +432,10 @@ bucket_stats(Penciller, LedgerCache, Bucket, Tag) ->
                                     source_penciller=Penciller},
     {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts),
     Folder = fun() ->
-                Increment = gb_trees:to_list(LedgerCache),
                 io:format("Length of increment in snapshot is ~w~n",
-                            [length(Increment)]),
+                            [gb_trees:size(LedgerCache)]),
                 ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot,
-                                                        {infinity, Increment}),   
+                                                            LedgerCache),
                 StartKey = leveled_codec:to_ledgerkey(Bucket, null, Tag),
                 EndKey = leveled_codec:to_ledgerkey(Bucket, null, Tag),
                 Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot,
@@ -456,11 +456,10 @@ index_query(Penciller, LedgerCache,
                                     source_penciller=Penciller},
     {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts),
     Folder = fun() ->
-                Increment = gb_trees:to_list(LedgerCache),
                 io:format("Length of increment in snapshot is ~w~n",
-                            [length(Increment)]),
+                            [gb_trees:size(LedgerCache)]),
                 ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot,
-                                                        {infinity, Increment}),   
+                                                            LedgerCache),
                 StartKey = leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG,
                                                         IdxField, StartValue),
                 EndKey = leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG,
@@ -487,11 +486,10 @@ allkey_query(Penciller, LedgerCache, Tag) ->
                                     source_penciller=Penciller},
     {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts),
     Folder = fun() ->
-                Increment = gb_trees:to_list(LedgerCache),
                 io:format("Length of increment in snapshot is ~w~n",
-                            [length(Increment)]),
+                            [gb_trees:size(LedgerCache)]),
                 ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot,
-                                                        {infinity, Increment}),   
+                                                            LedgerCache),
                 SK = leveled_codec:to_ledgerkey(null, null, Tag),
                 EK = leveled_codec:to_ledgerkey(null, null, Tag),
                 Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot,
@@ -648,13 +646,10 @@ maybepush_ledgercache(MaxCacheSize, Cache, Penciller) ->
     TimeToPush = maybe_withjitter(CacheSize, MaxCacheSize),
     if
         TimeToPush ->
-            Dump = gb_trees:to_list(Cache),
+            case leveled_penciller:pcl_pushmem(Penciller, Cache) of
-            case leveled_penciller:pcl_pushmem(Penciller, Dump) of
                 ok ->
                     {ok, gb_trees:empty()};
-                pause ->
+                {returned, _Reason} ->
-                    {pause, gb_trees:empty()};
-                returned ->
                     {ok, Cache}
             end;
         true ->
@@ -794,7 +789,7 @@ multi_key_test() ->
     {ok, F2B} = book_riakget(Bookie1, B2, K2),
     ?assertMatch(F2B, Obj2),
     ok = book_close(Bookie1),
-    %% Now reopen the file, and confirm that a fetch is still possible
+    % Now reopen the file, and confirm that a fetch is still possible
     {ok, Bookie2} = book_start(#bookie_options{root_path=RootPath}),
     {ok, F1C} = book_riakget(Bookie2, B1, K1),
     ?assertMatch(F1C, Obj1),

src/leveled_inker.erl

@@ -261,15 +261,15 @@ handle_call({load_pcl, StartSQN, FilterFun, Penciller}, _From, State) ->
 handle_call({register_snapshot, Requestor}, _From , State) ->
     Rs = [{Requestor,
             State#state.manifest_sqn}|State#state.registered_snapshots],
-    io:format("Inker snapshot ~w registered at SQN ~w~n",
+    io:format("Journal snapshot ~w registered at SQN ~w~n",
                     [Requestor, State#state.manifest_sqn]),
     {reply, {State#state.manifest,
                 State#state.active_journaldb},
                 State#state{registered_snapshots=Rs}};
 handle_call({release_snapshot, Snapshot}, _From , State) ->
     Rs = lists:keydelete(Snapshot, 1, State#state.registered_snapshots),
-    io:format("Ledger snapshot ~w released~n", [Snapshot]),
+    io:format("Journal snapshot ~w released~n", [Snapshot]),
-    io:format("Remaining ledger snapshots are ~w~n", [Rs]),
+    io:format("Remaining journal snapshots are ~w~n", [Rs]),
     {reply, ok, State#state{registered_snapshots=Rs}};
 handle_call({confirm_delete, ManSQN}, _From, State) ->
     Reply = lists:foldl(fun({_R, SnapSQN}, Bool) ->
@@ -646,15 +646,12 @@ load_between_sequence(MinSQN, MaxSQN, FilterFun, Penciller,
 push_to_penciller(Penciller, KeyTree) ->
     % The push to penciller must start as a tree to correctly de-duplicate
     % the list by order before becoming a de-duplicated list for loading
-    KeyList = gb_trees:to_list(KeyTree),
+    R = leveled_penciller:pcl_pushmem(Penciller, KeyTree),
-    R = leveled_penciller:pcl_pushmem(Penciller, KeyList),
+    case R of
-    if
+        {returned, _Reason} ->
-        R == pause ->
-            timer:sleep(?LOADING_PAUSE);
-        R == returned ->
             timer:sleep(?LOADING_PAUSE),
             push_to_penciller(Penciller, KeyTree);
-        R == ok ->
+        ok ->
             ok
     end.
             
@@ -742,8 +739,7 @@ initiate_penciller_snapshot(Bookie) ->
     {ok,
         {LedgerSnap, LedgerCache},
         _} = leveled_bookie:book_snapshotledger(Bookie, self(), undefined),
-    ok = leveled_penciller:pcl_loadsnapshot(LedgerSnap,
+    ok = leveled_penciller:pcl_loadsnapshot(LedgerSnap, LedgerCache),
-                                            gb_trees:to_list(LedgerCache)),
     MaxSQN = leveled_penciller:pcl_getstartupsequencenumber(LedgerSnap),
     {LedgerSnap, MaxSQN}.
 

src/leveled_pclerk.erl

@@ -59,9 +59,11 @@
         handle_cast/2,
         handle_info/2,
         terminate/2,
-        clerk_new/1,
+        clerk_new/2,
-        clerk_prompt/1,
+        mergeclerk_prompt/1,
-        clerk_manifestchange/3,
+        mergeclerk_manifestchange/3,
+        rollclerk_levelzero/5,
+        rollclerk_close/1,
         code_change/3]).      
 
 -include_lib("eunit/include/eunit.hrl").
@@ -71,24 +73,33 @@
 
 -record(state, {owner :: pid(),
                 change_pending=false :: boolean(),
-                work_item :: #penciller_work{}|null}).
+                work_item :: #penciller_work{}|null,
+                merge_clerk = false :: boolean(),
+                roll_clerk = false ::boolean()}).
 
 %%%============================================================================
 %%% API
 %%%============================================================================
 
-clerk_new(Owner) ->
+clerk_new(Owner, Type) ->
     {ok, Pid} = gen_server:start(?MODULE, [], []),
-    ok = gen_server:call(Pid, {register, Owner}, infinity),
+    ok = gen_server:call(Pid, {register, Owner, Type}, infinity),
     io:format("Penciller's clerk ~w started with owner ~w~n", [Pid, Owner]),
     {ok, Pid}.
 
-clerk_manifestchange(Pid, Action, Closing) ->
+mergeclerk_manifestchange(Pid, Action, Closing) ->
     gen_server:call(Pid, {manifest_change, Action, Closing}, infinity).
 
-clerk_prompt(Pid) ->
+mergeclerk_prompt(Pid) ->
     gen_server:cast(Pid, prompt).
 
+rollclerk_levelzero(Pid, LevelZero, LevelMinus1, LedgerSQN, PCL) ->
+    gen_server:cast(Pid,
+                    {roll_levelzero, LevelZero, LevelMinus1, LedgerSQN, PCL}).
+
+rollclerk_close(Pid) ->
+    gen_server:call(Pid, close, infinity).
+
 %%%============================================================================
 %%% gen_server callbacks
 %%%============================================================================
@@ -96,8 +107,18 @@ clerk_prompt(Pid) ->
 init([]) ->
     {ok, #state{}}.
 
-handle_call({register, Owner}, _From, State) ->
+handle_call({register, Owner, Type}, _From, State) ->
-    {reply, ok, State#state{owner=Owner}, ?MIN_TIMEOUT};
+    case Type of
+        merge ->
+            {reply,
+                ok,
+                State#state{owner=Owner, merge_clerk = true},
+                ?MIN_TIMEOUT};
+        roll ->
+            {reply,
+                ok,
+                State#state{owner=Owner, roll_clerk = true}}
+    end;
 handle_call({manifest_change, return, true}, _From, State) ->
     io:format("Request for manifest change from clerk on closing~n"),
     case State#state.change_pending of
@@ -124,20 +145,34 @@ handle_call({manifest_change, confirm, Closing}, From, State) ->
             {noreply,
                 State#state{work_item=null, change_pending=false},
                 ?MIN_TIMEOUT}
-    end.
+    end;
+handle_call(close, _From, State) ->
+    {stop, normal, ok, State}.
 
 handle_cast(prompt, State) ->
-    {noreply, State, ?MIN_TIMEOUT}.
+    {noreply, State, ?MIN_TIMEOUT};
+handle_cast({roll_levelzero, LevelZero, LevelMinus1, LedgerSQN, PCL}, State) ->
+    SW = os:timestamp(),
+    {NewL0, Size, MaxSQN} = leveled_penciller:roll_new_tree(LevelZero,
+                                                            LevelMinus1,
+                                                            LedgerSQN),
+    ok = leveled_penciller:pcl_updatelevelzero(PCL, NewL0, Size, MaxSQN),
+    io:format("Rolled tree to size ~w in ~w microseconds~n",
+                [Size, timer:now_diff(os:timestamp(), SW)]),
+    {noreply, State}.
 
 handle_info(timeout, State=#state{change_pending=Pnd}) when Pnd == false ->
-    case requestandhandle_work(State) of
+    if
-        {false, Timeout} ->
+        State#state.merge_clerk ->
-            {noreply, State, Timeout};
+            case requestandhandle_work(State) of
-        {true, WI} ->
+                {false, Timeout} ->
-            % No timeout now as will wait for call to return manifest
+                    {noreply, State, Timeout};
-            % change
+                {true, WI} ->
-            {noreply,
+                    % No timeout now as will wait for call to return manifest
-                State#state{change_pending=true, work_item=WI}}
+                    % change
+                    {noreply,
+                        State#state{change_pending=true, work_item=WI}}
+            end
     end.
 
 terminate(Reason, _State) ->

src/leveled_sft.erl

@@ -234,7 +234,7 @@ sft_new(Filename, KL1, KL2, LevelInfo, Options) ->
         false ->
             gen_server:cast(Pid,
                             {sft_new, Filename, KL1, KL2, LevelR}),
-            {ok, Pid}
+            {ok, Pid, noreply}
     end.
 
 sft_open(Filename) ->
@@ -532,25 +532,29 @@ complete_file(Handle, FileMD, KL1, KL2, LevelR, Rename) ->
         false ->
             open_file(FileMD);
         {true, OldName, NewName} ->
-            io:format("Renaming file from ~s to ~s~n", [OldName, NewName]),
+            ok = rename_file(OldName, NewName),
-            case filelib:is_file(NewName) of
-                true ->
-                    io:format("Filename ~s already exists~n",
-                                    [NewName]),
-                    AltName = filename:join(filename:dirname(NewName),
-                                            filename:basename(NewName))
-                                ++ ?DISCARD_EXT,
-                    io:format("Rename rogue filename ~s to ~s~n",
-                                    [NewName, AltName]),
-                    ok = file:rename(NewName, AltName);
-                false ->
-                    ok
-            end,
-            ok = file:rename(OldName, NewName),
             open_file(FileMD#state{filename=NewName})
     end,    
     {ReadHandle, UpdFileMD, KeyRemainders}.
 
+rename_file(OldName, NewName) ->
+    io:format("Renaming file from ~s to ~s~n", [OldName, NewName]),
+    case filelib:is_file(NewName) of
+        true ->
+            io:format("Filename ~s already exists~n",
+                            [NewName]),
+            AltName = filename:join(filename:dirname(NewName),
+                                    filename:basename(NewName))
+                        ++ ?DISCARD_EXT,
+            io:format("Rename rogue filename ~s to ~s~n",
+                            [NewName, AltName]),
+            ok = file:rename(NewName, AltName);
+        false ->
+            ok
+    end,
+    file:rename(OldName, NewName).
+
+
 %% Fetch a Key and Value from a file, returns
 %% {value, KV} or not_present
 %% The key must be pre-checked to ensure it is in the valid range for the file