Penciller Refactor
Removed o(100) lines of code by refactoring the Penciller to no longer use ETS tables. The code is less confusing, and probably not an awful lot slower.
This commit is contained in:
martinsumner
parent
30f4f2edf6
commit
20cc17f916
5 changed files with 465 additions and 622 deletions
|
|
@ -268,7 +268,8 @@ init([Opts]) ->
|
|||
{ok,
|
||||
{Penciller, LedgerCache},
|
||||
Inker} = book_snapshotstore(Bookie, self(), ?SNAPSHOT_TIMEOUT),
|
||||
ok = leveled_penciller:pcl_loadsnapshot(Penciller, []),
|
||||
ok = leveled_penciller:pcl_loadsnapshot(Penciller,
|
||||
gb_trees:empty()),
|
||||
io:format("Snapshot starting with Pcl ~w Ink ~w~n",
|
||||
[Penciller, Inker]),
|
||||
{ok, #state{penciller=Penciller,
|
||||
|
|
@ -431,11 +432,10 @@ bucket_stats(Penciller, LedgerCache, Bucket, Tag) ->
|
|||
source_penciller=Penciller},
|
||||
{ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts),
|
||||
Folder = fun() ->
|
||||
Increment = gb_trees:to_list(LedgerCache),
|
||||
io:format("Length of increment in snapshot is ~w~n",
|
||||
[length(Increment)]),
|
||||
[gb_trees:size(LedgerCache)]),
|
||||
ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot,
|
||||
{infinity, Increment}),
|
||||
LedgerCache),
|
||||
StartKey = leveled_codec:to_ledgerkey(Bucket, null, Tag),
|
||||
EndKey = leveled_codec:to_ledgerkey(Bucket, null, Tag),
|
||||
Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot,
|
||||
|
|
@ -456,11 +456,10 @@ index_query(Penciller, LedgerCache,
|
|||
source_penciller=Penciller},
|
||||
{ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts),
|
||||
Folder = fun() ->
|
||||
Increment = gb_trees:to_list(LedgerCache),
|
||||
io:format("Length of increment in snapshot is ~w~n",
|
||||
[length(Increment)]),
|
||||
[gb_trees:size(LedgerCache)]),
|
||||
ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot,
|
||||
{infinity, Increment}),
|
||||
LedgerCache),
|
||||
StartKey = leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG,
|
||||
IdxField, StartValue),
|
||||
EndKey = leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG,
|
||||
|
|
@ -487,11 +486,10 @@ allkey_query(Penciller, LedgerCache, Tag) ->
|
|||
source_penciller=Penciller},
|
||||
{ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts),
|
||||
Folder = fun() ->
|
||||
Increment = gb_trees:to_list(LedgerCache),
|
||||
io:format("Length of increment in snapshot is ~w~n",
|
||||
[length(Increment)]),
|
||||
[gb_trees:size(LedgerCache)]),
|
||||
ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot,
|
||||
{infinity, Increment}),
|
||||
LedgerCache),
|
||||
SK = leveled_codec:to_ledgerkey(null, null, Tag),
|
||||
EK = leveled_codec:to_ledgerkey(null, null, Tag),
|
||||
Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot,
|
||||
|
|
@ -648,13 +646,10 @@ maybepush_ledgercache(MaxCacheSize, Cache, Penciller) ->
|
|||
TimeToPush = maybe_withjitter(CacheSize, MaxCacheSize),
|
||||
if
|
||||
TimeToPush ->
|
||||
Dump = gb_trees:to_list(Cache),
|
||||
case leveled_penciller:pcl_pushmem(Penciller, Dump) of
|
||||
case leveled_penciller:pcl_pushmem(Penciller, Cache) of
|
||||
ok ->
|
||||
{ok, gb_trees:empty()};
|
||||
pause ->
|
||||
{pause, gb_trees:empty()};
|
||||
returned ->
|
||||
{returned, _Reason} ->
|
||||
{ok, Cache}
|
||||
end;
|
||||
true ->
|
||||
|
|
@ -794,7 +789,7 @@ multi_key_test() ->
|
|||
{ok, F2B} = book_riakget(Bookie1, B2, K2),
|
||||
?assertMatch(F2B, Obj2),
|
||||
ok = book_close(Bookie1),
|
||||
%% Now reopen the file, and confirm that a fetch is still possible
|
||||
% Now reopen the file, and confirm that a fetch is still possible
|
||||
{ok, Bookie2} = book_start(#bookie_options{root_path=RootPath}),
|
||||
{ok, F1C} = book_riakget(Bookie2, B1, K1),
|
||||
?assertMatch(F1C, Obj1),
|
||||
|
|
|
|||
|
|
@ -261,15 +261,15 @@ handle_call({load_pcl, StartSQN, FilterFun, Penciller}, _From, State) ->
|
|||
handle_call({register_snapshot, Requestor}, _From , State) ->
|
||||
Rs = [{Requestor,
|
||||
State#state.manifest_sqn}|State#state.registered_snapshots],
|
||||
io:format("Inker snapshot ~w registered at SQN ~w~n",
|
||||
io:format("Journal snapshot ~w registered at SQN ~w~n",
|
||||
[Requestor, State#state.manifest_sqn]),
|
||||
{reply, {State#state.manifest,
|
||||
State#state.active_journaldb},
|
||||
State#state{registered_snapshots=Rs}};
|
||||
handle_call({release_snapshot, Snapshot}, _From , State) ->
|
||||
Rs = lists:keydelete(Snapshot, 1, State#state.registered_snapshots),
|
||||
io:format("Ledger snapshot ~w released~n", [Snapshot]),
|
||||
io:format("Remaining ledger snapshots are ~w~n", [Rs]),
|
||||
io:format("Journal snapshot ~w released~n", [Snapshot]),
|
||||
io:format("Remaining journal snapshots are ~w~n", [Rs]),
|
||||
{reply, ok, State#state{registered_snapshots=Rs}};
|
||||
handle_call({confirm_delete, ManSQN}, _From, State) ->
|
||||
Reply = lists:foldl(fun({_R, SnapSQN}, Bool) ->
|
||||
|
|
@ -646,15 +646,12 @@ load_between_sequence(MinSQN, MaxSQN, FilterFun, Penciller,
|
|||
push_to_penciller(Penciller, KeyTree) ->
|
||||
% The push to penciller must start as a tree to correctly de-duplicate
|
||||
% the list by order before becoming a de-duplicated list for loading
|
||||
KeyList = gb_trees:to_list(KeyTree),
|
||||
R = leveled_penciller:pcl_pushmem(Penciller, KeyList),
|
||||
if
|
||||
R == pause ->
|
||||
timer:sleep(?LOADING_PAUSE);
|
||||
R == returned ->
|
||||
R = leveled_penciller:pcl_pushmem(Penciller, KeyTree),
|
||||
case R of
|
||||
{returned, _Reason} ->
|
||||
timer:sleep(?LOADING_PAUSE),
|
||||
push_to_penciller(Penciller, KeyTree);
|
||||
R == ok ->
|
||||
ok ->
|
||||
ok
|
||||
end.
|
||||
|
||||
|
|
@ -742,8 +739,7 @@ initiate_penciller_snapshot(Bookie) ->
|
|||
{ok,
|
||||
{LedgerSnap, LedgerCache},
|
||||
_} = leveled_bookie:book_snapshotledger(Bookie, self(), undefined),
|
||||
ok = leveled_penciller:pcl_loadsnapshot(LedgerSnap,
|
||||
gb_trees:to_list(LedgerCache)),
|
||||
ok = leveled_penciller:pcl_loadsnapshot(LedgerSnap, LedgerCache),
|
||||
MaxSQN = leveled_penciller:pcl_getstartupsequencenumber(LedgerSnap),
|
||||
{LedgerSnap, MaxSQN}.
|
||||
|
||||
|
|
|
|||
|
|
@ -59,9 +59,11 @@
|
|||
handle_cast/2,
|
||||
handle_info/2,
|
||||
terminate/2,
|
||||
clerk_new/1,
|
||||
clerk_prompt/1,
|
||||
clerk_manifestchange/3,
|
||||
clerk_new/2,
|
||||
mergeclerk_prompt/1,
|
||||
mergeclerk_manifestchange/3,
|
||||
rollclerk_levelzero/5,
|
||||
rollclerk_close/1,
|
||||
code_change/3]).
|
||||
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
|
|
@ -71,24 +73,33 @@
|
|||
|
||||
-record(state, {owner :: pid(),
|
||||
change_pending=false :: boolean(),
|
||||
work_item :: #penciller_work{}|null}).
|
||||
work_item :: #penciller_work{}|null,
|
||||
merge_clerk = false :: boolean(),
|
||||
roll_clerk = false ::boolean()}).
|
||||
|
||||
%%%============================================================================
|
||||
%%% API
|
||||
%%%============================================================================
|
||||
|
||||
clerk_new(Owner) ->
|
||||
clerk_new(Owner, Type) ->
|
||||
{ok, Pid} = gen_server:start(?MODULE, [], []),
|
||||
ok = gen_server:call(Pid, {register, Owner}, infinity),
|
||||
ok = gen_server:call(Pid, {register, Owner, Type}, infinity),
|
||||
io:format("Penciller's clerk ~w started with owner ~w~n", [Pid, Owner]),
|
||||
{ok, Pid}.
|
||||
|
||||
clerk_manifestchange(Pid, Action, Closing) ->
|
||||
mergeclerk_manifestchange(Pid, Action, Closing) ->
|
||||
gen_server:call(Pid, {manifest_change, Action, Closing}, infinity).
|
||||
|
||||
clerk_prompt(Pid) ->
|
||||
mergeclerk_prompt(Pid) ->
|
||||
gen_server:cast(Pid, prompt).
|
||||
|
||||
rollclerk_levelzero(Pid, LevelZero, LevelMinus1, LedgerSQN, PCL) ->
|
||||
gen_server:cast(Pid,
|
||||
{roll_levelzero, LevelZero, LevelMinus1, LedgerSQN, PCL}).
|
||||
|
||||
rollclerk_close(Pid) ->
|
||||
gen_server:call(Pid, close, infinity).
|
||||
|
||||
%%%============================================================================
|
||||
%%% gen_server callbacks
|
||||
%%%============================================================================
|
||||
|
|
@ -96,8 +107,18 @@ clerk_prompt(Pid) ->
|
|||
init([]) ->
|
||||
{ok, #state{}}.
|
||||
|
||||
handle_call({register, Owner}, _From, State) ->
|
||||
{reply, ok, State#state{owner=Owner}, ?MIN_TIMEOUT};
|
||||
handle_call({register, Owner, Type}, _From, State) ->
|
||||
case Type of
|
||||
merge ->
|
||||
{reply,
|
||||
ok,
|
||||
State#state{owner=Owner, merge_clerk = true},
|
||||
?MIN_TIMEOUT};
|
||||
roll ->
|
||||
{reply,
|
||||
ok,
|
||||
State#state{owner=Owner, roll_clerk = true}}
|
||||
end;
|
||||
handle_call({manifest_change, return, true}, _From, State) ->
|
||||
io:format("Request for manifest change from clerk on closing~n"),
|
||||
case State#state.change_pending of
|
||||
|
|
@ -124,12 +145,25 @@ handle_call({manifest_change, confirm, Closing}, From, State) ->
|
|||
{noreply,
|
||||
State#state{work_item=null, change_pending=false},
|
||||
?MIN_TIMEOUT}
|
||||
end.
|
||||
end;
|
||||
handle_call(close, _From, State) ->
|
||||
{stop, normal, ok, State}.
|
||||
|
||||
handle_cast(prompt, State) ->
|
||||
{noreply, State, ?MIN_TIMEOUT}.
|
||||
{noreply, State, ?MIN_TIMEOUT};
|
||||
handle_cast({roll_levelzero, LevelZero, LevelMinus1, LedgerSQN, PCL}, State) ->
|
||||
SW = os:timestamp(),
|
||||
{NewL0, Size, MaxSQN} = leveled_penciller:roll_new_tree(LevelZero,
|
||||
LevelMinus1,
|
||||
LedgerSQN),
|
||||
ok = leveled_penciller:pcl_updatelevelzero(PCL, NewL0, Size, MaxSQN),
|
||||
io:format("Rolled tree to size ~w in ~w microseconds~n",
|
||||
[Size, timer:now_diff(os:timestamp(), SW)]),
|
||||
{noreply, State}.
|
||||
|
||||
handle_info(timeout, State=#state{change_pending=Pnd}) when Pnd == false ->
|
||||
if
|
||||
State#state.merge_clerk ->
|
||||
case requestandhandle_work(State) of
|
||||
{false, Timeout} ->
|
||||
{noreply, State, Timeout};
|
||||
|
|
@ -138,6 +172,7 @@ handle_info(timeout, State=#state{change_pending=Pnd}) when Pnd == false ->
|
|||
% change
|
||||
{noreply,
|
||||
State#state{change_pending=true, work_item=WI}}
|
||||
end
|
||||
end.
|
||||
|
||||
terminate(Reason, _State) ->
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -234,7 +234,7 @@ sft_new(Filename, KL1, KL2, LevelInfo, Options) ->
|
|||
false ->
|
||||
gen_server:cast(Pid,
|
||||
{sft_new, Filename, KL1, KL2, LevelR}),
|
||||
{ok, Pid}
|
||||
{ok, Pid, noreply}
|
||||
end.
|
||||
|
||||
sft_open(Filename) ->
|
||||
|
|
@ -532,6 +532,12 @@ complete_file(Handle, FileMD, KL1, KL2, LevelR, Rename) ->
|
|||
false ->
|
||||
open_file(FileMD);
|
||||
{true, OldName, NewName} ->
|
||||
ok = rename_file(OldName, NewName),
|
||||
open_file(FileMD#state{filename=NewName})
|
||||
end,
|
||||
{ReadHandle, UpdFileMD, KeyRemainders}.
|
||||
|
||||
rename_file(OldName, NewName) ->
|
||||
io:format("Renaming file from ~s to ~s~n", [OldName, NewName]),
|
||||
case filelib:is_file(NewName) of
|
||||
true ->
|
||||
|
|
@ -546,10 +552,8 @@ complete_file(Handle, FileMD, KL1, KL2, LevelR, Rename) ->
|
|||
false ->
|
||||
ok
|
||||
end,
|
||||
ok = file:rename(OldName, NewName),
|
||||
open_file(FileMD#state{filename=NewName})
|
||||
end,
|
||||
{ReadHandle, UpdFileMD, KeyRemainders}.
|
||||
file:rename(OldName, NewName).
|
||||
|
||||
|
||||
%% Fetch a Key and Value from a file, returns
|
||||
%% {value, KV} or not_present
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue