umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Resolve issue of Remainders

Browse source 
Two issues looked at
- There shouldn't be a remainder after writing the L0 file, as this
could have overlapping sequence numbers which will be missed on restart
- There should be a safety-check to stop the Clerk from doing a fake
push too soon after a background L0 file ahs been written (as the fake
push would lock the ledger waiting for the L0 file write to finish)
This commit is contained in:
martinsumner 2016-08-16 12:45:48 +01:00
parent 4586e2340a
commit ce0c55a2ec
3 changed files with 177 additions and 131 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/leveled_clerk.erl
View file

@ -251,7 +251,7 @@ do_merge(KL1, KL2, Level, {Filepath, MSN}, FileCounter, OutList) ->
io:format("File to be created as part of MSN=~w Filename=~s~n", io:format("File to be created as part of MSN=~w Filename=~s~n",
[MSN, FileName]), [MSN, FileName]),
TS1 = os:timestamp(), TS1 = os:timestamp(),
case leveled_sft:sft_new(FileName, KL1, KL2, Level) of case leveled_sft:sft_new(FileName, KL1, KL2, Level + 1) of
{ok, _Pid, {error, Reason}} -> {ok, _Pid, {error, Reason}} ->
io:format("Exiting due to error~w~n", [Reason]), io:format("Exiting due to error~w~n", [Reason]),
error; error;
@ -263,7 +263,7 @@ do_merge(KL1, KL2, Level, {Filepath, MSN}, FileCounter, OutList) ->
owner=Pid, owner=Pid,
filename=FileName}]), filename=FileName}]),
MTime = timer:now_diff(os:timestamp(), TS1), MTime = timer:now_diff(os:timestamp(), TS1),
io:format("file creation took ~w microseconds ~n", [MTime]), io:format("File creation took ~w microseconds ~n", [MTime]),
do_merge(KL1Rem, KL2Rem, Level, {Filepath, MSN}, do_merge(KL1Rem, KL2Rem, Level, {Filepath, MSN},
FileCounter + 1, ExtMan) FileCounter + 1, ExtMan)
end. end.

156
src/leveled_penciller.erl
View file

@ -1,29 +1,35 @@
%% -------- PENCILLER --------- %% -------- PENCILLER ---------
%% %%
%% The penciller is repsonsible for writing and re-writing the ledger - a %% The penciller is responsible for writing and re-writing the ledger - a
%% persisted, ordered view of non-recent Keys and Metadata which have been %% persisted, ordered view of non-recent Keys and Metadata which have been
%% added to the store. %% added to the store.
%% - The penciller maintains a manifest of all the files within the current %% - The penciller maintains a manifest of all the files within the current
%% Ledger. %% Ledger.
%% - The Penciller queues re-write (compaction) work up to be managed by Clerks %% - The Penciller provides re-write (compaction) work up to be managed by
%% the Penciller's Clerk
%% - The Penciller mainatins a register of iterators who have requested %% - The Penciller mainatins a register of iterators who have requested
%% snapshots of the Ledger %% snapshots of the Ledger
%% - The accepts new dumps (in the form of immutable ets tables) from the %% - The accepts new dumps (in the form of lists of keys) from the Bookie, and
%% Bookie, and calls the Bookie once the process of pencilling this data in %% calls the Bookie once the process of pencilling this data in the Ledger is
%% the Ledger is complete - and the Bookie is free to forget about the data %% complete - and the Bookie is free to forget about the data
%% %%
%% -------- LEDGER --------- %% -------- LEDGER ---------
%% %%
%% The Ledger is divided into many levels %% The Ledger is divided into many levels
%% L0: ETS tables are received from the Bookie and merged into a single ETS %% - L0: New keys are received from the Bookie and merged into a single ETS
%% table, until that table is the size of a SFT file, and it is then persisted %% table, until that table is the size of a SFT file, and it is then persisted
%% as a SFT file at this level. Once the persistence is completed, the ETS %% as a SFT file at this level. L0 SFT files can be larger than the normal
%% table can be dropped. There can be only one SFT file at Level 0, so %% maximum size - so we don't have to consider problems of either having more
%% the work to merge that file to the lower level must be the highest priority, %% than one L0 file (and handling what happens on a crash between writing the
%% as otherwise the database will stall. %% files when the second may have overlapping sequence numbers), or having a
%% L1 TO L7: May contain multiple non-overlapping PIDs managing sft files. %% remainder with overlapping in sequence numbers in memory after the file is
%% Compaction work should be sheduled if the number of files exceeds the target %% written. Once the persistence is completed, the ETS table can be erased.
%% size of the level, where the target size is 8 ^ n. %% There can be only one SFT file at Level 0, so the work to merge that file
%% to the lower level must be the highest priority, as otherwise writes to the
%% ledger will stall, when there is next a need to persist.
%% - L1 TO L7: May contain multiple processes managing non-overlapping sft
%% files. Compaction work should be sheduled if the number of files exceeds
%% the target size of the level, where the target size is 8 ^ n.
%% %%
%% The most recent revision of a Key can be found by checking each level until %% The most recent revision of a Key can be found by checking each level until
%% the key is found. To check a level the correct file must be sought from the %% the key is found. To check a level the correct file must be sought from the
@ -33,28 +39,30 @@
%% If a compaction change takes the size of a level beyond the target size, %% If a compaction change takes the size of a level beyond the target size,
%% then compaction work for that level + 1 should be added to the compaction %% then compaction work for that level + 1 should be added to the compaction
%% work queue. %% work queue.
%% Compaction work is fetched by the Pencllier's Clerk because: %% Compaction work is fetched by the Penciller's Clerk because:
%% - it has timed out due to a period of inactivity %% - it has timed out due to a period of inactivity
%% - it has been triggered by the a cast to indicate the arrival of high %% - it has been triggered by the a cast to indicate the arrival of high
%% priority compaction work %% priority compaction work
%% The Penciller's Clerk (which performs compaction worker) will always call %% The Penciller's Clerk (which performs compaction worker) will always call
%% the Penciller to find out the highest priority work currently in the queue %% the Penciller to find out the highest priority work currently required
%% whenever it has either completed work, or a timeout has occurred since it %% whenever it has either completed work, or a timeout has occurred since it
%% was informed there was no work to do. %% was informed there was no work to do.
%% %%
%% When the clerk picks work off the queue it will take the current manifest %% When the clerk picks work it will take the current manifest, and the
%% for the level and level - 1. The clerk will choose which file to compact %% Penciller assumes the manifest sequence number is to be incremented.
%% from level - 1, and once the compaction is complete will call to the %% When the clerk has completed the work it cna request that the manifest
%% Penciller with the new version of the manifest to be written. %% change be committed by the Penciller. The commit is made through changing
%% the filename of the new manifest - so the Penciller is not held up by the
%% process of wiritng a file, just altering file system metadata.
%% %%
%% Once the new version of the manifest had been persisted, the state of any %% The manifest is locked by a clerk taking work, or by there being a need to
%% deleted files will be changed to pending deletion. In pending deletion they %% write a file to Level 0. If the manifest is locked, then new keys can still
%% will call the Penciller on a timeout to confirm that they are no longer in %% be added in memory - however, the response to that push will be to "pause",
%% use (by any iterators). %% that is to say the Penciller will ask the Bookie to slowdown.
%% %%
%% ---------- PUSH ---------- %% ---------- PUSH ----------
%% %%
%% The Penciller must support the PUSH of an ETS table from the Bookie. The %% The Penciller must support the PUSH of a dump of keys from the Bookie. The
%% call to PUSH should be immediately acknowledged, and then work should be %% call to PUSH should be immediately acknowledged, and then work should be
%% completed to merge the ETS table into the L0 ETS table. %% completed to merge the ETS table into the L0 ETS table.
%% %%
@ -177,7 +185,8 @@
-define(ARCHIVE_FILEX, "arc"). -define(ARCHIVE_FILEX, "arc").
-define(MEMTABLE, mem). -define(MEMTABLE, mem).
-define(MAX_TABLESIZE, 32000). -define(MAX_TABLESIZE, 32000).
-define(L0PEND_RESET, {false, [], none}). -define(PROMPT_WAIT_ONL0, 5).
-define(L0PEND_RESET, {false, null, null}).
-record(state, {manifest = [] :: list(), -record(state, {manifest = [] :: list(),
ongoing_work = [] :: list(), ongoing_work = [] :: list(),
@ -188,7 +197,7 @@
root_path = "../test" :: string(), root_path = "../test" :: string(),
table_size = 0 :: integer(), table_size = 0 :: integer(),
clerk :: pid(), clerk :: pid(),
levelzero_pending = {false, [], none} :: tuple(), levelzero_pending = ?L0PEND_RESET :: tuple(),
memtable, memtable,
backlog = false :: boolean()}). backlog = false :: boolean()}).
@ -230,13 +239,18 @@ pcl_close(Pid) ->
%%%============================================================================ %%%============================================================================
init([RootPath]) -> init([RootPath]) ->
TID = ets:new(?MEMTABLE, [ordered_set, private]), TID = ets:new(?MEMTABLE, [ordered_set]),
{ok, Clerk} = leveled_clerk:clerk_new(self()), {ok, Clerk} = leveled_clerk:clerk_new(self()),
InitState = #state{memtable=TID, clerk=Clerk, root_path=RootPath}, InitState = #state{memtable=TID, clerk=Clerk, root_path=RootPath},
%% Open manifest %% Open manifest
ManifestPath = InitState#state.root_path ++ "/" ++ ?MANIFEST_FP ++ "/", ManifestPath = InitState#state.root_path ++ "/" ++ ?MANIFEST_FP ++ "/",
{ok, Filenames} = file:list_dir(ManifestPath), {ok, Filenames} = case filelib:is_dir(ManifestPath) of
true ->
file:list_dir(ManifestPath);
false ->
{ok, []}
end,
CurrRegex = "nonzero_(?<MSN>[0-9]+)\\." ++ ?CURRENT_FILEX, CurrRegex = "nonzero_(?<MSN>[0-9]+)\\." ++ ?CURRENT_FILEX,
ValidManSQNs = lists:foldl(fun(FN, Acc) -> ValidManSQNs = lists:foldl(fun(FN, Acc) ->
case re:run(FN, case re:run(FN,
@ -270,8 +284,7 @@ init([RootPath]) ->
++ "found in nonzero levels~n", ++ "found in nonzero levels~n",
[MaxSQN]), [MaxSQN]),
%% TODO %% Find any L0 files
%% Find any L0 File left outstanding
L0FN = filepath(RootPath, L0FN = filepath(RootPath,
TopManSQN + 1, TopManSQN + 1,
new_merge_files) ++ "_0_0.sft", new_merge_files) ++ "_0_0.sft",
@ -311,6 +324,8 @@ handle_call({push_mem, DumpList}, _From, State) ->
Response = case assess_sqn(DumpList) of Response = case assess_sqn(DumpList) of
{MinSQN, MaxSQN} when MaxSQN > MinSQN, {MinSQN, MaxSQN} when MaxSQN > MinSQN,
MinSQN >= State#state.ledger_sqn -> MinSQN >= State#state.ledger_sqn ->
io:format("SQN check completed in ~w microseconds~n",
[timer:now_diff(os:timestamp(),StartWatch)]),
case push_to_memory(DumpList, State) of case push_to_memory(DumpList, State) of
{ok, UpdState} -> {ok, UpdState} ->
{reply, ok, UpdState}; {reply, ok, UpdState};
@ -346,6 +361,22 @@ handle_call({confirm_delete, FileName}, _From, State) ->
{reply, Reply, State} {reply, Reply, State}
end; end;
handle_call(prompt_compaction, _From, State) -> handle_call(prompt_compaction, _From, State) ->
%% If there is a prompt immediately after a L0 async write event then
%% there exists the potential for the prompt to stall the database.
%% Should only accept prompts if there has been a safe wait from the
%% last L0 write event.
Proceed = case State#state.levelzero_pending of
{true, _Pid, TS} ->
TD = timer:now_diff(os:timestamp(),TS),
if
TD < ?PROMPT_WAIT_ONL0 * 1000000 -> false;
true -> true
end;
?L0PEND_RESET ->
true
end,
if
Proceed ->
case push_to_memory([], State) of case push_to_memory([], State) of
{ok, UpdState} -> {ok, UpdState} ->
{reply, ok, UpdState#state{backlog=false}}; {reply, ok, UpdState#state{backlog=false}};
@ -353,6 +384,9 @@ handle_call(prompt_compaction, _From, State) ->
io:format("Excess work due to - " ++ Reason, Details), io:format("Excess work due to - " ++ Reason, Details),
{reply, pause, UpdState#state{backlog=true}} {reply, pause, UpdState#state{backlog=true}}
end; end;
true ->
{reply, ok, State#state{backlog=false}}
end;
handle_call({manifest_change, WI}, _From, State) -> handle_call({manifest_change, WI}, _From, State) ->
{ok, UpdState} = commit_manifest_change(WI, State), {ok, UpdState} = commit_manifest_change(WI, State),
{reply, ok, UpdState}; {reply, ok, UpdState};
@ -388,22 +422,21 @@ terminate(_Reason, State) ->
Dump = ets:tab2list(State#state.memtable), Dump = ets:tab2list(State#state.memtable),
case {State#state.levelzero_pending, case {State#state.levelzero_pending,
get_item(0, State#state.manifest, []), length(Dump)} of get_item(0, State#state.manifest, []), length(Dump)} of
{{false, _, _}, [], L} when L > 0 -> {?L0PEND_RESET, [], L} when L > 0 ->
MSN = State#state.manifest_sqn + 1, MSN = State#state.manifest_sqn + 1,
FileName = State#state.root_path FileName = State#state.root_path
++ "/" ++ ?FILES_FP ++ "/" ++ "/" ++ ?FILES_FP ++ "/"
++ integer_to_list(MSN) ++ "_0_0", ++ integer_to_list(MSN) ++ "_0_0",
{ok, {ok,
L0Pid, L0Pid,
{{KR1, _}, _SK, _HK}} = leveled_sft:sft_new(FileName ++ ".pnd", {{[], []}, _SK, _HK}} = leveled_sft:sft_new(FileName ++ ".pnd",
Dump, Dump,
[], [],
0), 0),
io:format("Dump of memory on close to filename ~s with" io:format("Dump of memory on close to filename ~s~n", [FileName]),
++ " remainder ~w~n", [FileName, length(KR1)]),
leveled_sft:sft_close(L0Pid), leveled_sft:sft_close(L0Pid),
file:rename(FileName ++ ".pnd", FileName ++ ".sft"); file:rename(FileName ++ ".pnd", FileName ++ ".sft");
{{false, _, _}, [], L} when L == 0 -> {?L0PEND_RESET, [], L} when L == 0 ->
io:format("No keys to dump from memory when closing~n"); io:format("No keys to dump from memory when closing~n");
_ -> _ ->
io:format("No opportunity to persist memory before closing " io:format("No opportunity to persist memory before closing "
@ -424,31 +457,36 @@ code_change(_OldVsn, State, _Extra) ->
push_to_memory(DumpList, State) -> push_to_memory(DumpList, State) ->
{TableSize, UpdState} = case State#state.levelzero_pending of {TableSize, UpdState} = case State#state.levelzero_pending of
{true, Remainder, {StartKey, EndKey, Pid}} -> {true, Pid, _TS} ->
%% Need to handle error scenarios? %% Need to handle error scenarios?
%% N.B. Sync call - so will be ready %% N.B. Sync call - so will be ready
{ok, SrcFN} = leveled_sft:sft_checkready(Pid), {ok, SrcFN, StartKey, EndKey} = leveled_sft:sft_checkready(Pid),
%% Reset ETS, but re-insert any remainder
true = ets:delete_all_objects(State#state.memtable), true = ets:delete_all_objects(State#state.memtable),
true = ets:insert(State#state.memtable, Remainder),
ManifestEntry = #manifest_entry{start_key=StartKey, ManifestEntry = #manifest_entry{start_key=StartKey,
end_key=EndKey, end_key=EndKey,
owner=Pid, owner=Pid,
filename=SrcFN}, filename=SrcFN},
{length(Remainder), {0,
State#state{manifest=lists:keystore(0, State#state{manifest=lists:keystore(0,
1, 1,
State#state.manifest, State#state.manifest,
{0, [ManifestEntry]}), {0, [ManifestEntry]}),
levelzero_pending=?L0PEND_RESET}}; levelzero_pending=?L0PEND_RESET}};
{false, _, _} -> ?L0PEND_RESET ->
{State#state.table_size, State} {State#state.table_size, State}
end, end,
%% Prompt clerk to ask about work - do this for every push_mem %% Prompt clerk to ask about work - do this for every push_mem
ok = leveled_clerk:clerk_prompt(UpdState#state.clerk, penciller), ok = leveled_clerk:clerk_prompt(UpdState#state.clerk, penciller),
case do_push_to_mem(DumpList, TableSize, UpdState#state.memtable) of SW2 = os:timestamp(),
MemoryInsertion = do_push_to_mem(DumpList,
TableSize,
UpdState#state.memtable),
io:format("Push into memory timed at ~w microseconds~n",
[timer:now_diff(os:timestamp(),SW2)]),
case MemoryInsertion of
{twist, ApproxTableSize} -> {twist, ApproxTableSize} ->
{ok, UpdState#state{table_size=ApproxTableSize}}; {ok, UpdState#state{table_size=ApproxTableSize}};
{roll, ApproxTableSize} -> {roll, ApproxTableSize} ->
@ -459,30 +497,16 @@ push_to_memory(DumpList, State) ->
FileName = UpdState#state.root_path FileName = UpdState#state.root_path
++ "/" ++ ?FILES_FP ++ "/" ++ "/" ++ ?FILES_FP ++ "/"
++ integer_to_list(MSN) ++ "_0_0", ++ integer_to_list(MSN) ++ "_0_0",
Dump = ets:tab2list(UpdState#state.memtable), Opts = #sft_options{wait=false},
L0_SFT = leveled_sft:sft_new(FileName, {ok, L0Pid} = leveled_sft:sft_new(FileName,
Dump, UpdState#state.memtable,
[], [],
0, 0,
#sft_options{wait=false}), Opts),
{ok, L0Pid, Reply} = L0_SFT, {ok,
{{KL1Rem, []}, L0StartKey, L0EndKey} = Reply,
Backlog = length(KL1Rem),
Rsp =
if
Backlog > ?MAX_TABLESIZE ->
{pause,
"Backlog of ~w in memory table~n",
[Backlog]};
true ->
ok
end,
{Rsp,
UpdState#state{levelzero_pending={true, UpdState#state{levelzero_pending={true,
KL1Rem, L0Pid,
{L0StartKey, os:timestamp()},
L0EndKey,
L0Pid}},
table_size=ApproxTableSize, table_size=ApproxTableSize,
manifest_sqn=MSN}}; manifest_sqn=MSN}};
{[], true} -> {[], true} ->
@ -558,20 +582,20 @@ manifest_locked(State) ->
true; true;
true -> true ->
case State#state.levelzero_pending of case State#state.levelzero_pending of
{true, _, _} -> {true, _Pid, _TS} ->
true; true;
_ -> _ ->
false false
end end
end. end.
%% Work out what the current work queue should be %% Work out what the current work queue should be
%% %%
%% The work queue should have a lower level work at the front, and no work %% The work queue should have a lower level work at the front, and no work
%% should be added to the queue if a compaction worker has already been asked %% should be added to the queue if a compaction worker has already been asked
%% to look at work at that level %% to look at work at that level
%%
%% The full queue is calculated for logging purposes only
return_work(State, From) -> return_work(State, From) ->
WorkQueue = assess_workqueue([], WorkQueue = assess_workqueue([],

126
src/leveled_sft.erl
View file

@ -184,8 +184,8 @@
-define(HEADER_LEN, 56). -define(HEADER_LEN, 56).
-define(ITERATOR_SCANWIDTH, 1). -define(ITERATOR_SCANWIDTH, 1).
-define(MERGE_SCANWIDTH, 8). -define(MERGE_SCANWIDTH, 8).
-define(MAX_KEYS, ?SLOT_COUNT * ?BLOCK_COUNT * ?BLOCK_SIZE).
-define(DELETE_TIMEOUT, 60000). -define(DELETE_TIMEOUT, 60000).
-define(MAX_KEYS, ?SLOT_COUNT * ?BLOCK_COUNT * ?BLOCK_SIZE).
-record(state, {version = ?CURRENT_VERSION :: tuple(), -record(state, {version = ?CURRENT_VERSION :: tuple(),
@ -204,6 +204,7 @@
handle :: file:fd(), handle :: file:fd(),
background_complete = false :: boolean(), background_complete = false :: boolean(),
background_failure = "Unknown" :: string(), background_failure = "Unknown" :: string(),
oversized_file = false :: boolean(),
ready_for_delete = false ::boolean(), ready_for_delete = false ::boolean(),
penciller :: pid()}). penciller :: pid()}).
@ -217,17 +218,17 @@ sft_new(Filename, KL1, KL2, Level) ->
sft_new(Filename, KL1, KL2, Level, Options) -> sft_new(Filename, KL1, KL2, Level, Options) ->
{ok, Pid} = gen_server:start(?MODULE, [], []), {ok, Pid} = gen_server:start(?MODULE, [], []),
Reply = case Options#sft_options.wait of case Options#sft_options.wait of
true -> true ->
gen_server:call(Pid, Reply = gen_server:call(Pid,
{sft_new, Filename, KL1, KL2, Level}, {sft_new, Filename, KL1, KL2, Level},
infinity); infinity),
{ok, Pid, Reply};
false -> false ->
gen_server:call(Pid, gen_server:cast(Pid,
{sft_new, Filename, KL1, KL2, Level, background}, {sft_new, Filename, KL1, KL2, Level}),
infinity) {ok, Pid}
end, end.
{ok, Pid, Reply}.
sft_open(Filename) -> sft_open(Filename) ->
{ok, Pid} = gen_server:start(?MODULE, [], []), {ok, Pid} = gen_server:start(?MODULE, [], []),
@ -278,47 +279,13 @@ sft_getmaxsequencenumber(Pid) ->
init([]) -> init([]) ->
{ok, #state{}}. {ok, #state{}}.
handle_call({sft_new, Filename, KL1, [], Level, background}, From, State) -> handle_call({sft_new, Filename, KL1, [], 0}, _From, _State) ->
{ListForFile, KL1Rem} = case length(KL1) of {ok, State} = create_levelzero(KL1, Filename),
L when L >= ?MAX_KEYS -> {reply,
lists:split(?MAX_KEYS, KL1); {{[], []},
_ -> State#state.smallest_key,
{KL1, []} State#state.highest_key},
end, State};
StartKey = strip_to_keyonly(lists:nth(1, ListForFile)),
EndKey = strip_to_keyonly(lists:last(ListForFile)),
Ext = filename:extension(Filename),
Components = filename:split(Filename),
{TmpFilename, PrmFilename} = case Ext of
[] ->
{filename:join(Components) ++ ".pnd", filename:join(Components) ++ ".sft"};
Ext ->
%% This seems unnecessarily hard
DN = filename:dirname(Filename),
FP = lists:last(Components),
FP_NOEXT = lists:sublist(FP, 1, 1 + length(FP) - length(Ext)),
{DN ++ "/" ++ FP_NOEXT ++ ".pnd", DN ++ "/" ++ FP_NOEXT ++ ".sft"}
end,
gen_server:reply(From, {{KL1Rem, []}, StartKey, EndKey}),
case create_file(TmpFilename) of
{error, Reason} ->
{noreply, State#state{background_complete=false,
background_failure=Reason}};
{Handle, FileMD} ->
io:format("Creating file in background with input of size ~w~n",
[length(ListForFile)]),
% Key remainders must match to empty
Rename = {true, TmpFilename, PrmFilename},
{ReadHandle, UpdFileMD, {[], []}} = complete_file(Handle,
FileMD,
ListForFile,
[],
Level,
Rename),
{noreply, UpdFileMD#state{handle=ReadHandle,
filename=PrmFilename,
background_complete=true}}
end;
handle_call({sft_new, Filename, KL1, KL2, Level}, _From, State) -> handle_call({sft_new, Filename, KL1, KL2, Level}, _From, State) ->
case create_file(Filename) of case create_file(Filename) of
{error, Reason} -> {error, Reason} ->
@ -365,7 +332,12 @@ handle_call(clear, _From, State) ->
handle_call(background_complete, _From, State) -> handle_call(background_complete, _From, State) ->
case State#state.background_complete of case State#state.background_complete of
true -> true ->
{reply, {ok, State#state.filename}, State}; {reply,
{ok,
State#state.filename,
State#state.smallest_key,
State#state.highest_key},
State};
false -> false ->
{reply, {error, State#state.background_failure}, State} {reply, {error, State#state.background_failure}, State}
end; end;
@ -380,6 +352,9 @@ handle_call({set_for_delete, Penciller}, _From, State) ->
handle_call(get_maxsqn, _From, State) -> handle_call(get_maxsqn, _From, State) ->
{reply, State#state.highest_sqn, State}. {reply, State#state.highest_sqn, State}.
handle_cast({sft_new, Filename, Inp1, [], 0}, _State) ->
{ok, State} = create_levelzero(Inp1, Filename),
{noreply, State};
handle_cast(_Msg, State) -> handle_cast(_Msg, State) ->
{noreply, State}. {noreply, State}.
@ -423,6 +398,47 @@ code_change(_OldVsn, State, _Extra) ->
%%%============================================================================ %%%============================================================================
create_levelzero(Inp1, Filename) ->
ListForFile = case is_list(Inp1) of
true ->
Inp1;
false ->
ets:tab2list(Inp1)
end,
Ext = filename:extension(Filename),
Components = filename:split(Filename),
{TmpFilename, PrmFilename} = case Ext of
[] ->
{filename:join(Components) ++ ".pnd",
filename:join(Components) ++ ".sft"};
Ext ->
%% This seems unnecessarily hard
DN = filename:dirname(Filename),
FP = lists:last(Components),
FP_NOEXT = lists:sublist(FP, 1, 1 + length(FP) - length(Ext)),
{DN ++ "/" ++ FP_NOEXT ++ ".pnd", DN ++ "/" ++ FP_NOEXT ++ ".sft"}
end,
case create_file(TmpFilename) of
{error, Reason} ->
{error,
#state{background_complete=false, background_failure=Reason}};
{Handle, FileMD} ->
InputSize = length(ListForFile),
io:format("Creating file with input of size ~w~n", [InputSize]),
Rename = {true, TmpFilename, PrmFilename},
{ReadHandle, UpdFileMD, {[], []}} = complete_file(Handle,
FileMD,
ListForFile,
[],
0,
Rename),
{ok,
UpdFileMD#state{handle=ReadHandle,
filename=PrmFilename,
background_complete=true,
oversized_file=InputSize>?MAX_KEYS}}
end.
%% Start a bare file with an initial header and no further details %% Start a bare file with an initial header and no further details
%% Return the {Handle, metadata record} %% Return the {Handle, metadata record}
create_file(FileName) when is_list(FileName) -> create_file(FileName) when is_list(FileName) ->
@ -446,7 +462,9 @@ create_file(FileName) when is_list(FileName) ->
create_header(initial) -> create_header(initial) ->
{Major, Minor} = ?CURRENT_VERSION, {Major, Minor} = ?CURRENT_VERSION,
Version = <<Major:5, Minor:3>>, Version = <<Major:5, Minor:3>>,
Options = <<0:8>>, % Not thought of any options %% Not thought of any options - options are ignored
Options = <<0:8>>,
%% Settings are currently ignored
{BlSize, BlCount, SlCount} = {?BLOCK_COUNT, ?BLOCK_SIZE, ?SLOT_COUNT}, {BlSize, BlCount, SlCount} = {?BLOCK_COUNT, ?BLOCK_SIZE, ?SLOT_COUNT},
Settings = <<BlSize:8, BlCount:8, SlCount:16>>, Settings = <<BlSize:8, BlCount:8, SlCount:16>>,
{SpareO, SpareL} = {<<0:48>>, <<0:192>>}, {SpareO, SpareL} = {<<0:48>>, <<0:192>>},
@ -871,6 +889,10 @@ sftwrite_function(finalise,
SNExtremes, SNExtremes,
KeyExtremes}). KeyExtremes}).
%% Level 0 files are of variable (infinite) size to avoid issues with having
%% any remainders when flushing from memory
maxslots_bylevel(_SlotTotal, 0) ->
continue;
maxslots_bylevel(SlotTotal, _Level) -> maxslots_bylevel(SlotTotal, _Level) ->
case SlotTotal of case SlotTotal of
?SLOT_COUNT -> ?SLOT_COUNT ->