umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Use the same file write/sync/rename path where needed

Browse source 
When we want to be sure a file has been written before proceeding - we need a safer (that `file:write_file/2`) mechanism to be sure that it is written before proceeding.

This will:
open, write, sync, rename and then optionally read-back.

Changed so that manifest writing uses the safest form (including read back), and that sst writing uses a slightly looser form (with no read back to avoid performance issues).
This commit is contained in:
Martin Sumner 2019-11-19 15:50:59 +00:00
parent e16b650823
commit 693defb6d3
4 changed files with 58 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

15
src/leveled_pmanifest.erl
View file

@ -53,6 +53,7 @@
-include_lib("eunit/include/eunit.hrl").
-define(MANIFEST_FILEX, "man").
-define(PENDING_FILEX, "pnd").
-define(MANIFEST_FP, "ledger_manifest").
-define(MAX_LEVELS, 8).
-define(TREE_TYPE, idxt).
@ -201,14 +202,15 @@ close_manifest(Manifest, CloseEntryFun) ->
%% @doc
%% Save the manifest to file (with a checksum)
save_manifest(Manifest, RootPath) ->
FP = filepath(RootPath, Manifest#manifest.manifest_sqn, current_manifest),
TFP = filepath(RootPath, Manifest#manifest.manifest_sqn, pending_manifest),
AFP = filepath(RootPath, Manifest#manifest.manifest_sqn, current_manifest),
ManBin = term_to_binary(Manifest#manifest{snapshots = [],
pending_deletes = dict:new(),
min_snapshot_sqn = 0,
blooms = dict:new()}),
CRC = erlang:crc32(ManBin),
ok = file:write_file(FP, <<CRC:32/integer, ManBin/binary>>),
{ok, <<CRC:32/integer, ManBin/binary>>} = file:read_file(FP),
ToPersist = <<CRC:32/integer, ManBin/binary>>,
ok = leveled_util:safe_rename(TFP, AFP, ToPersist, true),
GC_SQN = Manifest#manifest.manifest_sqn - ?MANIFESTS_TO_RETAIN,
% If a manifest is corrupted the previous one will be tried, so don't
% delete the previous one straight away. Retain until enough have been
@ -906,7 +908,12 @@ filepath(RootPath, manifest) ->
filepath(RootPath, NewMSN, current_manifest) ->
filepath(RootPath, manifest) ++ "nonzero_"
++ integer_to_list(NewMSN) ++ "." ++ ?MANIFEST_FILEX.
++ integer_to_list(NewMSN) ++ "." ++ ?MANIFEST_FILEX;
filepath(RootPath, NewMSN, pending_manifest) ->
filepath(RootPath, manifest) ++ "nonzero_"
++ integer_to_list(NewMSN) ++ "." ++ ?PENDING_FILEX.
open_manifestfile(_RootPath, L) when L == [] orelse L == [0] ->