From ed27a534529f1ebd3d72a1e07a84586b23c0a1b6 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 9 Jan 2017 14:52:26 +0000 Subject: [PATCH 01/37] New manifest code The manifest had previously been a list for eveyr leevl of the manifest, and keys were found by folding over the list. By Level 4 the list will be 4096 items long, and so the fold would be expensive, and would be required many times. To make this less expensive an ETS table is to use. However, the ETS table needs to be shared between snapshots and so in order to use the ETS the entries to the table need to support multi-versioning - whereby each clone can see a version of the table at the Manifest SQN the clone is supporting. --- src/leveled_log.erl | 7 +- src/leveled_manifest.erl | 482 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 488 insertions(+), 1 deletion(-) create mode 100644 src/leveled_manifest.erl diff --git a/src/leveled_log.erl b/src/leveled_log.erl index a8c94d9..898c910 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -125,7 +125,12 @@ {info, "Completion of update to levelzero"}}, {"P0032", {info, "Head timing for result ~w is sample ~w total ~w and max ~w"}}, - + {"P0033", + {error, "Corrupted manifest file at path ~s to be ignored " + ++ "due to error ~w"}}, + {"P0034", + {warn, "Snapshot with pid ~w timed out and so deletion will " + ++ "continue regardless"}}, {"PC001", {info, "Penciller's clerk ~w started with owner ~w"}}, {"PC002", diff --git a/src/leveled_manifest.erl b/src/leveled_manifest.erl new file mode 100644 index 0000000..5eccbc0 --- /dev/null +++ b/src/leveled_manifest.erl @@ -0,0 +1,482 @@ +%% -------- PENCILLER MANIFEST --------- +%% +%% The manifest is an ordered set of files for each level to be used to find +%% which file is relevant for a given key or range lookup at a given level. +%% +%% It is implemented as an ETS table, primarily to optimise lookup speed, but +%% also for the convenience of the tab2file format to support the persisting +%% of the manifest. However, it needs to be a multi-version table, as the +%% table may be accessed by both the real actor but also cloned actors too, +%% and they need to see state at different points in time. +%% +%% The ets table is an ordered set. The Key is a tuple of: +%% +%% {Level, LastKey, Filename} +%% +%% for the file. The manifest entry will have a value of: +%% +%% {FirstKey, {active, aSQN}, {tomb, tSQN}} +%% +%% When an item is added to the manifest it is added with aSQN set to the +%% manifets SQN which is intended to make this change current, and a tSQN +%% of infinity. When an item is removed the element is first altered so +%% that the tSQN is set to the next ManifestSQN. When the active +%% (non-cloned) actor reads items in the manifest it should also reap any +%% tombstone elements that have passed the lowest manifest SQN of any of +%% the registered clones. + +-module(leveled_manifest). + +-include("include/leveled.hrl"). + +-export([ + new_manifest/0, + open_manifest/1, + save_manifest/3, + key_lookup/4, + key_lookup/5, + range_lookup/5, + insert_manifest_entry/4, + remove_manifest_entry/4, + add_snapshot/4, + release_snapshot/2, + ready_to_delete/2 + ]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(MANIFEST_FILEX, "man"). +-define(MANIFEST_FP, "ledger_manifest"). + + +%%%============================================================================ +%%% API +%%%============================================================================ + +new_manifest() -> + ets:new(manifest, [ordered_set]). + +open_manifest(RootPath) -> + % Open the manifest in the file path which has the highest SQN, and will + % open without error + ManifestPath = filepath(RootPath, manifest), + {ok, Filenames} = file:list_dir(ManifestPath), + CurrRegex = "nonzero_(?[0-9]+)\\." ++ ?MANIFEST_FILEX, + ExtractSQNFun = + fun(FN, Acc) -> + case re:run(FN, CurrRegex, [{capture, ['MSN'], list}]) of + nomatch -> + Acc; + {match, [Int]} when is_list(Int) -> + Acc ++ [list_to_integer(Int)] + end + end, + ValidManSQNs = lists:reverse(lists:sort(lists:foldl(ExtractSQNFun, + [], + Filenames))), + open_manifestfile(RootPath, ValidManSQNs). + +save_manifest(Manifest, RootPath, ManSQN) -> + FP = filepath(RootPath, ManSQN, current_manifest), + ets:tab2file(Manifest, + FP, + [{extended_info, [md5sum]}, {sync, true}]). + +insert_manifest_entry(Manifest, ManSQN, Level, Entry) -> + Key = {Level, Entry#manifest_entry.end_key, Entry#manifest_entry.filename}, + Value = {Entry#manifest_entry.start_key, + {active, ManSQN}, + {tomb, infinity}}, + true = ets:insert_new(Manifest, {Key, Value}). + +remove_manifest_entry(Manifest, ManSQN, Level, Entry) -> + Key = {Level, Entry#manifest_entry.end_key, Entry#manifest_entry.filename}, + [{Key, Value0}] = ets:lookup(Manifest, Key), + {StartKey, {active, ActiveSQN}, {tomb, infinity}} = Value0, + Value1 = {StartKey, {active, ActiveSQN}, {tomb, ManSQN}}, + true = ets:insert(Manifest, {Key, Value1}). + +key_lookup(Manifest, Level, Key, ManSQN) -> + key_lookup(Manifest, Level, Key, ManSQN, false). + +key_lookup(Manifest, Level, Key, ManSQN, GC) -> + key_lookup(Manifest, Level, {Key, 0}, Key, ManSQN, GC). + +range_lookup(Manifest, Level, StartKey, EndKey, ManSQN) -> + range_lookup(Manifest, Level, {StartKey, 0}, StartKey, EndKey, [], ManSQN). + +add_snapshot(SnapList0, Pid, ManifestSQN, Timeout) -> + [{Pid, ManifestSQN, Timeout}|SnapList0]. + +release_snapshot(SnapList0, Pid) -> + FilterFun = + fun({P, SQN, TS}, Acc) -> + case P of + Pid -> + Acc; + _ -> + [{P, SQN, TS}|Acc] + end + end, + lists:foldl(FilterFun, [], SnapList0). + +ready_to_delete(SnapList0, DeleteSQN) -> + ready_to_delete(SnapList0, DeleteSQN, os:timestamp()). + +%%%============================================================================ +%%% Internal Functions +%%%============================================================================ + +ready_to_delete(SnapList, FileDeleteSQN, Now) -> + FilterFun = + fun({P, SnapSQN, ExpiryTS}, Acc) -> + case Acc of + false -> + false; + true -> + case FileDeleteSQN < SnapSQN of + true -> + % Snapshot taken after the file deletion + true; + false -> + case Now > ExpiryTS of + true -> + leveled_log:log("P0034", [P]), + true; + false -> + false + end + end + end + end, + lists:foldl(FilterFun, true, SnapList). + +filepath(RootPath, manifest) -> + RootPath ++ "/" ++ ?MANIFEST_FP ++ "/". + +filepath(RootPath, NewMSN, current_manifest) -> + filepath(RootPath, manifest) ++ "nonzero_" + ++ integer_to_list(NewMSN) ++ "." ++ ?MANIFEST_FILEX. + + +open_manifestfile(_RootPath, []) -> + leveled_log:log("P0013", []), + new_manifest(); +open_manifestfile(_RootPath, [0]) -> + leveled_log:log("P0013", []), + new_manifest(); +open_manifestfile(RootPath, [TopManSQN|Rest]) -> + CurrManFile = filepath(RootPath, TopManSQN, current_manifest), + case ets:file2tab(CurrManFile, [{verify,true}]) of + {error, Reason} -> + leveled_log:log("P0033", [CurrManFile, Reason]), + open_manifestfile(RootPath, Rest); + {ok, Table} -> + leveled_log:log("P0012", [TopManSQN]), + Table + end. + +key_lookup(Manifest, Level, {LastKey, LastFN}, KeyToFind, ManSQN, GC) -> + case ets:next(Manifest, {Level, LastKey, LastFN}) of + '$end_of_table' -> + false; + {Level, NextKey, NextFN} -> + [{K, V}] = ets:lookup(Manifest, {Level, NextKey, NextFN}), + {FirstKey, {active, ActiveSQN}, {tomb, TombSQN}} = V, + Active = (ManSQN >= ActiveSQN) and (ManSQN < TombSQN), + case Active of + true -> + InRange = KeyToFind >= FirstKey, + case InRange of + true -> + NextFN; + false -> + false + end; + false -> + case GC of + false -> + ok; + {true, GC_SQN} -> + case TombSQN < GC_SQN of + true -> + ets:delete(Manifest, K); + false -> + ok + end + end, + key_lookup(Manifest, + Level, + {NextKey, NextFN}, + KeyToFind, + ManSQN, + GC) + end; + {OtherLevel, _, _} when OtherLevel > Level -> + false + end. + +range_lookup(Manifest, Level, {LastKey, LastFN}, SK, EK, Acc, ManSQN) -> + case ets:next(Manifest, {Level, LastKey, LastFN}) of + '$end_of_table' -> + Acc; + {Level, NextKey, NextFN} -> + [{_K, V}] = ets:lookup(Manifest, {Level, NextKey, NextFN}), + {FirstKey, {active, ActiveSQN}, {tomb, TombSQN}} = V, + Active = (ManSQN >= ActiveSQN) and (ManSQN < TombSQN), + case Active of + true -> + PostEnd = leveled_codec:endkey_passed(EK, FirstKey), + case PostEnd of + true -> + Acc; + false -> + range_lookup(Manifest, + Level, + {NextKey, NextFN}, + SK, + EK, + Acc ++ [NextFN], + ManSQN) + end; + false -> + range_lookup(Manifest, + Level, + {NextKey, NextFN}, + SK, + EK, + Acc, + ManSQN) + end; + {OtherLevel, _, _} when OtherLevel > Level -> + Acc + end. + +%%%============================================================================ +%%% Test +%%%============================================================================ + +-ifdef(TEST). + +rangequery_manifest_test() -> + E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, + end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, + filename="Z1"}, + E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"}, + end_key={o, "Bucket1", "K71", null}, + filename="Z2"}, + E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null}, + end_key={o, "Bucket1", "K993", null}, + filename="Z3"}, + E4 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, + end_key={i, "Bucket1", {"Idx1", "Fld7"}, "K93"}, + filename="Z4"}, + E5 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld7"}, "K97"}, + end_key={o, "Bucket1", "K78", null}, + filename="Z5"}, + E6 = #manifest_entry{start_key={o, "Bucket1", "K81", null}, + end_key={o, "Bucket1", "K996", null}, + filename="Z6"}, + + Manifest = open_manifestfile(dummy, []), + insert_manifest_entry(Manifest, 1, 1, E1), + insert_manifest_entry(Manifest, 1, 1, E2), + insert_manifest_entry(Manifest, 1, 1, E3), + insert_manifest_entry(Manifest, 1, 2, E4), + insert_manifest_entry(Manifest, 1, 2, E5), + insert_manifest_entry(Manifest, 1, 2, E6), + + SK1 = {o, "Bucket1", "K711", null}, + EK1 = {o, "Bucket1", "K999", null}, + RL1_1 = range_lookup(Manifest, 1, SK1, EK1, 1), + ?assertMatch(["Z3"], RL1_1), + RL1_2 = range_lookup(Manifest, 2, SK1, EK1, 1), + ?assertMatch(["Z5", "Z6"], RL1_2), + SK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, + EK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, + RL2_1 = range_lookup(Manifest, 1, SK2, EK2, 1), + ?assertMatch(["Z1"], RL2_1), + RL2_2 = range_lookup(Manifest, 2, SK2, EK2, 1), + ?assertMatch(["Z5"], RL2_2), + + SK3 = {o, "Bucket1", "K994", null}, + EK3 = {o, "Bucket1", "K995", null}, + RL3_1 = range_lookup(Manifest, 1, SK3, EK3, 1), + ?assertMatch([], RL3_1), + RL3_2 = range_lookup(Manifest, 2, SK3, EK3, 1), + ?assertMatch(["Z6"], RL3_2), + + E1_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld4"}, "K8"}, + end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K62"}, + filename="Y1"}, + E2_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K67"}, + end_key={o, "Bucket1", "K45", null}, + filename="Y2"}, + E3_2 = #manifest_entry{start_key={o, "Bucket1", "K47", null}, + end_key={o, "Bucket1", "K812", null}, + filename="Y3"}, + E4_2 = #manifest_entry{start_key={o, "Bucket1", "K815", null}, + end_key={o, "Bucket1", "K998", null}, + filename="Y4"}, + + insert_manifest_entry(Manifest, 2, 1, E1_2), + insert_manifest_entry(Manifest, 2, 1, E2_2), + insert_manifest_entry(Manifest, 2, 1, E3_2), + insert_manifest_entry(Manifest, 2, 1, E4_2), + remove_manifest_entry(Manifest, 2, 1, E1), + remove_manifest_entry(Manifest, 2, 1, E2), + remove_manifest_entry(Manifest, 2, 1, E3), + + RL1_1A = range_lookup(Manifest, 1, SK1, EK1, 1), + ?assertMatch(["Z3"], RL1_1A), + RL2_1A = range_lookup(Manifest, 1, SK2, EK2, 1), + ?assertMatch(["Z1"], RL2_1A), + RL3_1A = range_lookup(Manifest, 1, SK3, EK3, 1), + ?assertMatch([], RL3_1A), + + RL1_1B = range_lookup(Manifest, 1, SK1, EK1, 2), + ?assertMatch(["Y3", "Y4"], RL1_1B), + RL2_1B = range_lookup(Manifest, 1, SK2, EK2, 2), + ?assertMatch(["Y1"], RL2_1B), + RL3_1B = range_lookup(Manifest, 1, SK3, EK3, 2), + ?assertMatch(["Y4"], RL3_1B). + + + +keyquery_manifest_test() -> + E1 = #manifest_entry{start_key={o, "Bucket1", "K0001", null}, + end_key={o, "Bucket1", "K0990", null}, + filename="Z1"}, + E2 = #manifest_entry{start_key={o, "Bucket1", "K1003", null}, + end_key={o, "Bucket1", "K3692", null}, + filename="Z2"}, + E3 = #manifest_entry{start_key={o, "Bucket1", "K3750", null}, + end_key={o, "Bucket1", "K9930", null}, + filename="Z3"}, + + + Manifest0 = open_manifestfile(dummy, []), + insert_manifest_entry(Manifest0, 1, 1, E1), + insert_manifest_entry(Manifest0, 1, 1, E2), + insert_manifest_entry(Manifest0, 1, 1, E3), + + RootPath = "../test", + ok = filelib:ensure_dir(filepath(RootPath, manifest)), + ok = save_manifest(Manifest0, RootPath, 1), + true = ets:delete(Manifest0), + ?assertMatch(true, filelib:is_file(filepath(RootPath, + 1, + current_manifest))), + + BadFP = filepath(RootPath, 2, current_manifest), + ok = file:write_file(BadFP, list_to_binary("nonsense")), + ?assertMatch(true, filelib:is_file(BadFP)), + + Manifest = open_manifest(RootPath), + + K1 = {o, "Bucket1", "K0000", null}, + K2 = {o, "Bucket1", "K0001", null}, + K3 = {o, "Bucket1", "K0002", null}, + K4 = {o, "Bucket1", "K0990", null}, + K5 = {o, "Bucket1", "K0991", null}, + K6 = {o, "Bucket1", "K1003", null}, + K7 = {o, "Bucket1", "K1004", null}, + K8 = {o, "Bucket1", "K3692", null}, + K9 = {o, "Bucket1", "K3693", null}, + K10 = {o, "Bucket1", "K3750", null}, + K11 = {o, "Bucket1", "K3751", null}, + K12 = {o, "Bucket1", "K9930", null}, + K13 = {o, "Bucket1", "K9931", null}, + + ?assertMatch(false, key_lookup(Manifest, 1, K1, 1)), + ?assertMatch("Z1", key_lookup(Manifest, 1, K2, 1)), + ?assertMatch("Z1", key_lookup(Manifest, 1, K3, 1)), + ?assertMatch("Z1", key_lookup(Manifest, 1, K4, 1)), + ?assertMatch(false, key_lookup(Manifest, 1, K5, 1)), + ?assertMatch("Z2", key_lookup(Manifest, 1, K6, 1)), + ?assertMatch("Z2", key_lookup(Manifest, 1, K7, 1)), + ?assertMatch("Z2", key_lookup(Manifest, 1, K8, 1)), + ?assertMatch(false, key_lookup(Manifest, 1, K9, 1)), + ?assertMatch("Z3", key_lookup(Manifest, 1, K10, 1)), + ?assertMatch("Z3", key_lookup(Manifest, 1, K11, 1)), + ?assertMatch("Z3", key_lookup(Manifest, 1, K12, 1)), + ?assertMatch(false, key_lookup(Manifest, 1, K13, 1)), + + E1_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld4"}, "K8"}, + end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K62"}, + filename="Y1"}, + E2_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K67"}, + end_key={o, "Bucket1", "K45", null}, + filename="Y2"}, + E3_2 = #manifest_entry{start_key={o, "Bucket1", "K47", null}, + end_key={o, "Bucket1", "K812", null}, + filename="Y3"}, + E4_2 = #manifest_entry{start_key={o, "Bucket1", "K815", null}, + end_key={o, "Bucket1", "K998", null}, + filename="Y4"}, + + insert_manifest_entry(Manifest, 2, 1, E1_2), + insert_manifest_entry(Manifest, 2, 1, E2_2), + insert_manifest_entry(Manifest, 2, 1, E3_2), + insert_manifest_entry(Manifest, 2, 1, E4_2), + + S1 = ets:info(Manifest, size), + + remove_manifest_entry(Manifest, 2, 1, E1), + remove_manifest_entry(Manifest, 2, 1, E2), + remove_manifest_entry(Manifest, 2, 1, E3), + + S2 = ets:info(Manifest, size), + ?assertMatch(true, S2 == S1), + + ?assertMatch("Y2", key_lookup(Manifest, 1, K1, 2)), + ?assertMatch("Y2", key_lookup(Manifest, 1, K10, 2)), + ?assertMatch("Y4", key_lookup(Manifest, 1, K12, 2)), + + S3 = ets:info(Manifest, size), + ?assertMatch(true, S3 == S1), + + ?assertMatch("Y2", key_lookup(Manifest, 1, K1, 2, {true, 2})), + ?assertMatch("Y2", key_lookup(Manifest, 1, K10, 2, {true, 2})), + ?assertMatch("Y4", key_lookup(Manifest, 1, K12, 2, {true, 2})), + + S4 = ets:info(Manifest, size), + ?assertMatch(true, S4 == S1), + + ?assertMatch("Y2", key_lookup(Manifest, 1, K1, 3, {true, 3})), + ?assertMatch("Y2", key_lookup(Manifest, 1, K10, 3, {true, 3})), + ?assertMatch("Y4", key_lookup(Manifest, 1, K12, 3, {true, 3})), + + S5 = ets:info(Manifest, size), + ?assertMatch(true, S5 < S1). + +snapshot_test() -> + Snap0 = [], + + ?assertMatch(true, ready_to_delete(Snap0, 1)), + + {MegaS0, S0, MicroS0} = os:timestamp(), + + Snap1 = add_snapshot(Snap0, pid_1, 3, {MegaS0, S0 + 100, MicroS0}), + Snap2 = add_snapshot(Snap1, pid_2, 4, {MegaS0, S0 + 200, MicroS0}), + Snap3 = add_snapshot(Snap2, pid_3, 4, {MegaS0, S0 + 150, MicroS0}), + Snap4 = add_snapshot(Snap3, pid_4, 5, {MegaS0, S0 + 300, MicroS0}), + + ?assertMatch(true, + ready_to_delete(Snap4, 2, {MegaS0, S0, MicroS0})), + ?assertMatch(false, + ready_to_delete(Snap4, 3, {MegaS0, S0, MicroS0})), + ?assertMatch(true, + ready_to_delete(Snap4, 3, {MegaS0, S0 + 150, MicroS0})), + ?assertMatch(false, + ready_to_delete(Snap4, 4, {MegaS0, S0 + 150, MicroS0})), + ?assertMatch(true, + ready_to_delete(Snap4, 4, {MegaS0, S0 + 250, MicroS0})), + + Snap5 = release_snapshot(Snap4, pid_1), + ?assertMatch(true, + ready_to_delete(Snap5, 3, {MegaS0, S0, MicroS0})). + + + +-endif. \ No newline at end of file From 439ddfa4eb29bcbe26f04aebf2ab36bbbc165cc3 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 9 Jan 2017 15:19:11 +0000 Subject: [PATCH 02/37] Add support for initiate The penciller requires a starting manifest SQN as well as a list of filenames to open on startup. It is assumed that the penciller will keep the mappings between fileames and PIDs outside of the manifest. --- src/leveled_manifest.erl | 88 +++++++++++++++++++++++++--------------- 1 file changed, 56 insertions(+), 32 deletions(-) diff --git a/src/leveled_manifest.erl b/src/leveled_manifest.erl index 5eccbc0..f7e4f68 100644 --- a/src/leveled_manifest.erl +++ b/src/leveled_manifest.erl @@ -33,6 +33,7 @@ new_manifest/0, open_manifest/1, save_manifest/3, + initiate_from_manifest/1, key_lookup/4, key_lookup/5, range_lookup/5, @@ -82,6 +83,21 @@ save_manifest(Manifest, RootPath, ManSQN) -> FP, [{extended_info, [md5sum]}, {sync, true}]). + +initiate_from_manifest(Manifest) -> + FlatManifest = ets:tab2list(Manifest), + InitiateFun = + fun({{_L, _EK, FN}, {_SK, ActSt, DelSt}}, {FNList, MaxSQN}) -> + case {ActSt, DelSt} of + {{active, ActSQN}, {tomb, infinity}} -> + {[FN|FNList], max(ActSQN, MaxSQN)}; + {_, {tomb, TombSQN}} -> + {FNList, max(TombSQN, MaxSQN)} + end + end, + lists:foldl(InitiateFun, {[], 0}, FlatManifest). + + insert_manifest_entry(Manifest, ManSQN, Level, Entry) -> Key = {Level, Entry#manifest_entry.end_key, Entry#manifest_entry.filename}, Value = {Entry#manifest_entry.start_key, @@ -353,26 +369,34 @@ keyquery_manifest_test() -> E3 = #manifest_entry{start_key={o, "Bucket1", "K3750", null}, end_key={o, "Bucket1", "K9930", null}, filename="Z3"}, + EToRemove = #manifest_entry{start_key={o, "Bucket99", "K3750", null}, + end_key={o, "Bucket99", "K9930", null}, + filename="ZR"}, Manifest0 = open_manifestfile(dummy, []), insert_manifest_entry(Manifest0, 1, 1, E1), insert_manifest_entry(Manifest0, 1, 1, E2), insert_manifest_entry(Manifest0, 1, 1, E3), + insert_manifest_entry(Manifest0, 1, 1, EToRemove), + remove_manifest_entry(Manifest0, 2, 1, EToRemove), RootPath = "../test", ok = filelib:ensure_dir(filepath(RootPath, manifest)), - ok = save_manifest(Manifest0, RootPath, 1), + ok = save_manifest(Manifest0, RootPath, 2), true = ets:delete(Manifest0), ?assertMatch(true, filelib:is_file(filepath(RootPath, - 1, + 2, current_manifest))), - BadFP = filepath(RootPath, 2, current_manifest), + BadFP = filepath(RootPath, 3, current_manifest), ok = file:write_file(BadFP, list_to_binary("nonsense")), ?assertMatch(true, filelib:is_file(BadFP)), Manifest = open_manifest(RootPath), + {FNList, ManSQN} = initiate_from_manifest(Manifest), + ?assertMatch(["Z1", "Z2", "Z3"], lists:sort(FNList)), + ?assertMatch(2, ManSQN), K1 = {o, "Bucket1", "K0000", null}, K2 = {o, "Bucket1", "K0001", null}, @@ -388,19 +412,19 @@ keyquery_manifest_test() -> K12 = {o, "Bucket1", "K9930", null}, K13 = {o, "Bucket1", "K9931", null}, - ?assertMatch(false, key_lookup(Manifest, 1, K1, 1)), - ?assertMatch("Z1", key_lookup(Manifest, 1, K2, 1)), - ?assertMatch("Z1", key_lookup(Manifest, 1, K3, 1)), - ?assertMatch("Z1", key_lookup(Manifest, 1, K4, 1)), - ?assertMatch(false, key_lookup(Manifest, 1, K5, 1)), - ?assertMatch("Z2", key_lookup(Manifest, 1, K6, 1)), - ?assertMatch("Z2", key_lookup(Manifest, 1, K7, 1)), - ?assertMatch("Z2", key_lookup(Manifest, 1, K8, 1)), - ?assertMatch(false, key_lookup(Manifest, 1, K9, 1)), - ?assertMatch("Z3", key_lookup(Manifest, 1, K10, 1)), - ?assertMatch("Z3", key_lookup(Manifest, 1, K11, 1)), - ?assertMatch("Z3", key_lookup(Manifest, 1, K12, 1)), - ?assertMatch(false, key_lookup(Manifest, 1, K13, 1)), + ?assertMatch(false, key_lookup(Manifest, 1, K1, 2)), + ?assertMatch("Z1", key_lookup(Manifest, 1, K2, 2)), + ?assertMatch("Z1", key_lookup(Manifest, 1, K3, 2)), + ?assertMatch("Z1", key_lookup(Manifest, 1, K4, 2)), + ?assertMatch(false, key_lookup(Manifest, 1, K5, 2)), + ?assertMatch("Z2", key_lookup(Manifest, 1, K6, 2)), + ?assertMatch("Z2", key_lookup(Manifest, 1, K7, 2)), + ?assertMatch("Z2", key_lookup(Manifest, 1, K8, 2)), + ?assertMatch(false, key_lookup(Manifest, 1, K9, 2)), + ?assertMatch("Z3", key_lookup(Manifest, 1, K10, 2)), + ?assertMatch("Z3", key_lookup(Manifest, 1, K11, 2)), + ?assertMatch("Z3", key_lookup(Manifest, 1, K12, 2)), + ?assertMatch(false, key_lookup(Manifest, 1, K13, 2)), E1_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld4"}, "K8"}, end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K62"}, @@ -415,37 +439,37 @@ keyquery_manifest_test() -> end_key={o, "Bucket1", "K998", null}, filename="Y4"}, - insert_manifest_entry(Manifest, 2, 1, E1_2), - insert_manifest_entry(Manifest, 2, 1, E2_2), - insert_manifest_entry(Manifest, 2, 1, E3_2), - insert_manifest_entry(Manifest, 2, 1, E4_2), + insert_manifest_entry(Manifest, 3, 1, E1_2), + insert_manifest_entry(Manifest, 3, 1, E2_2), + insert_manifest_entry(Manifest, 3, 1, E3_2), + insert_manifest_entry(Manifest, 3, 1, E4_2), S1 = ets:info(Manifest, size), - remove_manifest_entry(Manifest, 2, 1, E1), - remove_manifest_entry(Manifest, 2, 1, E2), - remove_manifest_entry(Manifest, 2, 1, E3), + remove_manifest_entry(Manifest, 3, 1, E1), + remove_manifest_entry(Manifest, 3, 1, E2), + remove_manifest_entry(Manifest, 3, 1, E3), S2 = ets:info(Manifest, size), ?assertMatch(true, S2 == S1), - ?assertMatch("Y2", key_lookup(Manifest, 1, K1, 2)), - ?assertMatch("Y2", key_lookup(Manifest, 1, K10, 2)), - ?assertMatch("Y4", key_lookup(Manifest, 1, K12, 2)), + ?assertMatch("Y2", key_lookup(Manifest, 1, K1, 3)), + ?assertMatch("Y2", key_lookup(Manifest, 1, K10, 3)), + ?assertMatch("Y4", key_lookup(Manifest, 1, K12, 3)), S3 = ets:info(Manifest, size), ?assertMatch(true, S3 == S1), - ?assertMatch("Y2", key_lookup(Manifest, 1, K1, 2, {true, 2})), - ?assertMatch("Y2", key_lookup(Manifest, 1, K10, 2, {true, 2})), - ?assertMatch("Y4", key_lookup(Manifest, 1, K12, 2, {true, 2})), + ?assertMatch("Y2", key_lookup(Manifest, 1, K1, 3, {true, 3})), + ?assertMatch("Y2", key_lookup(Manifest, 1, K10, 3, {true, 3})), + ?assertMatch("Y4", key_lookup(Manifest, 1, K12, 3, {true, 3})), S4 = ets:info(Manifest, size), ?assertMatch(true, S4 == S1), - ?assertMatch("Y2", key_lookup(Manifest, 1, K1, 3, {true, 3})), - ?assertMatch("Y2", key_lookup(Manifest, 1, K10, 3, {true, 3})), - ?assertMatch("Y4", key_lookup(Manifest, 1, K12, 3, {true, 3})), + ?assertMatch("Y2", key_lookup(Manifest, 1, K1, 4, {true, 4})), + ?assertMatch("Y2", key_lookup(Manifest, 1, K10, 4, {true, 4})), + ?assertMatch("Y4", key_lookup(Manifest, 1, K12, 4, {true, 4})), S5 = ets:info(Manifest, size), ?assertMatch(true, S5 < S1). From 08641e05cf569701cae8440004ad38adb9f7a812 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 12 Jan 2017 13:48:43 +0000 Subject: [PATCH 03/37] Manifest changes - BROKEN Going to abandond this branch for now. The change is beoming excessively time consuming, and it is not clear that a smaller change might not achieve more of the objectives. All this is broken - but perhaps could get picke dup another day. --- include/leveled.hrl | 5 +- src/leveled_log.erl | 8 +- src/leveled_manifest.erl | 34 ++- src/leveled_pclerk.erl | 137 +++------ src/leveled_penciller.erl | 590 ++++++++++---------------------------- 5 files changed, 216 insertions(+), 558 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index 25216f6..7843739 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -24,12 +24,11 @@ {next_sqn :: integer(), clerk :: pid(), src_level :: integer(), - manifest :: list(), start_time :: tuple(), ledger_filepath :: string(), - manifest_file :: string(), - new_manifest :: list(), unreferenced_files :: list(), + new_files :: list(), + level_counts :: dict(), target_is_basement = false ::boolean()}). -record(level, diff --git a/src/leveled_log.erl b/src/leveled_log.erl index 898c910..f4b59a2 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -74,8 +74,6 @@ ++ "reason ~w"}}, {"P0008", {info, "Penciller closing for reason ~w"}}, - {"P0009", - {info, "Level 0 cache empty at close of Penciller"}}, {"P0010", {info, "No level zero action on close of Penciller ~w"}}, {"P0011", @@ -97,9 +95,6 @@ ++ "L0 pending ~w and merge backlog ~w"}}, {"P0019", {info, "Rolling level zero to filename ~s at ledger sqn ~w"}}, - {"P0020", - {info, "Work at Level ~w to be scheduled for ~w with ~w " - ++ "queue items outstanding at all levels"}}, {"P0021", {info, "Allocation of work blocked as L0 pending"}}, {"P0022", @@ -108,7 +103,8 @@ {info, "Manifest entry of startkey ~s ~s ~s endkey ~s ~s ~s " ++ "filename=~s~n"}}, {"P0024", - {info, "Outstanding compaction work items of ~w at level ~w"}}, + {info, "Outstanding compaction work items of ~w with backlog status " + ++ "of ~w"}}, {"P0025", {info, "Merge to sqn ~w from Level ~w completed"}}, {"P0026", diff --git a/src/leveled_manifest.erl b/src/leveled_manifest.erl index f7e4f68..82c323a 100644 --- a/src/leveled_manifest.erl +++ b/src/leveled_manifest.erl @@ -87,15 +87,17 @@ save_manifest(Manifest, RootPath, ManSQN) -> initiate_from_manifest(Manifest) -> FlatManifest = ets:tab2list(Manifest), InitiateFun = - fun({{_L, _EK, FN}, {_SK, ActSt, DelSt}}, {FNList, MaxSQN}) -> + fun({{L, _EK, FN}, {_SK, ActSt, DelSt}}, {FNList, MaxSQN, LCount}) -> case {ActSt, DelSt} of {{active, ActSQN}, {tomb, infinity}} -> - {[FN|FNList], max(ActSQN, MaxSQN)}; + {[FN|FNList], + max(ActSQN, MaxSQN), + dict:update_counter(L, 1, LCount)}; {_, {tomb, TombSQN}} -> - {FNList, max(TombSQN, MaxSQN)} + {FNList, max(TombSQN, MaxSQN), LCount} end end, - lists:foldl(InitiateFun, {[], 0}, FlatManifest). + lists:foldl(InitiateFun, {[], 0, dict:new()}, FlatManifest). insert_manifest_entry(Manifest, ManSQN, Level, Entry) -> @@ -274,6 +276,8 @@ range_lookup(Manifest, Level, {LastKey, LastFN}, SK, EK, Acc, ManSQN) -> -ifdef(TEST). + + rangequery_manifest_test() -> E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, @@ -358,8 +362,7 @@ rangequery_manifest_test() -> ?assertMatch(["Y4"], RL3_1B). - -keyquery_manifest_test() -> +startup_manifest() E1 = #manifest_entry{start_key={o, "Bucket1", "K0001", null}, end_key={o, "Bucket1", "K0990", null}, filename="Z1"}, @@ -369,15 +372,19 @@ keyquery_manifest_test() -> E3 = #manifest_entry{start_key={o, "Bucket1", "K3750", null}, end_key={o, "Bucket1", "K9930", null}, filename="Z3"}, - EToRemove = #manifest_entry{start_key={o, "Bucket99", "K3750", null}, - end_key={o, "Bucket99", "K9930", null}, - filename="ZR"}, - Manifest0 = open_manifestfile(dummy, []), insert_manifest_entry(Manifest0, 1, 1, E1), insert_manifest_entry(Manifest0, 1, 1, E2), insert_manifest_entry(Manifest0, 1, 1, E3), + Manifest0 + +keyquery_manifest_test() -> + Manifest0 = startup_manifest(), + + EToRemove = #manifest_entry{start_key={o, "Bucket99", "K3750", null}, + end_key={o, "Bucket99", "K9930", null}, + filename="ZR"}, insert_manifest_entry(Manifest0, 1, 1, EToRemove), remove_manifest_entry(Manifest0, 2, 1, EToRemove), @@ -394,9 +401,10 @@ keyquery_manifest_test() -> ?assertMatch(true, filelib:is_file(BadFP)), Manifest = open_manifest(RootPath), - {FNList, ManSQN} = initiate_from_manifest(Manifest), + {FNList, ManSQN, LCount} = initiate_from_manifest(Manifest), ?assertMatch(["Z1", "Z2", "Z3"], lists:sort(FNList)), ?assertMatch(2, ManSQN), + ?assertMatch(3, dict:fetch(1, LCount)), K1 = {o, "Bucket1", "K0000", null}, K2 = {o, "Bucket1", "K0001", null}, @@ -502,5 +510,9 @@ snapshot_test() -> ready_to_delete(Snap5, 3, {MegaS0, S0, MicroS0})). +allatlevel_test() -> + Manifest0 = startup_manifest(), + AllAtL1 = range_lookup(Manifest, 1, all, {null, null, null, null}, 1), + ?assertMatch(["Z1", "Z2", "Z3"], AllAtL1). -endif. \ No newline at end of file diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 9ccc791..58ed347 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -2,10 +2,9 @@ %% %% The Penciller's clerk is responsible for compaction work within the Ledger. %% -%% The Clerk will periodically poll the Penciller to see if there is work for -%% it to complete, except if the Clerk has informed the Penciller that it has -%% readied a manifest change to be committed - in which case it will wait to -%% be called by the Penciller. +%% The Clerk will periodically poll the Penciller to check there is no work +%% at level zero pending completion, and if not the Clerk will examine the +%% manifest to see if work is necessary. %% %% -------- COMMITTING MANIFEST CHANGES --------- %% @@ -18,35 +17,7 @@ %% certain that the manifest change has been committed. Some uncollected %% garbage is considered acceptable. %% -%% The process of committing a manifest change is as follows: -%% -%% A - The Clerk completes a merge, and casts a prompt to the Penciller with -%% a work item describing the change -%% -%% B - The Penciller commits the change to disk, and then calls the Clerk to -%% confirm the manifest change -%% -%% C - The Clerk replies immediately to acknowledge this call, then marks the -%% removed files for deletion -%% -%% Shutdown < A/B - If the Penciller starts the shutdown process before the -%% merge is complete, in the shutdown the Penciller will call a request for the -%% manifest change which will pick up the pending change. It will then confirm -%% the change, and now the Clerk will mark the files for delete before it -%% replies to the Penciller so it can complete the shutdown process (which will -%% prompt erasing of the removed files). -%% -%% The clerk will not request work on timeout if the committing of a manifest -%% change is pending confirmation. -%% -%% -------- TIMEOUTS --------- -%% -%% The Penciller may prompt the Clerk to callback soon (i.e. reduce the -%% Timeout) if it has urgent work ready (i.e. it has written a L0 file). -%% -%% There will also be a natural quick timeout once the committing of a manifest -%% change has occurred. -%% + -module(leveled_pclerk). @@ -68,8 +39,10 @@ -define(MAX_TIMEOUT, 2000). -define(MIN_TIMEOUT, 50). +-define(END_KEY, {null, null, null, null}). -record(state, {owner :: pid(), + manifest, % ets table reference change_pending=false :: boolean(), work_item :: #penciller_work{}|null}). @@ -77,19 +50,17 @@ %%% API %%%============================================================================ -clerk_new(Owner) -> +clerk_new(Owner, Manifest) -> {ok, Pid} = gen_server:start(?MODULE, [], []), - ok = gen_server:call(Pid, {register, Owner}, infinity), + ok = gen_server:call(Pid, {load, Owner, Manifest}, infinity), leveled_log:log("PC001", [Pid, Owner]), {ok, Pid}. -clerk_manifestchange(Pid, Action, Closing) -> - gen_server:call(Pid, {manifest_change, Action, Closing}, infinity). - clerk_prompt(Pid) -> gen_server:cast(Pid, prompt). - +clerk_close(Pid) -> + gen_server:cast(Pid, close). %%%============================================================================ %%% gen_server callbacks @@ -98,41 +69,16 @@ clerk_prompt(Pid) -> init([]) -> {ok, #state{}}. -handle_call({register, Owner}, _From, State) -> +handle_call({load, Owner, Manifest}, _From, State) -> {reply, ok, - State#state{owner=Owner}, - ?MIN_TIMEOUT}; -handle_call({manifest_change, return, true}, _From, State) -> - leveled_log:log("PC002", []), - case State#state.change_pending of - true -> - WI = State#state.work_item, - {reply, {ok, WI}, State}; - false -> - {stop, normal, no_change, State} - end; -handle_call({manifest_change, confirm, Closing}, From, State) -> - case Closing of - true -> - leveled_log:log("PC003", []), - WI = State#state.work_item, - ok = mark_for_delete(WI#penciller_work.unreferenced_files, - State#state.owner), - {stop, normal, ok, State}; - false -> - leveled_log:log("PC004", []), - gen_server:reply(From, ok), - WI = State#state.work_item, - ok = mark_for_delete(WI#penciller_work.unreferenced_files, - State#state.owner), - {noreply, - State#state{work_item=null, change_pending=false}, - ?MIN_TIMEOUT} - end. + State#state{owner=Owner, manifest=Manifest}, + ?MIN_TIMEOUT}. handle_cast(prompt, State) -> - {noreply, State, ?MIN_TIMEOUT}. + {noreply, State, ?MIN_TIMEOUT}; +handle_cast(close, State) -> + (stop, normal, State). handle_info(timeout, State=#state{change_pending=Pnd}) when Pnd == false -> case requestandhandle_work(State) of @@ -159,26 +105,28 @@ code_change(_OldVsn, State, _Extra) -> requestandhandle_work(State) -> case leveled_penciller:pcl_workforclerk(State#state.owner) of - none -> + false -> leveled_log:log("PC006", []), - {false, ?MAX_TIMEOUT}; - WI -> - {NewManifest, FilesToDelete} = merge(WI), - UpdWI = WI#penciller_work{new_manifest=NewManifest, - unreferenced_files=FilesToDelete}, + false; + {SrcLevel, ManifestSQN} -> + {Additions, Removals} = merge(Level, + State#state.manifest, + ManifestSQN), leveled_log:log("PC007", []), - ok = leveled_penciller:pcl_promptmanifestchange(State#state.owner, - UpdWI), - {true, UpdWI} + ok = leveled_penciller:pcl_commitmanifestchange(State#state.owner, + SrcLevel, + Additions, + Removals, + ManifestSQN), + true end. -merge(WI) -> - SrcLevel = WI#penciller_work.src_level, - {SrcF, UpdMFest1} = select_filetomerge(SrcLevel, - WI#penciller_work.manifest), - SinkFiles = get_item(SrcLevel + 1, UpdMFest1, []), - {Candidates, Others} = check_for_merge_candidates(SrcF, SinkFiles), +merge(SrcLevel, Manifest, ManifestSQN) -> + SrcF = select_filetomerge(SrcLevel, Manifest), + + + Candidates = check_for_merge_candidates(SrcF, SinkFiles), %% TODO: %% Need to work out if this is the top level %% And then tell merge process to create files at the top level @@ -254,17 +202,14 @@ check_for_merge_candidates(SrcF, SinkFiles) -> %% %% Hence, the initial implementation is to select files to merge at random -select_filetomerge(SrcLevel, Manifest) -> - {SrcLevel, LevelManifest} = lists:keyfind(SrcLevel, 1, Manifest), - Selected = lists:nth(random:uniform(length(LevelManifest)), - LevelManifest), - UpdManifest = lists:keyreplace(SrcLevel, - 1, - Manifest, - {SrcLevel, - lists:delete(Selected, - LevelManifest)}), - {Selected, UpdManifest}. +select_filetomerge(SrcLevel, Manifest, ManifestSQN) -> + Level = leveled_manifest:range_lookup(Manifest, + 1, + all, + ?END_KEY, + ManifestSQN), + + FN = lists:nth(random:uniform(length(Level)), Level). diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index a07065b..0f4fa2d 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -176,7 +176,7 @@ pcl_fetchnextkey/5, pcl_checksequencenumber/3, pcl_workforclerk/1, - pcl_promptmanifestchange/2, + pcl_confirmmanifestchange/2, pcl_confirml0complete/4, pcl_confirmdelete/2, pcl_close/1, @@ -206,13 +206,17 @@ -define(COIN_SIDECOUNT, 5). -define(SLOW_FETCH, 20000). -define(ITERATOR_SCANWIDTH, 4). +-define(SNAPSHOT_TIMEOUT, 3600). --record(state, {manifest = [] :: list(), +-record(state, {manifest, % an ETS table reference manifest_sqn = 0 :: integer(), - ledger_sqn = 0 :: integer(), % The highest SQN added to L0 persisted_sqn = 0 :: integer(), % The highest SQN persisted registered_snapshots = [] :: list(), - unreferenced_files = [] :: list(), + pidmap = dict:new() :: dict(), + level_counts :: dict(), + deletions_pending = dict:new() ::dict(), + + ledger_sqn = 0 :: integer(), % The highest SQN added to L0 root_path = "../test" :: string(), clerk :: pid(), @@ -286,7 +290,7 @@ pcl_checksequencenumber(Pid, Key, SQN) -> pcl_workforclerk(Pid) -> gen_server:call(Pid, work_for_clerk, infinity). -pcl_promptmanifestchange(Pid, WI) -> +pcl_confirmmanifestchange(Pid, WI) -> gen_server:cast(Pid, {manifest_change, WI}). pcl_confirml0complete(Pid, FN, StartKey, EndKey) -> @@ -371,18 +375,24 @@ handle_call({push_mem, {PushedTree, PushedIdx, MinSQN, MaxSQN}}, State)} end; handle_call({fetch, Key, Hash}, _From, State) -> + Structure = {State#state.manifest, + State#state.pid_map, + State#state.manifest_sqn}, {R, HeadTimer} = timed_fetch_mem(Key, Hash, - State#state.manifest, + Structure, State#state.levelzero_cache, State#state.levelzero_index, State#state.head_timing), {reply, R, State#state{head_timing=HeadTimer}}; handle_call({check_sqn, Key, Hash, SQN}, _From, State) -> + Structure = {State#state.manifest, + State#state.pid_map, + State#state.manifest_sqn}, {reply, compare_to_sqn(plain_fetch_mem(Key, Hash, - State#state.manifest, + Structure, State#state.levelzero_cache, State#state.levelzero_index), SQN), @@ -401,23 +411,53 @@ handle_call({fetch_keys, StartKey, EndKey, AccFun, InitAcc, MaxKeys}, List -> List end, - SSTiter = initiate_rangequery_frommanifest(StartKey, + + ConvertToPointerFun = + fun(FN) -> {next, dict:fetch(FN, State#state.pid_map), StartKey} end, + SetupFoldFun = + fun(Level, Acc) -> + FNs = leveled_manifest:range_lookup(State#state.manifest, + Level, + StartKey, EndKey, - State#state.manifest), + State#state.manifest_sqn), + Pointers = lists:map(ConvertToPointerFun, FNs), + case Pointers of + [] -> Acc; + PL -> Acc ++ [{L, PL}] + end + end, + SSTiter = lists:foldl(SetupFoldFun, [], lists:seq(0, ?MAX_LEVELS - 1)), + Acc = keyfolder({L0AsList, SSTiter}, {StartKey, EndKey}, {AccFun, InitAcc}, MaxKeys), + {reply, Acc, State#state{levelzero_astree = L0AsList}}; handle_call(work_for_clerk, From, State) -> - {UpdState, Work} = return_work(State, From), - {reply, Work, UpdState}; + DelayForPendingL0 = State#state.levelzero_pending, + {WL, WC} = check_for_work(State#state.level_counts), + case WC of + 0 -> + {reply, none, State#state{work_backlog=false}}; + N when N > ?WORKQUEUE_BACKLOG_TOLERANCE -> + leveled_log:log("P0024", [N, true]), + [TL|_Tail] = WL, + {reply, TL, State#state{work_backlog=true}}; + N -> + leveled_log:log("P0024", [N, false]), + [TL|_Tail] = WL, + {reply, TL, State#state{work_backlog=false}} + end; handle_call(get_startup_sqn, _From, State) -> {reply, State#state.persisted_sqn, State}; handle_call({register_snapshot, Snapshot}, _From, State) -> - Rs = [{Snapshot, - State#state.manifest_sqn}|State#state.registered_snapshots], - {reply, {ok, State}, State#state{registered_snapshots = Rs}}; + RegisteredSnaps = add_snapshot(State#state.registered_snapshots, + Snapshot, + State#state.manifest_sqn, + ?SNAPSHOT_TIMEOUT), + {reply, {ok, State}, State#state{registered_snapshots = RegisteredSnaps}}; handle_call({load_snapshot, {BookieIncrTree, BookieIdx, MinSQN, MaxSQN}}, _From, State) -> L0D = leveled_pmem:add_to_cache(State#state.levelzero_size, @@ -444,28 +484,36 @@ handle_call(doom, _From, State) -> {stop, normal, {ok, [ManifestFP, FilesFP]}, State}. handle_cast({manifest_change, WI}, State) -> - {ok, UpdState} = commit_manifest_change(WI, State), - ok = leveled_pclerk:clerk_manifestchange(State#state.clerk, - confirm, - false), - {noreply, UpdState}; + NewManifestSQN = WI#next_sqn, + UnreferenceFun = + fun(FN, Acc) -> + dict:store(FN, NewManifestSQN, Acc) + end, + DelPending = lists:foldl(UnreferenceFun, + State#state.deletions_pending, + WI#unreferenced_files), + {noreply, State{deletions_pending = DelPending, + manifest_sqn = NewManifestSQN}}; handle_cast({release_snapshot, Snapshot}, State) -> - Rs = lists:keydelete(Snapshot, 1, State#state.registered_snapshots), + Rs = leveled_manifest:release_snapshot(State#state.registered_snapshots, + Snapshot), leveled_log:log("P0003", [Snapshot]), leveled_log:log("P0004", [Rs]), {noreply, State#state{registered_snapshots=Rs}}; -handle_cast({confirm_delete, FileName}, State=#state{is_snapshot=Snap}) +handle_cast({confirm_delete, Filename}, State=#state{is_snapshot=Snap}) when Snap == false -> - Reply = confirm_delete(FileName, - State#state.unreferenced_files, - State#state.registered_snapshots), - case Reply of - {true, Pid} -> - UF1 = lists:keydelete(FileName, 1, State#state.unreferenced_files), + DeleteSQN = dict:fetch(Filename, State#state.deletions_pending), + R2D = leveled_manifest:ready_to_delete(State#state.registered_snapshots, + DeleteSQN), + case R2D of + true -> + PidToDelete = dict:fetch(Filename, State#state.pidmap), leveled_log:log("P0005", [FileName]), + DP0 = dict:erase(Filename, State#state.deletions_pending), + PM0 = dict:erase(Filename, State#state.pidmap), ok = leveled_sst:sst_deleteconfirmed(Pid), - {noreply, State#state{unreferenced_files=UF1}}; - _ -> + {noreply, State#state{deletions_pending = DP0, pidmap = PM0}}; + false -> {noreply, State} end; handle_cast({levelzero_complete, FN, StartKey, EndKey}, State) -> @@ -478,11 +526,13 @@ handle_cast({levelzero_complete, FN, StartKey, EndKey}, State) -> % Prompt clerk to ask about work - do this for every L0 roll UpdIndex = leveled_pmem:clear_index(State#state.levelzero_index), ok = leveled_pclerk:clerk_prompt(State#state.clerk), + UpdLevelCounts = dict:store(0, 1, State#state.level_counts), {noreply, State#state{levelzero_cache=[], levelzero_index=UpdIndex, levelzero_pending=false, levelzero_constructor=undefined, levelzero_size=0, + level_counts=UpdLevelCounts, manifest=UpdMan, persisted_sqn=State#state.ledger_sqn}}. @@ -495,10 +545,6 @@ terminate(Reason, State=#state{is_snapshot=Snap}) when Snap == true -> leveled_log:log("P0007", [Reason]), ok; terminate(Reason, State) -> - %% When a Penciller shuts down it isn't safe to try an manage the safe - %% finishing of any outstanding work. The last commmitted manifest will - %% be used. - %% %% Level 0 files lie outside of the manifest, and so if there is no L0 %% file present it is safe to write the current contents of memory. If %% there is a L0 file present - then the memory can be dropped (it is @@ -506,44 +552,27 @@ terminate(Reason, State) -> %% as presumably the ETS file has been recently flushed, hence the presence %% of a L0 file). %% - %% The penciller should close each file in the unreferenced files, and - %% then each file in the manifest, and cast a close on the clerk. - %% The cast may not succeed as the clerk could be synchronously calling - %% the penciller looking for a manifest commit - %% + %% The penciller should close each file in the manifest, and cast a close + %% on the clerk. + ok = leveled_pclerk:clerk_close(State#state.clerk), + leveled_log:log("P0008", [Reason]), - MC = leveled_pclerk:clerk_manifestchange(State#state.clerk, - return, - true), - UpdState = case MC of - {ok, WI} -> - {ok, NewState} = commit_manifest_change(WI, State), - Clerk = State#state.clerk, - ok = leveled_pclerk:clerk_manifestchange(Clerk, - confirm, - true), - NewState; - no_change -> - State - end, - case {UpdState#state.levelzero_pending, - get_item(0, UpdState#state.manifest, []), - UpdState#state.levelzero_size} of - {false, [], 0} -> - leveled_log:log("P0009", []); - {false, [], _N} -> - L0Pid = roll_memory(UpdState, true), + L0 = key_lookup(State#state.manifest, 0, all, State#state.manifest_sqn), + case {UpdState#state.levelzero_pending, L0} of + {false, false} -> + L0Pid = roll_memory(UpdState, true), ok = leveled_sst:sst_close(L0Pid); StatusTuple -> leveled_log:log("P0010", [StatusTuple]) end, % Tidy shutdown of individual files - ok = close_files(0, UpdState#state.manifest), - lists:foreach(fun({_FN, Pid, _SN}) -> - ok = leveled_sst:sst_close(Pid) end, - UpdState#state.unreferenced_files), + lists:foreach(fun({_FN, Pid}) -> + ok = leveled_sst:sst_close(Pid) + end, + dict:to_list(State#state.pidmap)), leveled_log:log("P0011", []), + ok. @@ -579,47 +608,21 @@ start_from_file(PCLopts) -> levelzero_index=leveled_pmem:new_index()}, %% Open manifest - ManifestPath = filepath(InitState#state.root_path, manifest) ++ "/", - SSTPath = filepath(InitState#state.root_path, files) ++ "/", - ok = filelib:ensure_dir(ManifestPath), - ok = filelib:ensure_dir(SSTPath), - - {ok, Filenames} = file:list_dir(ManifestPath), - CurrRegex = "nonzero_(?[0-9]+)\\." ++ ?CURRENT_FILEX, - ValidManSQNs = lists:foldl(fun(FN, Acc) -> - case re:run(FN, - CurrRegex, - [{capture, ['MSN'], list}]) of - nomatch -> - Acc; - {match, [Int]} when is_list(Int) -> - Acc ++ [list_to_integer(Int)] - end - end, - [], - Filenames), - TopManSQN = lists:foldl(fun(X, MaxSQN) -> max(X, MaxSQN) end, - 0, - ValidManSQNs), - leveled_log:log("P0012", [TopManSQN]), - ManUpdate = case TopManSQN of - 0 -> - leveled_log:log("P0013", []), - {[], 0}; - _ -> - CurrManFile = filepath(InitState#state.root_path, - TopManSQN, - current_manifest), - {ok, Bin} = file:read_file(CurrManFile), - Manifest = binary_to_term(Bin), - open_all_filesinmanifest(Manifest) - end, - - {UpdManifest, MaxSQN} = ManUpdate, + Manifest = leveled_manifest:open_manifest(RootPath), + {FNList, + ManSQN, + LevelCounts) = leveled_manifest:initiate_from_manifest(Manifest), + InitiateFun = + fun(FN, {AccMaxSQN, AccPidMap}) -> + {ok, P, {_FK, _LK}} = leveled_sst:sst_open(FN), + FileMaxSQN = leveled_sst:sst_getmaxsequencenumber(P), + {max(AccMaxSQN, FileMaxSQN), dict:store(FN, P, AccPidMap)} + end, + {MaxSQN, PidMap} = lists:foldl(InitiateFun, {0, dict:new()}, FNList), leveled_log:log("P0014", [MaxSQN]), %% Find any L0 files - L0FN = filepath(RootPath, TopManSQN, new_merge_files) ++ "_0_0.sst", + L0FN = filepath(RootPath, ManSQN, new_merge_files) ++ "_0_0.sst", case filelib:is_file(L0FN) of true -> leveled_log:log("P0015", [L0FN]), @@ -627,30 +630,42 @@ start_from_file(PCLopts) -> L0Pid, {L0StartKey, L0EndKey}} = leveled_sst:sst_open(L0FN), L0SQN = leveled_sst:sst_getmaxsequencenumber(L0Pid), - ManifestEntry = #manifest_entry{start_key=L0StartKey, - end_key=L0EndKey, - owner=L0Pid, - filename=L0FN}, - UpdManifest2 = lists:keystore(0, - 1, - UpdManifest, - {0, [ManifestEntry]}), + L0Entry = #manifest_entry{start_key = L0StartKey, + end_key = L0EndKey, + filename = L0FN}, + PidMap0 = dict:store(L0FN, L0Pid, PidMap), + insert_manifest_entry(Manifest, ManSQN, 0, L0Entry) leveled_log:log("P0016", [L0SQN]), LedgerSQN = max(MaxSQN, L0SQN), {ok, - InitState#state{manifest=UpdManifest2, - manifest_sqn=TopManSQN, - ledger_sqn=LedgerSQN, - persisted_sqn=LedgerSQN}}; + InitState#state{manifest = Manifest, + manifest_sqn = ManSQN, + ledger_sqn = LedgerSQN, + persisted_sqn = LedgerSQN, + level_counts = LevelCounts, + pid_map = PidMap0}}; false -> leveled_log:log("P0017", []), {ok, - InitState#state{manifest=UpdManifest, - manifest_sqn=TopManSQN, - ledger_sqn=MaxSQN, - persisted_sqn=MaxSQN}} + InitState#state{manifest = Manifest, + manifest_sqn = ManSQN, + ledger_sqn = MaxSQN, + persisted_sqn = MaxSQN, + level_counts = LevelCounts, + pid_map = PidMap}} end. +check_for_work(LevelCounts) -> + CheckLevelFun = + fun({Level, MaxCount}, {AccL, AccC}) -> + case dict:fetch(Level, LevelCounts) of + LC when LC > MaxCount -> + {[Level|AccL], AccC + LC - MaxCount}; + _ -> + {AccL, AccC} + end + end, + lists:foldl(CheckLevelFun, {[], 0}, ?LEVEL_SCALEFACTOR). update_levelzero(L0Size, {PushedTree, PushedIdx, MinSQN, MaxSQN}, @@ -738,9 +753,9 @@ levelzero_filename(State) -> ++ integer_to_list(MSN) ++ "_0_0", FileName. -timed_fetch_mem(Key, Hash, Manifest, L0Cache, L0Index, HeadTimer) -> +timed_fetch_mem(Key, Hash, Structure, L0Cache, L0Index, HeadTimer) -> SW = os:timestamp(), - {R, Level} = fetch_mem(Key, Hash, Manifest, L0Cache, L0Index), + {R, Level} = fetch_mem(Key, Hash, Structure, L0Cache, L0Index), UpdHeadTimer = case R of not_present -> @@ -750,41 +765,32 @@ timed_fetch_mem(Key, Hash, Manifest, L0Cache, L0Index, HeadTimer) -> end, {R, UpdHeadTimer}. -plain_fetch_mem(Key, Hash, Manifest, L0Cache, L0Index) -> - R = fetch_mem(Key, Hash, Manifest, L0Cache, L0Index), +plain_fetch_mem(Key, Hash, Structure, L0Cache, L0Index) -> + R = fetch_mem(Key, Hash, Structure, L0Cache, L0Index), element(1, R). -fetch_mem(Key, Hash, Manifest, L0Cache, L0Index) -> +fetch_mem(Key, Hash, Structure, L0Cache, L0Index) -> PosList = leveled_pmem:check_index(Hash, L0Index), L0Check = leveled_pmem:check_levelzero(Key, Hash, PosList, L0Cache), case L0Check of {false, not_found} -> - fetch(Key, Hash, Manifest, 0, fun timed_sst_get/3); + fetch(Key, Hash, Structure, 0, fun timed_sst_get/3); {true, KV} -> {KV, 0} end. -fetch(_Key, _Hash, _Manifest, ?MAX_LEVELS + 1, _FetchFun) -> +fetch(_Key, _Hash, _Structure, ?MAX_LEVELS + 1, _FetchFun) -> {not_present, basement}; -fetch(Key, Hash, Manifest, Level, FetchFun) -> - LevelManifest = get_item(Level, Manifest, []), - case lists:foldl(fun(File, Acc) -> - case Acc of - not_present when - Key >= File#manifest_entry.start_key, - File#manifest_entry.end_key >= Key -> - File#manifest_entry.owner; - FoundDetails -> - FoundDetails - end end, - not_present, - LevelManifest) of - not_present -> - fetch(Key, Hash, Manifest, Level + 1, FetchFun); - FileToCheck -> - case FetchFun(FileToCheck, Key, Hash) of +fetch(Key, Hash, Structure, Level, FetchFun) -> + {Manifest, PidMap, ManSQN} = Structure, + case leveled_manifest:key_lookup(Manifest, Level, Key, ManSQN) of + false -> + fetch(Key, Hash, Structure, Level + 1, FetchFun); + FN -> + FP = dict:fetch(FN, PidMap), + case FetchFun(FP, Key, Hash) of not_present -> - fetch(Key, Hash, Manifest, Level + 1, FetchFun); + fetch(Key, Hash, Structure, Level + 1, FetchFun); ObjectFound -> {ObjectFound, Level} end @@ -821,134 +827,6 @@ compare_to_sqn(Obj, SQN) -> end. -%% Work out what the current work queue should be -%% -%% The work queue should have a lower level work at the front, and no work -%% should be added to the queue if a compaction worker has already been asked -%% to look at work at that level -%% -%% The full queue is calculated for logging purposes only - -return_work(State, From) -> - {WorkQ, BasementL} = assess_workqueue([], 0, State#state.manifest, 0), - case length(WorkQ) of - L when L > 0 -> - Excess = lists:foldl(fun({_, _, OH}, Acc) -> Acc+OH end, 0, WorkQ), - [{SrcLevel, Manifest, _Overhead}|_OtherWork] = WorkQ, - leveled_log:log("P0020", [SrcLevel, From, Excess]), - IsBasement = if - SrcLevel + 1 == BasementL -> - true; - true -> - false - end, - Backlog = Excess >= ?WORKQUEUE_BACKLOG_TOLERANCE, - case State#state.levelzero_pending of - true -> - % Once the L0 file is completed there will be more work - % - so don't be busy doing other work now - leveled_log:log("P0021", []), - {State#state{work_backlog=Backlog}, none}; - false -> - %% No work currently outstanding - %% Can allocate work - NextSQN = State#state.manifest_sqn + 1, - FP = filepath(State#state.root_path, - NextSQN, - new_merge_files), - ManFile = filepath(State#state.root_path, - NextSQN, - pending_manifest), - WI = #penciller_work{next_sqn=NextSQN, - clerk=From, - src_level=SrcLevel, - manifest=Manifest, - start_time = os:timestamp(), - ledger_filepath = FP, - manifest_file = ManFile, - target_is_basement = IsBasement}, - {State#state{ongoing_work=[WI], work_backlog=Backlog}, WI} - end; - _ -> - {State#state{work_backlog=false}, none} - end. - - -close_files(?MAX_LEVELS - 1, _Manifest) -> - ok; -close_files(Level, Manifest) -> - LevelList = get_item(Level, Manifest, []), - lists:foreach(fun(F) -> - ok = leveled_sst:sst_close(F#manifest_entry.owner) end, - LevelList), - close_files(Level + 1, Manifest). - - -open_all_filesinmanifest(Manifest) -> - open_all_filesinmanifest({Manifest, 0}, 0). - -open_all_filesinmanifest(Result, ?MAX_LEVELS - 1) -> - Result; -open_all_filesinmanifest({Manifest, TopSQN}, Level) -> - LevelList = get_item(Level, Manifest, []), - %% The Pids in the saved manifest related to now closed references - %% Need to roll over the manifest at this level starting new processes to - %5 replace them - LvlR = lists:foldl(fun(F, {FL, FL_SQN}) -> - FN = F#manifest_entry.filename, - {ok, P, _Keys} = leveled_sst:sst_open(FN), - F_SQN = leveled_sst:sst_getmaxsequencenumber(P), - {lists:append(FL, - [F#manifest_entry{owner = P}]), - max(FL_SQN, F_SQN)} - end, - {[], 0}, - LevelList), - %% Result is tuple of revised file list for this level in manifest, and - %% the maximum sequence number seen at this level - {LvlFL, LvlSQN} = LvlR, - UpdManifest = lists:keystore(Level, 1, Manifest, {Level, LvlFL}), - open_all_filesinmanifest({UpdManifest, max(TopSQN, LvlSQN)}, Level + 1). - -print_manifest(Manifest) -> - lists:foreach(fun(L) -> - leveled_log:log("P0022", [L]), - Level = get_item(L, Manifest, []), - lists:foreach(fun print_manifest_entry/1, Level) - end, - lists:seq(0, ?MAX_LEVELS - 1)), - ok. - -print_manifest_entry(Entry) -> - {S1, S2, S3} = leveled_codec:print_key(Entry#manifest_entry.start_key), - {E1, E2, E3} = leveled_codec:print_key(Entry#manifest_entry.end_key), - leveled_log:log("P0023", - [S1, S2, S3, E1, E2, E3, Entry#manifest_entry.filename]). - -initiate_rangequery_frommanifest(StartKey, EndKey, Manifest) -> - CompareFun = fun(M) -> - C1 = StartKey > M#manifest_entry.end_key, - C2 = leveled_codec:endkey_passed(EndKey, - M#manifest_entry.start_key), - not (C1 or C2) end, - FoldFun = - fun(L, AccL) -> - Level = get_item(L, Manifest, []), - FL = lists:foldl(fun(M, Acc) -> - case CompareFun(M) of - true -> - Acc ++ [{next, M, StartKey}]; - false -> - Acc - end end, - [], - Level), - case FL of - [] -> AccL; - FL -> AccL ++ [{L, FL}] - end - end, - lists:foldl(FoldFun, [], lists:seq(0, ?MAX_LEVELS - 1)). %% Looks to find the best choice for the next key across the levels (other %% than in-memory table) @@ -1153,143 +1031,12 @@ keyfolder({[{IMMKey, IMMVal}|NxIMMiterator], SSTiterator}, KeyRange, end. -assess_workqueue(WorkQ, ?MAX_LEVELS - 1, _Man, BasementLevel) -> - {WorkQ, BasementLevel}; -assess_workqueue(WorkQ, LevelToAssess, Man, BasementLevel) -> - MaxFiles = get_item(LevelToAssess, ?LEVEL_SCALEFACTOR, 0), - case length(get_item(LevelToAssess, Man, [])) of - FileCount when FileCount > 0 -> - NewWQ = maybe_append_work(WorkQ, - LevelToAssess, - Man, - MaxFiles, - FileCount), - assess_workqueue(NewWQ, LevelToAssess + 1, Man, LevelToAssess); - 0 -> - assess_workqueue(WorkQ, LevelToAssess + 1, Man, BasementLevel) - end. - - -maybe_append_work(WorkQ, Level, Manifest, - MaxFiles, FileCount) - when FileCount > MaxFiles -> - Overhead = FileCount - MaxFiles, - leveled_log:log("P0024", [Overhead, Level]), - lists:append(WorkQ, [{Level, Manifest, Overhead}]); -maybe_append_work(WorkQ, _Level, _Manifest, - _MaxFiles, _FileCount) -> - WorkQ. - - -get_item(Index, List, Default) -> - case lists:keysearch(Index, 1, List) of - {value, {Index, Value}} -> - Value; - false -> - Default - end. - - -%% Request a manifest change -%% The clerk should have completed the work, and created a new manifest -%% and persisted the new view of the manifest -%% -%% To complete the change of manifest: -%% - the state of the manifest file needs to be changed from pending to current -%% - the list of unreferenced files needs to be updated on State -%% - the current manifest needs to be update don State -%% - the list of ongoing work needs to be cleared of this item - - -commit_manifest_change(ReturnedWorkItem, State) -> - NewMSN = State#state.manifest_sqn + 1, - [SentWorkItem] = State#state.ongoing_work, - RootPath = State#state.root_path, - UnreferencedFiles = State#state.unreferenced_files, - - if - NewMSN == SentWorkItem#penciller_work.next_sqn -> - WISrcLevel = SentWorkItem#penciller_work.src_level, - leveled_log:log_timer("P0025", - [SentWorkItem#penciller_work.next_sqn, - WISrcLevel], - SentWorkItem#penciller_work.start_time), - ok = rename_manifest_files(RootPath, NewMSN), - FilesToDelete = ReturnedWorkItem#penciller_work.unreferenced_files, - UnreferencedFilesUpd = update_deletions(FilesToDelete, - NewMSN, - UnreferencedFiles), - leveled_log:log("P0026", [NewMSN]), - NewManifest = ReturnedWorkItem#penciller_work.new_manifest, - - CurrL0 = get_item(0, State#state.manifest, []), - % If the work isn't L0 work, then we may have an uncommitted - % manifest change at L0 - so add this back into the Manifest loop - % state - RevisedManifest = case {WISrcLevel, CurrL0} of - {0, _} -> - NewManifest; - {_, []} -> - NewManifest; - {_, [L0ManEntry]} -> - lists:keystore(0, - 1, - NewManifest, - {0, [L0ManEntry]}) - end, - {ok, State#state{ongoing_work=[], - manifest_sqn=NewMSN, - manifest=RevisedManifest, - unreferenced_files=UnreferencedFilesUpd}} - end. - - -rename_manifest_files(RootPath, NewMSN) -> - OldFN = filepath(RootPath, NewMSN, pending_manifest), - NewFN = filepath(RootPath, NewMSN, current_manifest), - leveled_log:log("P0027", [OldFN, filelib:is_file(OldFN), - NewFN, filelib:is_file(NewFN)]), - ok = file:rename(OldFN,NewFN). - -filepath(RootPath, manifest) -> - RootPath ++ "/" ++ ?MANIFEST_FP; filepath(RootPath, files) -> RootPath ++ "/" ++ ?FILES_FP. -filepath(RootPath, NewMSN, pending_manifest) -> - filepath(RootPath, manifest) ++ "/" ++ "nonzero_" - ++ integer_to_list(NewMSN) ++ "." ++ ?PENDING_FILEX; -filepath(RootPath, NewMSN, current_manifest) -> - filepath(RootPath, manifest) ++ "/" ++ "nonzero_" - ++ integer_to_list(NewMSN) ++ "." ++ ?CURRENT_FILEX; filepath(RootPath, NewMSN, new_merge_files) -> filepath(RootPath, files) ++ "/" ++ integer_to_list(NewMSN). -update_deletions([], _NewMSN, UnreferencedFiles) -> - UnreferencedFiles; -update_deletions([ClearedFile|Tail], MSN, UnreferencedFiles) -> - leveled_log:log("P0028", [ClearedFile#manifest_entry.filename]), - update_deletions(Tail, - MSN, - lists:append(UnreferencedFiles, - [{ClearedFile#manifest_entry.filename, - ClearedFile#manifest_entry.owner, - MSN}])). - -confirm_delete(Filename, UnreferencedFiles, RegisteredSnapshots) -> - case lists:keyfind(Filename, 1, UnreferencedFiles) of - {Filename, Pid, MSN} -> - LowSQN = lists:foldl(fun({_, SQN}, MinSQN) -> min(SQN, MinSQN) end, - infinity, - RegisteredSnapshots), - if - MSN >= LowSQN -> - false; - true -> - {true, Pid} - end - end. - %%%============================================================================ @@ -1338,47 +1085,6 @@ clean_subdir(DirPath) -> end. -compaction_work_assessment_test() -> - L0 = [{{o, "B1", "K1", null}, {o, "B3", "K3", null}, dummy_pid}], - L1 = [{{o, "B1", "K1", null}, {o, "B2", "K2", null}, dummy_pid}, - {{o, "B2", "K3", null}, {o, "B4", "K4", null}, dummy_pid}], - Manifest = [{0, L0}, {1, L1}], - {WorkQ1, 1} = assess_workqueue([], 0, Manifest, 0), - ?assertMatch([{0, Manifest, 1}], WorkQ1), - L1Alt = lists:append(L1, - [{{o, "B5", "K0001", null}, {o, "B5", "K9999", null}, - dummy_pid}, - {{o, "B6", "K0001", null}, {o, "B6", "K9999", null}, - dummy_pid}, - {{o, "B7", "K0001", null}, {o, "B7", "K9999", null}, - dummy_pid}, - {{o, "B8", "K0001", null}, {o, "B8", "K9999", null}, - dummy_pid}, - {{o, "B9", "K0001", null}, {o, "B9", "K9999", null}, - dummy_pid}, - {{o, "BA", "K0001", null}, {o, "BA", "K9999", null}, - dummy_pid}, - {{o, "BB", "K0001", null}, {o, "BB", "K9999", null}, - dummy_pid}]), - Manifest3 = [{0, []}, {1, L1Alt}], - {WorkQ3, 1} = assess_workqueue([], 0, Manifest3, 0), - ?assertMatch([{1, Manifest3, 1}], WorkQ3). - -confirm_delete_test() -> - Filename = 'test.sst', - UnreferencedFiles = [{'other.sst', dummy_owner, 15}, - {Filename, dummy_owner, 10}], - RegisteredIterators1 = [{dummy_pid, 16}, {dummy_pid, 12}], - R1 = confirm_delete(Filename, UnreferencedFiles, RegisteredIterators1), - ?assertMatch(R1, {true, dummy_owner}), - RegisteredIterators2 = [{dummy_pid, 10}, {dummy_pid, 12}], - R2 = confirm_delete(Filename, UnreferencedFiles, RegisteredIterators2), - ?assertMatch(R2, false), - RegisteredIterators3 = [{dummy_pid, 9}, {dummy_pid, 12}], - R3 = confirm_delete(Filename, UnreferencedFiles, RegisteredIterators3), - ?assertMatch(R3, false). - - maybe_pause_push(PCL, KL) -> T0 = leveled_skiplist:empty(true), I0 = leveled_pmem:new_index(), From 0204a23a589afef3fe8f2c11c20bb1252609d087 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 13 Jan 2017 18:23:57 +0000 Subject: [PATCH 04/37] Refactor - STILL BROKEN Will at least compile, but in need of a massive eunit rewrite and associated debug to get back to a potentially verifiable state again --- include/leveled.hrl | 11 - src/leveled_log.erl | 2 +- src/leveled_manifest.erl | 552 +++++++++++++++++++++----------------- src/leveled_pclerk.erl | 355 ++++++++---------------- src/leveled_penciller.erl | 373 ++++++++------------------ 5 files changed, 527 insertions(+), 766 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index 7843739..13d862e 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -20,17 +20,6 @@ expire_tombstones = false :: boolean(), penciller :: pid()}). --record(penciller_work, - {next_sqn :: integer(), - clerk :: pid(), - src_level :: integer(), - start_time :: tuple(), - ledger_filepath :: string(), - unreferenced_files :: list(), - new_files :: list(), - level_counts :: dict(), - target_is_basement = false ::boolean()}). - -record(level, {level :: integer(), is_basement = false :: boolean(), diff --git a/src/leveled_log.erl b/src/leveled_log.erl index f4b59a2..f5829be 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -148,7 +148,7 @@ {"PC010", {info, "Merge to be commenced for FileToMerge=~s with MSN=~w"}}, {"PC011", - {info, "Merge completed with MSN=~w Level=~w and FileCounter=~w"}}, + {info, "Merge completed with MSN=~w to Level=~w and FileCounter=~w"}}, {"PC012", {info, "File to be created as part of MSN=~w Filename=~s"}}, {"PC013", diff --git a/src/leveled_manifest.erl b/src/leveled_manifest.erl index 82c323a..f7efed9 100644 --- a/src/leveled_manifest.erl +++ b/src/leveled_manifest.erl @@ -32,30 +32,56 @@ -export([ new_manifest/0, open_manifest/1, - save_manifest/3, - initiate_from_manifest/1, - key_lookup/4, - key_lookup/5, - range_lookup/5, + copy_manifest/1, + load_manifest/3, + save_manifest/2, + get_manifest_sqn/1, + key_lookup/3, + range_lookup/4, + merge_lookup/4, insert_manifest_entry/4, remove_manifest_entry/4, - add_snapshot/4, + mergefile_selector/2, + add_snapshot/3, release_snapshot/2, - ready_to_delete/2 + ready_to_delete/2, + check_for_work/2, + is_basement/2, + dump_pidmap/1, + levelzero_present/1, + pointer_convert/2 ]). -include_lib("eunit/include/eunit.hrl"). -define(MANIFEST_FILEX, "man"). -define(MANIFEST_FP, "ledger_manifest"). +-define(MAX_LEVELS, 8). +-define(END_KEY, {null, null, null, null}). +-record(manifest, {table, + % A Multi-Version ETS table for lookup + pidmap, + % A dictionary to map filenames to {Pid, DeleteSQN} + manifest_sqn = 0 :: integer(), + % The current manifest SQN + is_clone = false :: boolean(), + % Is this manifest held by a clone (i.e. snapshot) + level_counts, + % An array of level counts to speed up compation work assessment + snapshots :: list(), + % A list of snaphots (i.e. clones) + delete_sqn :: integer()|infinity + % The lowest SQN of any clone + }). %%%============================================================================ %%% API %%%============================================================================ new_manifest() -> - ets:new(manifest, [ordered_set]). + Table = ets:new(manifest, [ordered_set]), + new_manifest(Table). open_manifest(RootPath) -> % Open the manifest in the file path which has the highest SQN, and will @@ -75,76 +101,289 @@ open_manifest(RootPath) -> ValidManSQNs = lists:reverse(lists:sort(lists:foldl(ExtractSQNFun, [], Filenames))), - open_manifestfile(RootPath, ValidManSQNs). + {ManSQN, Table} = open_manifestfile(RootPath, ValidManSQNs), + Manifest = new_manifest(Table), + Manifest#manifest{manifest_sqn = ManSQN}. + +copy_manifest(Manifest) -> + % Copy the manifest ensuring anything only the master process should care + % about is switched to undefined + #manifest{is_clone = true, + table = Manifest#manifest.table, + manifest_sqn = Manifest#manifest.manifest_sqn, + pidmap = Manifest#manifest.pidmap}. -save_manifest(Manifest, RootPath, ManSQN) -> - FP = filepath(RootPath, ManSQN, current_manifest), - ets:tab2file(Manifest, +load_manifest(Manifest, PidFun, SQNFun) -> + FlatManifest = ets:tab2list(Manifest#manifest.table), + InitiateFun = + fun({{L, _EK, FN}, {_SK, ActSt, DelSt}}, {MaxSQN, AccMan}) -> + case {ActSt, DelSt} of + {{active, _ActSQN}, {tomb, infinity}} -> + Pid = PidFun(FN), + PidMap0 = dict:store(FN, + {Pid, infinity}, + AccMan#manifest.pidmap), + LC = array:get(L, AccMan#manifest.level_counts), + LC0 = array:set(L, LC + 1, AccMan#manifest.level_counts), + AccMan0 = AccMan#manifest{pidmap = PidMap0, + level_counts = LC0}, + SQN = SQNFun(Pid), + MaxSQN0 = max(MaxSQN, SQN), + {MaxSQN0, AccMan0}; + {_, {tomb, _TombSQN}} -> + {MaxSQN, AccMan} + end + end, + lists:foldl(InitiateFun, {1, Manifest}, FlatManifest). + +save_manifest(Manifest, RootPath) -> + FP = filepath(RootPath, Manifest#manifest.manifest_sqn, current_manifest), + ets:tab2file(Manifest#manifest.table, FP, [{extended_info, [md5sum]}, {sync, true}]). -initiate_from_manifest(Manifest) -> - FlatManifest = ets:tab2list(Manifest), - InitiateFun = - fun({{L, _EK, FN}, {_SK, ActSt, DelSt}}, {FNList, MaxSQN, LCount}) -> - case {ActSt, DelSt} of - {{active, ActSQN}, {tomb, infinity}} -> - {[FN|FNList], - max(ActSQN, MaxSQN), - dict:update_counter(L, 1, LCount)}; - {_, {tomb, TombSQN}} -> - {FNList, max(TombSQN, MaxSQN), LCount} - end - end, - lists:foldl(InitiateFun, {[], 0, dict:new()}, FlatManifest). - - insert_manifest_entry(Manifest, ManSQN, Level, Entry) -> Key = {Level, Entry#manifest_entry.end_key, Entry#manifest_entry.filename}, + Pid = Entry#manifest_entry.owner, Value = {Entry#manifest_entry.start_key, {active, ManSQN}, {tomb, infinity}}, - true = ets:insert_new(Manifest, {Key, Value}). + true = ets:insert_new(Manifest#manifest.table, {Key, Value}), + PidMap0 = dict:store(Entry#manifest_entry.filename, + {Pid, infinity}, + Manifest#manifest.pidmap), + LC = array:get(Level, Manifest#manifest.level_counts), + LCArray0 = array:set(Level, LC + 1, Manifest#manifest.level_counts), + MaxManSQN = max(ManSQN, Manifest#manifest.manifest_sqn), + Manifest#manifest{pidmap = PidMap0, + level_counts = LCArray0, + manifest_sqn = MaxManSQN}. remove_manifest_entry(Manifest, ManSQN, Level, Entry) -> Key = {Level, Entry#manifest_entry.end_key, Entry#manifest_entry.filename}, [{Key, Value0}] = ets:lookup(Manifest, Key), {StartKey, {active, ActiveSQN}, {tomb, infinity}} = Value0, Value1 = {StartKey, {active, ActiveSQN}, {tomb, ManSQN}}, - true = ets:insert(Manifest, {Key, Value1}). + true = ets:insert(Manifest#manifest.table, {Key, Value1}), + {Pid, infinity} = dict:fetch(Entry#manifest_entry.filename, + Manifest#manifest.pidmap), + PidMap0 = dict:store(Entry#manifest_entry.filename, + {Pid, ManSQN}, + Manifest#manifest.pidmap), + LC = array:get(Level, Manifest#manifest.level_counts), + LCArray0 = array:set(Level, LC - 1, Manifest#manifest.level_counts), + MaxManSQN = max(ManSQN, Manifest#manifest.manifest_sqn), + Manifest#manifest{pidmap = PidMap0, + level_counts = LCArray0, + manifest_sqn = MaxManSQN}. -key_lookup(Manifest, Level, Key, ManSQN) -> - key_lookup(Manifest, Level, Key, ManSQN, false). +get_manifest_sqn(Manifest) -> + Manifest#manifest.manifest_sqn. -key_lookup(Manifest, Level, Key, ManSQN, GC) -> - key_lookup(Manifest, Level, {Key, 0}, Key, ManSQN, GC). +key_lookup(Manifest, Level, Key) -> + GC = + case Manifest#manifest.is_clone of + true -> + false; + false -> + {true, Manifest#manifest.delete_sqn} + end, + FN = key_lookup(Manifest#manifest.table, + Level, + Key, + Manifest#manifest.manifest_sqn, + GC), + case FN of + false -> + false; + _ -> + {Pid, _TombSQN} = dict:fetch(FN, Manifest#manifest.pidmap), + Pid + end. + +range_lookup(Manifest, Level, StartKey, EndKey) -> + MapFun = + fun({{_Level, _LastKey, FN}, FirstKey}) -> + {next, dict:fetch(FN, Manifest#manifest.pidmap), FirstKey} + end, + range_lookup(Manifest, Level, StartKey, EndKey, MapFun). -range_lookup(Manifest, Level, StartKey, EndKey, ManSQN) -> - range_lookup(Manifest, Level, {StartKey, 0}, StartKey, EndKey, [], ManSQN). +merge_lookup(Manifest, Level, StartKey, EndKey) -> + MapFun = + fun({{_Level, LastKey, FN}, FirstKey}) -> + Owner = dict:fetch(FN, Manifest#manifest.pidmap), + #manifest_entry{filename = FN, + owner = Owner, + start_key = FirstKey, + end_key = LastKey} + end, + range_lookup(Manifest, Level, StartKey, EndKey, MapFun). -add_snapshot(SnapList0, Pid, ManifestSQN, Timeout) -> - [{Pid, ManifestSQN, Timeout}|SnapList0]. +pointer_convert(Manifest, EntryList) -> + MapFun = + fun(Entry) -> + {next, + dict:fetch(Entry#manifest_entry.filename, + Manifest#manifest.pidmap), + all} + end, + lists:map(MapFun, EntryList). -release_snapshot(SnapList0, Pid) -> +%% An algorithm for discovering which files to merge .... +%% We can find the most optimal file: +%% - The one with the most overlapping data below? +%% - The one that overlaps with the fewest files below? +%% - The smallest file? +%% We could try and be fair in some way (merge oldest first) +%% Ultimately, there is a lack of certainty that being fair or optimal is +%% genuinely better - eventually every file has to be compacted. +%% +%% Hence, the initial implementation is to select files to merge at random +mergefile_selector(Manifest, Level) -> + KL = range_lookup(Manifest#manifest.table, + Level, + {all, 0}, + all, + ?END_KEY, + [], + Manifest#manifest.manifest_sqn), + {{Level, LastKey, FN}, + FirstKey} = lists:nth(random:uniform(length(KL)), KL), + {Owner, infinity} = dict:fetch(FN, Manifest#manifest.pidmap), + #manifest_entry{filename = FN, + owner = Owner, + start_key = FirstKey, + end_key = LastKey}. + +add_snapshot(Manifest, Pid, Timeout) -> + SnapEntry = {Pid, Manifest#manifest.manifest_sqn, Timeout}, + SnapList0 = [SnapEntry|Manifest#manifest.snapshots], + MinDelSQN = min(Manifest#manifest.delete_sqn, Manifest#manifest.manifest_sqn), + Manifest#manifest{snapshots = SnapList0, delete_sqn = MinDelSQN}. + +release_snapshot(Manifest, Pid) -> FilterFun = - fun({P, SQN, TS}, Acc) -> + fun({P, SQN, TS}, {Acc, MinSQN}) -> case P of Pid -> Acc; _ -> - [{P, SQN, TS}|Acc] + {[{P, SQN, TS}|Acc], min(SQN, MinSQN)} end end, - lists:foldl(FilterFun, [], SnapList0). + {SnapList0, DeleteSQN} = lists:foldl(FilterFun, + {[], infinity}, + Manifest#manifest.snapshots), + leveled_log:log("P0004", [SnapList0]), + Manifest#manifest{snapshots = SnapList0, delete_sqn = DeleteSQN}. -ready_to_delete(SnapList0, DeleteSQN) -> - ready_to_delete(SnapList0, DeleteSQN, os:timestamp()). +ready_to_delete(Manifest, Filename) -> + case dict:fetch(Filename, Manifest#manifest.pidmap) of + {P, infinity} -> + {false, P}; + {P, DeleteSQN} -> + {ready_to_delete(Manifest#manifest.snapshots, + DeleteSQN, + os:timestamp()), + P} + end. + +check_for_work(Manifest, Thresholds) -> + CheckLevelFun = + fun({Level, MaxCount}, {AccL, AccC}) -> + case dict:fetch(Level, Manifest#manifest.level_counts) of + LC when LC > MaxCount -> + {[Level|AccL], AccC + LC - MaxCount}; + _ -> + {AccL, AccC} + end + end, + lists:foldl(CheckLevelFun, {[], 0}, Thresholds). + +is_basement(Manifest, Level) -> + CheckFun = + fun(L, Acc) -> + case array:get(L, Manifest#manifest.level_counts) of + 0 -> + Acc; + _N -> + false + end + end, + lists:foldl(CheckFun, true, lists:seq(Level + 1, ?MAX_LEVELS)). + +dump_pidmap(Manifest) -> + dict:to_list(Manifest#manifest.pidmap). + +levelzero_present(Manifest) -> + case key_lookup(Manifest, 0, all) of + false -> + false; + _ -> + true + end. %%%============================================================================ %%% Internal Functions %%%============================================================================ + +new_manifest(Table) -> + #manifest{ + table = Table, + pidmap = dict:new(), + level_counts = array:new([{size, ?MAX_LEVELS + 1}, {default, 0}]), + snapshots = [], + delete_sqn = infinity + }. + +range_lookup(Manifest, Level, StartKey, EndKey, MapFun) -> + KL = range_lookup(Manifest#manifest.table, + Level, + {StartKey, 0}, + StartKey, + EndKey, + [], + Manifest#manifest.manifest_sqn), + lists:map(MapFun, KL). + +range_lookup(Manifest, Level, {LastKey, LastFN}, SK, EK, Acc, ManSQN) -> + case ets:next(Manifest, {Level, LastKey, LastFN}) of + '$end_of_table' -> + Acc; + {Level, NextKey, NextFN} -> + [{K, V}] = ets:lookup(Manifest, {Level, NextKey, NextFN}), + {FirstKey, {active, ActiveSQN}, {tomb, TombSQN}} = V, + Active = (ManSQN >= ActiveSQN) and (ManSQN < TombSQN), + case Active of + true -> + PostEnd = leveled_codec:endkey_passed(EK, FirstKey), + case PostEnd of + true -> + Acc; + false -> + range_lookup(Manifest, + Level, + {NextKey, NextFN}, + SK, + EK, + Acc ++ [{K, FirstKey}], + ManSQN) + end; + false -> + range_lookup(Manifest, + Level, + {NextKey, NextFN}, + SK, + EK, + Acc, + ManSQN) + end; + {OtherLevel, _, _} when OtherLevel > Level -> + Acc + end. + ready_to_delete(SnapList, FileDeleteSQN, Now) -> FilterFun = fun({P, SnapSQN, ExpiryTS}, Acc) -> @@ -179,10 +418,10 @@ filepath(RootPath, NewMSN, current_manifest) -> open_manifestfile(_RootPath, []) -> leveled_log:log("P0013", []), - new_manifest(); + {0, new_manifest()}; open_manifestfile(_RootPath, [0]) -> leveled_log:log("P0013", []), - new_manifest(); + {0, new_manifest()}; open_manifestfile(RootPath, [TopManSQN|Rest]) -> CurrManFile = filepath(RootPath, TopManSQN, current_manifest), case ets:file2tab(CurrManFile, [{verify,true}]) of @@ -191,9 +430,12 @@ open_manifestfile(RootPath, [TopManSQN|Rest]) -> open_manifestfile(RootPath, Rest); {ok, Table} -> leveled_log:log("P0012", [TopManSQN]), - Table + {TopManSQN, Table} end. +key_lookup(Manifest, Level, KeyToFind, ManSQN, GC) -> + key_lookup(Manifest, Level, {KeyToFind, any}, KeyToFind, ManSQN, GC). + key_lookup(Manifest, Level, {LastKey, LastFN}, KeyToFind, ManSQN, GC) -> case ets:next(Manifest, {Level, LastKey, LastFN}) of '$end_of_table' -> @@ -234,41 +476,6 @@ key_lookup(Manifest, Level, {LastKey, LastFN}, KeyToFind, ManSQN, GC) -> false end. -range_lookup(Manifest, Level, {LastKey, LastFN}, SK, EK, Acc, ManSQN) -> - case ets:next(Manifest, {Level, LastKey, LastFN}) of - '$end_of_table' -> - Acc; - {Level, NextKey, NextFN} -> - [{_K, V}] = ets:lookup(Manifest, {Level, NextKey, NextFN}), - {FirstKey, {active, ActiveSQN}, {tomb, TombSQN}} = V, - Active = (ManSQN >= ActiveSQN) and (ManSQN < TombSQN), - case Active of - true -> - PostEnd = leveled_codec:endkey_passed(EK, FirstKey), - case PostEnd of - true -> - Acc; - false -> - range_lookup(Manifest, - Level, - {NextKey, NextFN}, - SK, - EK, - Acc ++ [NextFN], - ManSQN) - end; - false -> - range_lookup(Manifest, - Level, - {NextKey, NextFN}, - SK, - EK, - Acc, - ManSQN) - end; - {OtherLevel, _, _} when OtherLevel > Level -> - Acc - end. %%%============================================================================ %%% Test @@ -276,8 +483,6 @@ range_lookup(Manifest, Level, {LastKey, LastFN}, SK, EK, Acc, ManSQN) -> -ifdef(TEST). - - rangequery_manifest_test() -> E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, @@ -298,7 +503,7 @@ rangequery_manifest_test() -> end_key={o, "Bucket1", "K996", null}, filename="Z6"}, - Manifest = open_manifestfile(dummy, []), + Manifest = new_manifest(), insert_manifest_entry(Manifest, 1, 1, E1), insert_manifest_entry(Manifest, 1, 1, E2), insert_manifest_entry(Manifest, 1, 1, E3), @@ -308,22 +513,22 @@ rangequery_manifest_test() -> SK1 = {o, "Bucket1", "K711", null}, EK1 = {o, "Bucket1", "K999", null}, - RL1_1 = range_lookup(Manifest, 1, SK1, EK1, 1), + RL1_1 = range_lookup(Manifest, 1, SK1, EK1), ?assertMatch(["Z3"], RL1_1), - RL1_2 = range_lookup(Manifest, 2, SK1, EK1, 1), + RL1_2 = range_lookup(Manifest, 2, SK1, EK1), ?assertMatch(["Z5", "Z6"], RL1_2), SK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, EK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, - RL2_1 = range_lookup(Manifest, 1, SK2, EK2, 1), + RL2_1 = range_lookup(Manifest, 1, SK2, EK2), ?assertMatch(["Z1"], RL2_1), - RL2_2 = range_lookup(Manifest, 2, SK2, EK2, 1), + RL2_2 = range_lookup(Manifest, 2, SK2, EK2), ?assertMatch(["Z5"], RL2_2), SK3 = {o, "Bucket1", "K994", null}, EK3 = {o, "Bucket1", "K995", null}, - RL3_1 = range_lookup(Manifest, 1, SK3, EK3, 1), + RL3_1 = range_lookup(Manifest, 1, SK3, EK3), ?assertMatch([], RL3_1), - RL3_2 = range_lookup(Manifest, 2, SK3, EK3, 1), + RL3_2 = range_lookup(Manifest, 2, SK3, EK3), ?assertMatch(["Z6"], RL3_2), E1_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld4"}, "K8"}, @@ -347,172 +552,19 @@ rangequery_manifest_test() -> remove_manifest_entry(Manifest, 2, 1, E2), remove_manifest_entry(Manifest, 2, 1, E3), - RL1_1A = range_lookup(Manifest, 1, SK1, EK1, 1), + RL1_1A = range_lookup(Manifest, 1, SK1, EK1), ?assertMatch(["Z3"], RL1_1A), - RL2_1A = range_lookup(Manifest, 1, SK2, EK2, 1), + RL2_1A = range_lookup(Manifest, 1, SK2, EK2), ?assertMatch(["Z1"], RL2_1A), - RL3_1A = range_lookup(Manifest, 1, SK3, EK3, 1), + RL3_1A = range_lookup(Manifest, 1, SK3, EK3), ?assertMatch([], RL3_1A), - RL1_1B = range_lookup(Manifest, 1, SK1, EK1, 2), + RL1_1B = range_lookup(Manifest, 1, SK1, EK1), ?assertMatch(["Y3", "Y4"], RL1_1B), - RL2_1B = range_lookup(Manifest, 1, SK2, EK2, 2), + RL2_1B = range_lookup(Manifest, 1, SK2, EK2), ?assertMatch(["Y1"], RL2_1B), - RL3_1B = range_lookup(Manifest, 1, SK3, EK3, 2), + RL3_1B = range_lookup(Manifest, 1, SK3, EK3), ?assertMatch(["Y4"], RL3_1B). -startup_manifest() - E1 = #manifest_entry{start_key={o, "Bucket1", "K0001", null}, - end_key={o, "Bucket1", "K0990", null}, - filename="Z1"}, - E2 = #manifest_entry{start_key={o, "Bucket1", "K1003", null}, - end_key={o, "Bucket1", "K3692", null}, - filename="Z2"}, - E3 = #manifest_entry{start_key={o, "Bucket1", "K3750", null}, - end_key={o, "Bucket1", "K9930", null}, - filename="Z3"}, - - Manifest0 = open_manifestfile(dummy, []), - insert_manifest_entry(Manifest0, 1, 1, E1), - insert_manifest_entry(Manifest0, 1, 1, E2), - insert_manifest_entry(Manifest0, 1, 1, E3), - Manifest0 - -keyquery_manifest_test() -> - Manifest0 = startup_manifest(), - - EToRemove = #manifest_entry{start_key={o, "Bucket99", "K3750", null}, - end_key={o, "Bucket99", "K9930", null}, - filename="ZR"}, - insert_manifest_entry(Manifest0, 1, 1, EToRemove), - remove_manifest_entry(Manifest0, 2, 1, EToRemove), - - RootPath = "../test", - ok = filelib:ensure_dir(filepath(RootPath, manifest)), - ok = save_manifest(Manifest0, RootPath, 2), - true = ets:delete(Manifest0), - ?assertMatch(true, filelib:is_file(filepath(RootPath, - 2, - current_manifest))), - - BadFP = filepath(RootPath, 3, current_manifest), - ok = file:write_file(BadFP, list_to_binary("nonsense")), - ?assertMatch(true, filelib:is_file(BadFP)), - - Manifest = open_manifest(RootPath), - {FNList, ManSQN, LCount} = initiate_from_manifest(Manifest), - ?assertMatch(["Z1", "Z2", "Z3"], lists:sort(FNList)), - ?assertMatch(2, ManSQN), - ?assertMatch(3, dict:fetch(1, LCount)), - - K1 = {o, "Bucket1", "K0000", null}, - K2 = {o, "Bucket1", "K0001", null}, - K3 = {o, "Bucket1", "K0002", null}, - K4 = {o, "Bucket1", "K0990", null}, - K5 = {o, "Bucket1", "K0991", null}, - K6 = {o, "Bucket1", "K1003", null}, - K7 = {o, "Bucket1", "K1004", null}, - K8 = {o, "Bucket1", "K3692", null}, - K9 = {o, "Bucket1", "K3693", null}, - K10 = {o, "Bucket1", "K3750", null}, - K11 = {o, "Bucket1", "K3751", null}, - K12 = {o, "Bucket1", "K9930", null}, - K13 = {o, "Bucket1", "K9931", null}, - - ?assertMatch(false, key_lookup(Manifest, 1, K1, 2)), - ?assertMatch("Z1", key_lookup(Manifest, 1, K2, 2)), - ?assertMatch("Z1", key_lookup(Manifest, 1, K3, 2)), - ?assertMatch("Z1", key_lookup(Manifest, 1, K4, 2)), - ?assertMatch(false, key_lookup(Manifest, 1, K5, 2)), - ?assertMatch("Z2", key_lookup(Manifest, 1, K6, 2)), - ?assertMatch("Z2", key_lookup(Manifest, 1, K7, 2)), - ?assertMatch("Z2", key_lookup(Manifest, 1, K8, 2)), - ?assertMatch(false, key_lookup(Manifest, 1, K9, 2)), - ?assertMatch("Z3", key_lookup(Manifest, 1, K10, 2)), - ?assertMatch("Z3", key_lookup(Manifest, 1, K11, 2)), - ?assertMatch("Z3", key_lookup(Manifest, 1, K12, 2)), - ?assertMatch(false, key_lookup(Manifest, 1, K13, 2)), - - E1_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld4"}, "K8"}, - end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K62"}, - filename="Y1"}, - E2_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K67"}, - end_key={o, "Bucket1", "K45", null}, - filename="Y2"}, - E3_2 = #manifest_entry{start_key={o, "Bucket1", "K47", null}, - end_key={o, "Bucket1", "K812", null}, - filename="Y3"}, - E4_2 = #manifest_entry{start_key={o, "Bucket1", "K815", null}, - end_key={o, "Bucket1", "K998", null}, - filename="Y4"}, - - insert_manifest_entry(Manifest, 3, 1, E1_2), - insert_manifest_entry(Manifest, 3, 1, E2_2), - insert_manifest_entry(Manifest, 3, 1, E3_2), - insert_manifest_entry(Manifest, 3, 1, E4_2), - - S1 = ets:info(Manifest, size), - - remove_manifest_entry(Manifest, 3, 1, E1), - remove_manifest_entry(Manifest, 3, 1, E2), - remove_manifest_entry(Manifest, 3, 1, E3), - - S2 = ets:info(Manifest, size), - ?assertMatch(true, S2 == S1), - - ?assertMatch("Y2", key_lookup(Manifest, 1, K1, 3)), - ?assertMatch("Y2", key_lookup(Manifest, 1, K10, 3)), - ?assertMatch("Y4", key_lookup(Manifest, 1, K12, 3)), - - S3 = ets:info(Manifest, size), - ?assertMatch(true, S3 == S1), - - ?assertMatch("Y2", key_lookup(Manifest, 1, K1, 3, {true, 3})), - ?assertMatch("Y2", key_lookup(Manifest, 1, K10, 3, {true, 3})), - ?assertMatch("Y4", key_lookup(Manifest, 1, K12, 3, {true, 3})), - - S4 = ets:info(Manifest, size), - ?assertMatch(true, S4 == S1), - - ?assertMatch("Y2", key_lookup(Manifest, 1, K1, 4, {true, 4})), - ?assertMatch("Y2", key_lookup(Manifest, 1, K10, 4, {true, 4})), - ?assertMatch("Y4", key_lookup(Manifest, 1, K12, 4, {true, 4})), - - S5 = ets:info(Manifest, size), - ?assertMatch(true, S5 < S1). - -snapshot_test() -> - Snap0 = [], - - ?assertMatch(true, ready_to_delete(Snap0, 1)), - - {MegaS0, S0, MicroS0} = os:timestamp(), - - Snap1 = add_snapshot(Snap0, pid_1, 3, {MegaS0, S0 + 100, MicroS0}), - Snap2 = add_snapshot(Snap1, pid_2, 4, {MegaS0, S0 + 200, MicroS0}), - Snap3 = add_snapshot(Snap2, pid_3, 4, {MegaS0, S0 + 150, MicroS0}), - Snap4 = add_snapshot(Snap3, pid_4, 5, {MegaS0, S0 + 300, MicroS0}), - - ?assertMatch(true, - ready_to_delete(Snap4, 2, {MegaS0, S0, MicroS0})), - ?assertMatch(false, - ready_to_delete(Snap4, 3, {MegaS0, S0, MicroS0})), - ?assertMatch(true, - ready_to_delete(Snap4, 3, {MegaS0, S0 + 150, MicroS0})), - ?assertMatch(false, - ready_to_delete(Snap4, 4, {MegaS0, S0 + 150, MicroS0})), - ?assertMatch(true, - ready_to_delete(Snap4, 4, {MegaS0, S0 + 250, MicroS0})), - - Snap5 = release_snapshot(Snap4, pid_1), - ?assertMatch(true, - ready_to_delete(Snap5, 3, {MegaS0, S0, MicroS0})). - - -allatlevel_test() -> - Manifest0 = startup_manifest(), - AllAtL1 = range_lookup(Manifest, 1, all, {null, null, null, null}, 1), - ?assertMatch(["Z1", "Z2", "Z3"], AllAtL1). - -endif. \ No newline at end of file diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 58ed347..a0ddbcd 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -25,26 +25,28 @@ -include("include/leveled.hrl"). --export([init/1, +-export([ + init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - clerk_new/1, + code_change/3 + ]). + +-export([ + clerk_new/2, clerk_prompt/1, - clerk_manifestchange/3, - code_change/3]). + clerk_close/1 + ]). -include_lib("eunit/include/eunit.hrl"). --define(MAX_TIMEOUT, 2000). --define(MIN_TIMEOUT, 50). --define(END_KEY, {null, null, null, null}). +-define(MAX_TIMEOUT, 1000). +-define(MIN_TIMEOUT, 200). -record(state, {owner :: pid(), - manifest, % ets table reference - change_pending=false :: boolean(), - work_item :: #penciller_work{}|null}). + root_path :: string()}). %%%============================================================================ %%% API @@ -69,26 +71,22 @@ clerk_close(Pid) -> init([]) -> {ok, #state{}}. -handle_call({load, Owner, Manifest}, _From, State) -> - {reply, - ok, - State#state{owner=Owner, manifest=Manifest}, - ?MIN_TIMEOUT}. +handle_call({load, Owner, RootPath}, _From, State) -> + {reply, ok, State#state{owner=Owner, root_path=RootPath}, ?MIN_TIMEOUT}. handle_cast(prompt, State) -> - {noreply, State, ?MIN_TIMEOUT}; + handle_info(timeout, State); handle_cast(close, State) -> - (stop, normal, State). + {stop, normal, State}. -handle_info(timeout, State=#state{change_pending=Pnd}) when Pnd == false -> +handle_info(timeout, State) -> case requestandhandle_work(State) of - {false, Timeout} -> - {noreply, State, Timeout}; - {true, WI} -> + false -> + {noreply, State, ?MAX_TIMEOUT}; + true -> % No timeout now as will wait for call to return manifest % change - {noreply, - State#state{change_pending=true, work_item=WI}} + {noreply, State, ?MIN_TIMEOUT} end. @@ -105,182 +103,116 @@ code_change(_OldVsn, State, _Extra) -> requestandhandle_work(State) -> case leveled_penciller:pcl_workforclerk(State#state.owner) of - false -> + none -> leveled_log:log("PC006", []), false; - {SrcLevel, ManifestSQN} -> - {Additions, Removals} = merge(Level, - State#state.manifest, - ManifestSQN), + {SrcLevel, Manifest} -> + {UpdManifest, EntriesToDelete} = merge(SrcLevel, + Manifest, + State#state.root_path), leveled_log:log("PC007", []), ok = leveled_penciller:pcl_commitmanifestchange(State#state.owner, - SrcLevel, - Additions, - Removals, - ManifestSQN), + UpdManifest), + ok = leveled_manifest:save_manifest(UpdManifest, + State#state.root_path), + ok = notify_deletions(EntriesToDelete, State#state.owner), true end. -merge(SrcLevel, Manifest, ManifestSQN) -> - SrcF = select_filetomerge(SrcLevel, Manifest), - - - Candidates = check_for_merge_candidates(SrcF, SinkFiles), - %% TODO: - %% Need to work out if this is the top level - %% And then tell merge process to create files at the top level - %% Which will include the reaping of expired tombstones - leveled_log:log("PC008", [SrcLevel, length(Candidates)]), - - MergedFiles = case length(Candidates) of +merge(SrcLevel, Manifest, RootPath) -> + Src = leveled_manifest:mergefile_selector(Manifest, SrcLevel), + NewSQN = leveled_manifest:get_manifest_sqn(Manifest) + 1, + SinkList = leveled_manifest:merge_lookup(Manifest, + SrcLevel + 1, + Src#manifest_entry.start_key, + Src#manifest_entry.end_key), + Candidates = length(SinkList), + leveled_log:log("PC008", [SrcLevel, Candidates]), + case Candidates of 0 -> %% If no overlapping candiates, manifest change only required %% %% TODO: need to think still about simply renaming when at %% lower level leveled_log:log("PC009", - [SrcF#manifest_entry.filename, SrcLevel + 1]), - [SrcF]; + [Src#manifest_entry.filename, SrcLevel + 1]), + Man0 = leveled_manifest:remove_manifest_entry(Manifest, + NewSQN, + SrcLevel, + Src), + Man1 = leveled_manifest:insert_manifest_entry(Man0, + NewSQN, + SrcLevel + 1, + Src), + {Man1, []}; _ -> - perform_merge({SrcF#manifest_entry.owner, - SrcF#manifest_entry.filename}, - Candidates, - {SrcLevel, WI#penciller_work.target_is_basement}, - {WI#penciller_work.ledger_filepath, - WI#penciller_work.next_sqn}) - end, - NewLevel = lists:sort(lists:append(MergedFiles, Others)), - UpdMFest2 = lists:keystore(SrcLevel + 1, - 1, - UpdMFest1, - {SrcLevel + 1, NewLevel}), - - ok = filelib:ensure_dir(WI#penciller_work.manifest_file), - {ok, Handle} = file:open(WI#penciller_work.manifest_file, - [binary, raw, write]), - ok = file:write(Handle, term_to_binary(UpdMFest2)), - ok = file:close(Handle), - case lists:member(SrcF, MergedFiles) of - true -> - {UpdMFest2, Candidates}; - false -> - %% Can rub out src file as it is not part of output - {UpdMFest2, Candidates ++ [SrcF]} + FilePath = leveled_penciller:filepath(RootPath, + NewSQN, + new_merge_files), + perform_merge(Manifest, Src, SinkList, SrcLevel, FilePath, NewSQN) end. - -mark_for_delete([], _Penciller) -> +notify_deletions([], _Penciller) -> ok; -mark_for_delete([Head|Tail], Penciller) -> +notify_deletions([Head|Tail], Penciller) -> ok = leveled_sst:sst_setfordelete(Head#manifest_entry.owner, Penciller), - mark_for_delete(Tail, Penciller). - - -check_for_merge_candidates(SrcF, SinkFiles) -> - lists:partition(fun(Ref) -> - case {Ref#manifest_entry.start_key, - Ref#manifest_entry.end_key} of - {_, EK} when SrcF#manifest_entry.start_key > EK -> - false; - {SK, _} when SrcF#manifest_entry.end_key < SK -> - false; - _ -> - true - end end, - SinkFiles). - - -%% An algorithm for discovering which files to merge .... -%% We can find the most optimal file: -%% - The one with the most overlapping data below? -%% - The one that overlaps with the fewest files below? -%% - The smallest file? -%% We could try and be fair in some way (merge oldest first) -%% Ultimately, there is a lack of certainty that being fair or optimal is -%% genuinely better - eventually every file has to be compacted. -%% -%% Hence, the initial implementation is to select files to merge at random - -select_filetomerge(SrcLevel, Manifest, ManifestSQN) -> - Level = leveled_manifest:range_lookup(Manifest, - 1, - all, - ?END_KEY, - ManifestSQN), - - FN = lists:nth(random:uniform(length(Level)), Level). - - + notify_deletions(Tail, Penciller). + %% Assumption is that there is a single SST from a higher level that needs -%% to be merged into multiple SSTs at a lower level. This should create an -%% entirely new set of SSTs, and the calling process can then update the -%% manifest. +%% to be merged into multiple SSTs at a lower level. %% -%% Once the FileToMerge has been emptied, the remainder of the candidate list -%% needs to be placed in a remainder SST that may be of a sub-optimal (small) -%% size. This stops the need to perpetually roll over the whole level if the -%% level consists of already full files. Some smartness may be required when -%% selecting the candidate list so that small files just outside the candidate -%% list be included to avoid a proliferation of small files. -%% -%% FileToMerge should be a tuple of {FileName, Pid} where the Pid is the Pid of -%% the gen_server leveled_sft process representing the file. -%% -%% CandidateList should be a list of {StartKey, EndKey, Pid} tuples -%% representing different gen_server leveled_sft processes, sorted by StartKey. -%% -%% The level is the level which the new files should be created at. +%% SrcLevel is the level of the src sst file, the sink should be srcLevel + 1 -perform_merge({SrcPid, SrcFN}, CandidateList, LevelInfo, {Filepath, MSN}) -> - leveled_log:log("PC010", [SrcFN, MSN]), - PointerList = lists:map(fun(P) -> - {next, P#manifest_entry.owner, all} end, - CandidateList), - MaxSQN = leveled_sst:sst_getmaxsequencenumber(SrcPid), - do_merge([{next, SrcPid, all}], - PointerList, - LevelInfo, - {Filepath, MSN}, - MaxSQN, - 0, - []). +perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN) -> + leveled_log:log("PC010", [Src#manifest_entry.filename, NewSQN]), + SrcList = [{next, Src#manifest_entry.owner, all}], + SinkPointerList = leveled_manifest:pointer_convert(Manifest, SinkList), + MaxSQN = leveled_sst:sst_getmaxsequencenumber(Src#manifest_entry.owner), + SinkLevel = SrcLevel + 1, + SinkBasement = leveled_basement:is_basement(Manifest, SinkLevel), + Man0 = do_merge(SrcList, SinkPointerList, + SinkLevel, SinkBasement, + RootPath, NewSQN, MaxSQN, + 0, Manifest), + RemoveFun = + fun(Entry, AccMan) -> + leveled_manifest:remove_manifest_entry(AccMan, + NewSQN, + SinkLevel, + Entry) + end, + Man1 = lists:foldl(RemoveFun, Man0, SinkList), + leveled_manifest:remove_manifest_entry(Man1, NewSQN, SrcLevel, Src). -do_merge([], [], {SrcLevel, _IsB}, {_Filepath, MSN}, _MaxSQN, - FileCounter, OutList) -> - leveled_log:log("PC011", [MSN, SrcLevel, FileCounter]), - OutList; -do_merge(KL1, KL2, {SrcLevel, IsB}, {Filepath, MSN}, MaxSQN, - FileCounter, OutList) -> - FileName = lists:flatten(io_lib:format(Filepath ++ "_~w_~w.sst", - [SrcLevel + 1, FileCounter])), - leveled_log:log("PC012", [MSN, FileName]), +do_merge([], [], SinkLevel, _SinkB, _RP, NewSQN, _MaxSQN, Counter, Man0) -> + leveled_log:log("PC011", [NewSQN, SinkLevel, Counter]), + Man0; +do_merge(KL1, KL2, SinkLevel, SinkB, RP, NewSQN, MaxSQN, Counter, Man0) -> + FileName = lists:flatten(io_lib:format(RP ++ "_~w_~w.sst", + [SinkLevel, Counter])), + leveled_log:log("PC012", [NewSQN, FileName]), TS1 = os:timestamp(), - case leveled_sst:sst_new(FileName, KL1, KL2, IsB, SrcLevel + 1, MaxSQN) of + case leveled_sst:sst_new(FileName, KL1, KL2, SinkB, SinkLevel, MaxSQN) of empty -> leveled_log:log("PC013", [FileName]), - OutList; + Man0; {ok, Pid, Reply} -> {{KL1Rem, KL2Rem}, SmallestKey, HighestKey} = Reply, - ExtMan = lists:append(OutList, - [#manifest_entry{start_key=SmallestKey, - end_key=HighestKey, - owner=Pid, - filename=FileName}]), + Entry = #manifest_entry{start_key=SmallestKey, + end_key=HighestKey, + owner=Pid, + filename=FileName}, + Man1 = leveled_manifest:insert_manifest_entry(Man0, + NewSQN, + SinkLevel, + Entry), leveled_log:log_timer("PC015", [], TS1), do_merge(KL1Rem, KL2Rem, - {SrcLevel, IsB}, {Filepath, MSN}, MaxSQN, - FileCounter + 1, ExtMan) - end. - - -get_item(Index, List, Default) -> - case lists:keysearch(Index, 1, List) of - {value, {Index, Value}} -> - Value; - false -> - Default + SinkLevel, SinkB, + RP, NewSQN, MaxSQN, + Counter + 1, Man1) end. @@ -306,26 +238,6 @@ generate_randomkeys(Count, Acc, BucketLow, BRange) -> null}}, generate_randomkeys(Count - 1, [RandKey|Acc], BucketLow, BRange). -choose_pid_toquery([ManEntry|_T], Key) when - Key >= ManEntry#manifest_entry.start_key, - ManEntry#manifest_entry.end_key >= Key -> - ManEntry#manifest_entry.owner; -choose_pid_toquery([_H|T], Key) -> - choose_pid_toquery(T, Key). - - -find_randomkeys(_FList, 0, _Source) -> - ok; -find_randomkeys(FList, Count, Source) -> - KV1 = lists:nth(random:uniform(length(Source)), Source), - K1 = leveled_codec:strip_to_keyonly(KV1), - P1 = choose_pid_toquery(FList, K1), - FoundKV = leveled_sst:sst_get(P1, K1), - Found = leveled_codec:strip_to_keyonly(FoundKV), - io:format("success finding ~w in ~w~n", [K1, P1]), - ?assertMatch(K1, Found), - find_randomkeys(FList, Count - 1, Source). - merge_file_test() -> KL1_L1 = lists:sort(generate_randomkeys(8000, 0, 1000)), @@ -353,57 +265,22 @@ merge_file_test() -> 2, KL4_L2, undefined), - Result = perform_merge({PidL1_1, "../test/KL1_L1.sst"}, - [#manifest_entry{owner=PidL2_1}, - #manifest_entry{owner=PidL2_2}, - #manifest_entry{owner=PidL2_3}, - #manifest_entry{owner=PidL2_4}], - {2, false}, {"../test/", 99}), - lists:foreach(fun(ManEntry) -> - {o, B1, K1} = ManEntry#manifest_entry.start_key, - {o, B2, K2} = ManEntry#manifest_entry.end_key, - io:format("Result of ~s ~s and ~s ~s with Pid ~w~n", - [B1, K1, B2, K2, ManEntry#manifest_entry.owner]) end, - Result), - io:format("Finding keys in KL1_L1~n"), - ok = find_randomkeys(Result, 50, KL1_L1), - io:format("Finding keys in KL1_L2~n"), - ok = find_randomkeys(Result, 50, KL1_L2), - io:format("Finding keys in KL2_L2~n"), - ok = find_randomkeys(Result, 50, KL2_L2), - io:format("Finding keys in KL3_L2~n"), - ok = find_randomkeys(Result, 50, KL3_L2), - io:format("Finding keys in KL4_L2~n"), - ok = find_randomkeys(Result, 50, KL4_L2), - leveled_sst:sst_clear(PidL1_1), - leveled_sst:sst_clear(PidL2_1), - leveled_sst:sst_clear(PidL2_2), - leveled_sst:sst_clear(PidL2_3), - leveled_sst:sst_clear(PidL2_4), - lists:foreach(fun(ManEntry) -> - leveled_sst:sst_clear(ManEntry#manifest_entry.owner) end, - Result). - -select_merge_candidates_test() -> - Sink1 = #manifest_entry{start_key = {o, "Bucket", "Key1"}, - end_key = {o, "Bucket", "Key20000"}}, - Sink2 = #manifest_entry{start_key = {o, "Bucket", "Key20001"}, - end_key = {o, "Bucket1", "Key1"}}, - Src1 = #manifest_entry{start_key = {o, "Bucket", "Key40001"}, - end_key = {o, "Bucket", "Key60000"}}, - {Candidates, Others} = check_for_merge_candidates(Src1, [Sink1, Sink2]), - ?assertMatch([Sink2], Candidates), - ?assertMatch([Sink1], Others). - - -select_merge_file_test() -> - L0 = [{{o, "B1", "K1"}, {o, "B3", "K3"}, dummy_pid}], - L1 = [{{o, "B1", "K1"}, {o, "B2", "K2"}, dummy_pid}, - {{o, "B2", "K3"}, {o, "B4", "K4"}, dummy_pid}], - Manifest = [{0, L0}, {1, L1}], - {FileRef, NewManifest} = select_filetomerge(0, Manifest), - ?assertMatch(FileRef, {{o, "B1", "K1"}, {o, "B3", "K3"}, dummy_pid}), - ?assertMatch(NewManifest, [{0, []}, {1, L1}]). + E1 = #manifest_entry{owner = PidL1_1, filename = "../test/KL1_L1.sst"}, + E2 = #manifest_entry{owner = PidL2_1, filename = "../test/KL1_L2.sst"}, + E3 = #manifest_entry{owner = PidL2_2, filename = "../test/KL2_L2.sst"}, + E4 = #manifest_entry{owner = PidL2_3, filename = "../test/KL3_L2.sst"}, + E5 = #manifest_entry{owner = PidL2_4, filename = "../test/KL4_L2.sst"}, + + Man0 = leveled_manifest:new_manifest(), + Man1 = leveled_manifest:insert_manifest_entry(Man0, 1, 2, E1), + Man2 = leveled_manifest:insert_manifest_entry(Man1, 1, 2, E1), + Man3 = leveled_manifest:insert_manifest_entry(Man2, 1, 2, E1), + Man4 = leveled_manifest:insert_manifest_entry(Man3, 1, 2, E1), + Man5 = leveled_manifest:insert_manifest_entry(Man4, 2, 1, E1), + + Man6 = perform_merge(Man5, E1, [E2, E3, E4, E5], 1, "../test", 3), + + ?assertMatch(3, leveled_manifest:get_manifest_sqn(Man6)). coverage_cheat_test() -> {ok, _State1} = code_change(null, #state{}, null). diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 0f4fa2d..d4423cb 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -161,12 +161,15 @@ -include("include/leveled.hrl"). --export([init/1, +-export([ + init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3, + code_change/3]). + +-export([ pcl_start/1, pcl_pushmem/2, pcl_fetchlevelzero/2, @@ -184,7 +187,10 @@ pcl_registersnapshot/2, pcl_releasesnapshot/2, pcl_loadsnapshot/2, - pcl_getstartupsequencenumber/1, + pcl_getstartupsequencenumber/1]). + +-export([ + filepath/3, clean_testdir/1]). -include_lib("eunit/include/eunit.hrl"). @@ -208,13 +214,8 @@ -define(ITERATOR_SCANWIDTH, 4). -define(SNAPSHOT_TIMEOUT, 3600). --record(state, {manifest, % an ETS table reference - manifest_sqn = 0 :: integer(), +-record(state, {manifest, % a manifest record from the leveled_manifest module persisted_sqn = 0 :: integer(), % The highest SQN persisted - registered_snapshots = [] :: list(), - pidmap = dict:new() :: dict(), - level_counts :: dict(), - deletions_pending = dict:new() ::dict(), ledger_sqn = 0 :: integer(), % The highest SQN added to L0 root_path = "../test" :: string(), @@ -290,8 +291,8 @@ pcl_checksequencenumber(Pid, Key, SQN) -> pcl_workforclerk(Pid) -> gen_server:call(Pid, work_for_clerk, infinity). -pcl_confirmmanifestchange(Pid, WI) -> - gen_server:cast(Pid, {manifest_change, WI}). +pcl_confirmmanifestchange(Pid, Manifest) -> + gen_server:cast(Pid, {manifest_change, Manifest}). pcl_confirml0complete(Pid, FN, StartKey, EndKey) -> gen_server:cast(Pid, {levelzero_complete, FN, StartKey, EndKey}). @@ -328,9 +329,11 @@ init([PCLopts]) -> {undefined, true} -> SrcPenciller = PCLopts#penciller_options.source_penciller, {ok, State} = pcl_registersnapshot(SrcPenciller, self()), + ManifestClone = leveled_manifest:copy_manifest(State#state.manifest), leveled_log:log("P0001", [self()]), - io:format("Snapshot ledger sqn at ~w~n", [State#state.ledger_sqn]), - {ok, State#state{is_snapshot=true, source_penciller=SrcPenciller}}; + {ok, State#state{is_snapshot=true, + source_penciller=SrcPenciller, + manifest=ManifestClone}}; %% Need to do something about timeout {_RootPath, false} -> start_from_file(PCLopts) @@ -375,24 +378,18 @@ handle_call({push_mem, {PushedTree, PushedIdx, MinSQN, MaxSQN}}, State)} end; handle_call({fetch, Key, Hash}, _From, State) -> - Structure = {State#state.manifest, - State#state.pid_map, - State#state.manifest_sqn}, {R, HeadTimer} = timed_fetch_mem(Key, Hash, - Structure, + State#state.manifest, State#state.levelzero_cache, State#state.levelzero_index, State#state.head_timing), {reply, R, State#state{head_timing=HeadTimer}}; handle_call({check_sqn, Key, Hash, SQN}, _From, State) -> - Structure = {State#state.manifest, - State#state.pid_map, - State#state.manifest_sqn}, {reply, compare_to_sqn(plain_fetch_mem(Key, Hash, - Structure, + State#state.manifest, State#state.levelzero_cache, State#state.levelzero_index), SQN), @@ -412,19 +409,15 @@ handle_call({fetch_keys, StartKey, EndKey, AccFun, InitAcc, MaxKeys}, List end, - ConvertToPointerFun = - fun(FN) -> {next, dict:fetch(FN, State#state.pid_map), StartKey} end, SetupFoldFun = fun(Level, Acc) -> - FNs = leveled_manifest:range_lookup(State#state.manifest, - Level, - StartKey, - EndKey, - State#state.manifest_sqn), - Pointers = lists:map(ConvertToPointerFun, FNs), + Pointers = leveled_manifest:range_lookup(State#state.manifest, + Level, + StartKey, + EndKey), case Pointers of [] -> Acc; - PL -> Acc ++ [{L, PL}] + PL -> Acc ++ [{Level, PL}] end end, SSTiter = lists:foldl(SetupFoldFun, [], lists:seq(0, ?MAX_LEVELS - 1)), @@ -435,29 +428,37 @@ handle_call({fetch_keys, StartKey, EndKey, AccFun, InitAcc, MaxKeys}, MaxKeys), {reply, Acc, State#state{levelzero_astree = L0AsList}}; -handle_call(work_for_clerk, From, State) -> - DelayForPendingL0 = State#state.levelzero_pending, - {WL, WC} = check_for_work(State#state.level_counts), - case WC of - 0 -> - {reply, none, State#state{work_backlog=false}}; - N when N > ?WORKQUEUE_BACKLOG_TOLERANCE -> - leveled_log:log("P0024", [N, true]), - [TL|_Tail] = WL, - {reply, TL, State#state{work_backlog=true}}; - N -> - leveled_log:log("P0024", [N, false]), - [TL|_Tail] = WL, - {reply, TL, State#state{work_backlog=false}} +handle_call(work_for_clerk, _From, State) -> + case State#state.levelzero_pending of + true -> + {reply, none, State}; + false -> + {WL, WC} = leveled_manifest:check_for_work(State#state.manifest, + ?LEVEL_SCALEFACTOR), + case WC of + 0 -> + {reply, none, State#state{work_backlog=false}}; + N when N > ?WORKQUEUE_BACKLOG_TOLERANCE -> + leveled_log:log("P0024", [N, true]), + [TL|_Tail] = WL, + {reply, + {TL, State#state.manifest}, + State#state{work_backlog=true}}; + N -> + leveled_log:log("P0024", [N, false]), + [TL|_Tail] = WL, + {reply, + {TL, State#state.manifest}, + State#state{work_backlog=false}} + end end; handle_call(get_startup_sqn, _From, State) -> {reply, State#state.persisted_sqn, State}; handle_call({register_snapshot, Snapshot}, _From, State) -> - RegisteredSnaps = add_snapshot(State#state.registered_snapshots, - Snapshot, - State#state.manifest_sqn, - ?SNAPSHOT_TIMEOUT), - {reply, {ok, State}, State#state{registered_snapshots = RegisteredSnaps}}; + Manifest0 = leveled_manifest:add_snapshot(State#state.manifest, + Snapshot, + ?SNAPSHOT_TIMEOUT), + {reply, {ok, State}, State#state{manifest = Manifest0}}; handle_call({load_snapshot, {BookieIncrTree, BookieIdx, MinSQN, MaxSQN}}, _From, State) -> L0D = leveled_pmem:add_to_cache(State#state.levelzero_size, @@ -483,37 +484,22 @@ handle_call(doom, _From, State) -> FilesFP = State#state.root_path ++ "/" ++ ?FILES_FP ++ "/", {stop, normal, {ok, [ManifestFP, FilesFP]}, State}. -handle_cast({manifest_change, WI}, State) -> - NewManifestSQN = WI#next_sqn, - UnreferenceFun = - fun(FN, Acc) -> - dict:store(FN, NewManifestSQN, Acc) - end, - DelPending = lists:foldl(UnreferenceFun, - State#state.deletions_pending, - WI#unreferenced_files), - {noreply, State{deletions_pending = DelPending, - manifest_sqn = NewManifestSQN}}; +handle_cast({manifest_change, NewManifest}, State) -> + {noreply, State#state{manifest = NewManifest}}; handle_cast({release_snapshot, Snapshot}, State) -> - Rs = leveled_manifest:release_snapshot(State#state.registered_snapshots, - Snapshot), + Manifest0 = leveled_manifest:release_snapshot(State#state.manifest, + Snapshot), leveled_log:log("P0003", [Snapshot]), - leveled_log:log("P0004", [Rs]), - {noreply, State#state{registered_snapshots=Rs}}; + {noreply, State#state{manifest=Manifest0}}; handle_cast({confirm_delete, Filename}, State=#state{is_snapshot=Snap}) when Snap == false -> - DeleteSQN = dict:fetch(Filename, State#state.deletions_pending), - R2D = leveled_manifest:ready_to_delete(State#state.registered_snapshots, - DeleteSQN), + R2D = leveled_manifest:ready_to_delete(State#state.manifest, Filename), case R2D of - true -> - PidToDelete = dict:fetch(Filename, State#state.pidmap), - leveled_log:log("P0005", [FileName]), - DP0 = dict:erase(Filename, State#state.deletions_pending), - PM0 = dict:erase(Filename, State#state.pidmap), + {true, Pid} -> + leveled_log:log("P0005", [Filename]), ok = leveled_sst:sst_deleteconfirmed(Pid), - {noreply, State#state{deletions_pending = DP0, pidmap = PM0}}; - false -> + {noreply, State}; + {false, _Pid} -> {noreply, State} end; handle_cast({levelzero_complete, FN, StartKey, EndKey}, State) -> @@ -522,17 +508,19 @@ handle_cast({levelzero_complete, FN, StartKey, EndKey}, State) -> end_key=EndKey, owner=State#state.levelzero_constructor, filename=FN}, - UpdMan = lists:keystore(0, 1, State#state.manifest, {0, [ManEntry]}), + ManifestSQN = leveled_manifest:get_manifest_sqn(State#state.manifest) + 1, + UpdMan = leveled_manifest:insert_manifest_entry(State#state.manifest, + ManifestSQN, + 0, + ManEntry), % Prompt clerk to ask about work - do this for every L0 roll UpdIndex = leveled_pmem:clear_index(State#state.levelzero_index), ok = leveled_pclerk:clerk_prompt(State#state.clerk), - UpdLevelCounts = dict:store(0, 1, State#state.level_counts), {noreply, State#state{levelzero_cache=[], levelzero_index=UpdIndex, levelzero_pending=false, levelzero_constructor=undefined, levelzero_size=0, - level_counts=UpdLevelCounts, manifest=UpdMan, persisted_sqn=State#state.ledger_sqn}}. @@ -557,22 +545,21 @@ terminate(Reason, State) -> ok = leveled_pclerk:clerk_close(State#state.clerk), leveled_log:log("P0008", [Reason]), - L0 = key_lookup(State#state.manifest, 0, all, State#state.manifest_sqn), - case {UpdState#state.levelzero_pending, L0} of + L0 = leveled_manifest:key_lookup(State#state.manifest, 0, all), + case {State#state.levelzero_pending, L0} of {false, false} -> - L0Pid = roll_memory(UpdState, true), + L0Pid = roll_memory(State, true), ok = leveled_sst:sst_close(L0Pid); StatusTuple -> leveled_log:log("P0010", [StatusTuple]) end, % Tidy shutdown of individual files - lists:foreach(fun({_FN, Pid}) -> + lists:foreach(fun({_FN, {Pid, _DSQN}}) -> ok = leveled_sst:sst_close(Pid) end, - dict:to_list(State#state.pidmap)), + leveled_manifest:dump_pidmap(State#state.manifest)), leveled_log:log("P0011", []), - ok. @@ -594,7 +581,7 @@ start_from_file(PCLopts) -> M end, - {ok, MergeClerk} = leveled_pclerk:clerk_new(self()), + {ok, MergeClerk} = leveled_pclerk:clerk_new(self(), RootPath), CoinToss = PCLopts#penciller_options.levelzero_cointoss, % Used to randomly defer the writing of L0 file. Intended to help with @@ -608,19 +595,19 @@ start_from_file(PCLopts) -> levelzero_index=leveled_pmem:new_index()}, %% Open manifest - Manifest = leveled_manifest:open_manifest(RootPath), - {FNList, - ManSQN, - LevelCounts) = leveled_manifest:initiate_from_manifest(Manifest), - InitiateFun = - fun(FN, {AccMaxSQN, AccPidMap}) -> - {ok, P, {_FK, _LK}} = leveled_sst:sst_open(FN), - FileMaxSQN = leveled_sst:sst_getmaxsequencenumber(P), - {max(AccMaxSQN, FileMaxSQN), dict:store(FN, P, AccPidMap)} + Manifest0 = leveled_manifest:open_manifest(RootPath), + OpenFun = + fun(FN) -> + {ok, Pid, {_FK, _LK}} = leveled_sst:sst_open(FN), + Pid end, - {MaxSQN, PidMap} = lists:foldl(InitiateFun, {0, dict:new()}, FNList), + SQNFun = fun leveled_sst:sst_getmaxsequencenumber/1, + {MaxSQN, Manifest1} = leveled_manifest:load_manifest(Manifest0, + OpenFun, + SQNFun), leveled_log:log("P0014", [MaxSQN]), - + ManSQN = leveled_manifest:get_manifest_sqn(Manifest1), + %% Find any L0 files L0FN = filepath(RootPath, ManSQN, new_merge_files) ++ "_0_0.sst", case filelib:is_file(L0FN) of @@ -632,41 +619,26 @@ start_from_file(PCLopts) -> L0SQN = leveled_sst:sst_getmaxsequencenumber(L0Pid), L0Entry = #manifest_entry{start_key = L0StartKey, end_key = L0EndKey, - filename = L0FN}, - PidMap0 = dict:store(L0FN, L0Pid, PidMap), - insert_manifest_entry(Manifest, ManSQN, 0, L0Entry) + filename = L0FN, + owner = L0Pid}, + Manifest2 = leveled_manifest:insert_manifest_entry(Manifest1, + ManSQN + 1, + 0, + L0Entry), leveled_log:log("P0016", [L0SQN]), LedgerSQN = max(MaxSQN, L0SQN), {ok, - InitState#state{manifest = Manifest, - manifest_sqn = ManSQN, + InitState#state{manifest = Manifest2, ledger_sqn = LedgerSQN, - persisted_sqn = LedgerSQN, - level_counts = LevelCounts, - pid_map = PidMap0}}; + persisted_sqn = LedgerSQN}}; false -> leveled_log:log("P0017", []), {ok, - InitState#state{manifest = Manifest, - manifest_sqn = ManSQN, + InitState#state{manifest = Manifest1, ledger_sqn = MaxSQN, - persisted_sqn = MaxSQN, - level_counts = LevelCounts, - pid_map = PidMap}} + persisted_sqn = MaxSQN}} end. -check_for_work(LevelCounts) -> - CheckLevelFun = - fun({Level, MaxCount}, {AccL, AccC}) -> - case dict:fetch(Level, LevelCounts) of - LC when LC > MaxCount -> - {[Level|AccL], AccC + LC - MaxCount}; - _ -> - {AccL, AccC} - end - end, - lists:foldl(CheckLevelFun, {[], 0}, ?LEVEL_SCALEFACTOR). - update_levelzero(L0Size, {PushedTree, PushedIdx, MinSQN, MaxSQN}, LedgerSQN, L0Cache, State) -> @@ -688,7 +660,7 @@ update_levelzero(L0Size, {PushedTree, PushedIdx, MinSQN, MaxSQN}, ledger_sqn=UpdMaxSQN}, CacheTooBig = NewL0Size > State#state.levelzero_maxcachesize, CacheMuchTooBig = NewL0Size > ?SUPER_MAX_TABLE_SIZE, - Level0Free = length(get_item(0, State#state.manifest, [])) == 0, + L0Free = not leveled_manifest:levelzero_present(State#state.manifest), RandomFactor = case State#state.levelzero_cointoss of true -> @@ -702,7 +674,7 @@ update_levelzero(L0Size, {PushedTree, PushedIdx, MinSQN, MaxSQN}, true end, JitterCheck = RandomFactor or CacheMuchTooBig, - case {CacheTooBig, Level0Free, JitterCheck} of + case {CacheTooBig, L0Free, JitterCheck} of {true, true, true} -> L0Constructor = roll_memory(UpdState, false), leveled_log:log_timer("P0031", [], SW), @@ -747,15 +719,15 @@ roll_memory(State, true) -> Constructor. levelzero_filename(State) -> - MSN = State#state.manifest_sqn, + ManSQN = leveled_manifest:get_manifest_sqn(State#state.manifest), FileName = State#state.root_path ++ "/" ++ ?FILES_FP ++ "/" - ++ integer_to_list(MSN) ++ "_0_0", + ++ integer_to_list(ManSQN) ++ "_0_0", FileName. -timed_fetch_mem(Key, Hash, Structure, L0Cache, L0Index, HeadTimer) -> +timed_fetch_mem(Key, Hash, Manifest, L0Cache, L0Index, HeadTimer) -> SW = os:timestamp(), - {R, Level} = fetch_mem(Key, Hash, Structure, L0Cache, L0Index), + {R, Level} = fetch_mem(Key, Hash, Manifest, L0Cache, L0Index), UpdHeadTimer = case R of not_present -> @@ -765,32 +737,30 @@ timed_fetch_mem(Key, Hash, Structure, L0Cache, L0Index, HeadTimer) -> end, {R, UpdHeadTimer}. -plain_fetch_mem(Key, Hash, Structure, L0Cache, L0Index) -> - R = fetch_mem(Key, Hash, Structure, L0Cache, L0Index), +plain_fetch_mem(Key, Hash, Manifest, L0Cache, L0Index) -> + R = fetch_mem(Key, Hash, Manifest, L0Cache, L0Index), element(1, R). -fetch_mem(Key, Hash, Structure, L0Cache, L0Index) -> +fetch_mem(Key, Hash, Manifest, L0Cache, L0Index) -> PosList = leveled_pmem:check_index(Hash, L0Index), L0Check = leveled_pmem:check_levelzero(Key, Hash, PosList, L0Cache), case L0Check of {false, not_found} -> - fetch(Key, Hash, Structure, 0, fun timed_sst_get/3); + fetch(Key, Hash, Manifest, 0, fun timed_sst_get/3); {true, KV} -> {KV, 0} end. -fetch(_Key, _Hash, _Structure, ?MAX_LEVELS + 1, _FetchFun) -> +fetch(_Key, _Hash, _Manifest, ?MAX_LEVELS + 1, _FetchFun) -> {not_present, basement}; -fetch(Key, Hash, Structure, Level, FetchFun) -> - {Manifest, PidMap, ManSQN} = Structure, - case leveled_manifest:key_lookup(Manifest, Level, Key, ManSQN) of +fetch(Key, Hash, Manifest, Level, FetchFun) -> + case leveled_manifest:key_lookup(Manifest, Level, Key) of false -> - fetch(Key, Hash, Structure, Level + 1, FetchFun); - FN -> - FP = dict:fetch(FN, PidMap), + fetch(Key, Hash, Manifest, Level + 1, FetchFun); + FP -> case FetchFun(FP, Key, Hash) of not_present -> - fetch(Key, Hash, Structure, Level + 1, FetchFun); + fetch(Key, Hash, Manifest, Level + 1, FetchFun); ObjectFound -> {ObjectFound, Level} end @@ -827,7 +797,6 @@ compare_to_sqn(Obj, SQN) -> end. - %% Looks to find the best choice for the next key across the levels (other %% than in-memory table) %% In finding the best choice, the next key in a given level may be a next @@ -1246,57 +1215,6 @@ simple_server_test() -> clean_testdir(RootPath). -rangequery_manifest_test() -> - {E1, - E2, - E3} = {#manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, - end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, - filename="Z1"}, - #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"}, - end_key={o, "Bucket1", "K71", null}, - filename="Z2"}, - #manifest_entry{start_key={o, "Bucket1", "K75", null}, - end_key={o, "Bucket1", "K993", null}, - filename="Z3"}}, - {E4, - E5, - E6} = {#manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, - end_key={i, "Bucket1", {"Idx1", "Fld7"}, "K93"}, - filename="Z4"}, - #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld7"}, "K97"}, - end_key={o, "Bucket1", "K78", null}, - filename="Z5"}, - #manifest_entry{start_key={o, "Bucket1", "K81", null}, - end_key={o, "Bucket1", "K996", null}, - filename="Z6"}}, - Man = [{1, [E1, E2, E3]}, {2, [E4, E5, E6]}], - SK1 = {o, "Bucket1", "K711", null}, - EK1 = {o, "Bucket1", "K999", null}, - R1 = initiate_rangequery_frommanifest(SK1, EK1, Man), - ?assertMatch([{1, [{next, E3, SK1}]}, - {2, [{next, E5, SK1}, {next, E6, SK1}]}], - R1), - SK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, - EK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, - R2 = initiate_rangequery_frommanifest(SK2, EK2, Man), - ?assertMatch([{1, [{next, E1, SK2}]}, {2, [{next, E5, SK2}]}], R2), - R3 = initiate_rangequery_frommanifest({i, "Bucket1", {"Idx0", "Fld8"}, null}, - {i, "Bucket1", {"Idx0", "Fld9"}, null}, - Man), - ?assertMatch([], R3). - -print_manifest_test() -> - M1 = #manifest_entry{start_key={i, "Bucket1", {<<"Idx1">>, "Fld1"}, "K8"}, - end_key={i, 4565, {"Idx1", "Fld9"}, "K93"}, - filename="Z1"}, - M2 = #manifest_entry{start_key={i, self(), {null, "Fld1"}, "K8"}, - end_key={i, <<200:32/integer>>, {"Idx1", "Fld9"}, "K93"}, - filename="Z1"}, - M3 = #manifest_entry{start_key={?STD_TAG, self(), {null, "Fld1"}, "K8"}, - end_key={?RIAK_TAG, <<200:32/integer>>, {"Idx1", "Fld9"}, "K93"}, - filename="Z1"}, - print_manifest([{1, [M1, M2, M3]}]). - simple_findnextkey_test() -> QueryArray = [ {2, [{{o, "Bucket1", "Key1"}, {5, {active, infinity}, null}}, @@ -1463,81 +1381,6 @@ create_file_test() -> {ok, Bin} = file:read_file("../test/new_file.sst.discarded"), ?assertMatch("hello", binary_to_term(Bin)). -commit_manifest_test() -> - Sent_WI = #penciller_work{next_sqn=1, - src_level=0, - start_time=os:timestamp()}, - Resp_WI = #penciller_work{next_sqn=1, - src_level=0}, - State = #state{ongoing_work = [Sent_WI], - root_path = "test", - manifest_sqn = 0}, - ManifestFP = "test" ++ "/" ++ ?MANIFEST_FP ++ "/", - ok = filelib:ensure_dir(ManifestFP), - ok = file:write_file(ManifestFP ++ "nonzero_1.pnd", - term_to_binary("dummy data")), - - L1_0 = [{1, [#manifest_entry{filename="1.sst"}]}], - Resp_WI0 = Resp_WI#penciller_work{new_manifest=L1_0, - unreferenced_files=[]}, - {ok, State0} = commit_manifest_change(Resp_WI0, State), - ?assertMatch(1, State0#state.manifest_sqn), - ?assertMatch([], get_item(0, State0#state.manifest, [])), - - L0Entry = [#manifest_entry{filename="0.sst"}], - ManifestPlus = [{0, L0Entry}|State0#state.manifest], - - NxtSent_WI = #penciller_work{next_sqn=2, - src_level=1, - start_time=os:timestamp()}, - NxtResp_WI = #penciller_work{next_sqn=2, - src_level=1}, - State1 = State0#state{ongoing_work=[NxtSent_WI], - manifest = ManifestPlus}, - - ok = file:write_file(ManifestFP ++ "nonzero_2.pnd", - term_to_binary("dummy data")), - - L2_0 = [#manifest_entry{filename="2.sst"}], - NxtResp_WI0 = NxtResp_WI#penciller_work{new_manifest=[{2, L2_0}], - unreferenced_files=[]}, - {ok, State2} = commit_manifest_change(NxtResp_WI0, State1), - - ?assertMatch(1, State1#state.manifest_sqn), - ?assertMatch(2, State2#state.manifest_sqn), - ?assertMatch(L0Entry, get_item(0, State2#state.manifest, [])), - ?assertMatch(L2_0, get_item(2, State2#state.manifest, [])), - - clean_testdir(State#state.root_path). - - -badmanifest_test() -> - RootPath = "../test/ledger", - clean_testdir(RootPath), - {ok, PCL} = pcl_start(#penciller_options{root_path=RootPath, - max_inmemory_tablesize=1000}), - Key1_pre = {{o,"Bucket0001", "Key0001", null}, - {1001, {active, infinity}, null}}, - Key1 = add_missing_hash(Key1_pre), - KL1 = generate_randomkeys({1000, 1}), - - ok = maybe_pause_push(PCL, KL1 ++ [Key1]), - %% Added together, as split apart there will be a race between the close - %% call to the penciller and the second fetch of the cache entry - ?assertMatch(Key1, pcl_fetch(PCL, {o, "Bucket0001", "Key0001", null})), - - timer:sleep(100), % Avoids confusion if L0 file not written before close - ok = pcl_close(PCL), - - ManifestFP = filepath(RootPath, manifest), - ok = file:write_file(filename:join(ManifestFP, "yeszero_123.man"), - term_to_binary("hello")), - {ok, PCLr} = pcl_start(#penciller_options{root_path=RootPath, - max_inmemory_tablesize=1000}), - ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001", null})), - ok = pcl_close(PCLr), - clean_testdir(RootPath). - checkready(Pid) -> try leveled_sst:sst_checkready(Pid) From 76bdd833461d8d2d68c8601f706e442e896d09d2 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 14 Jan 2017 16:36:05 +0000 Subject: [PATCH 05/37] Manifest refactor - STILL BROKEN Some working tests now, but sitll broken --- src/leveled_codec.erl | 2 + src/leveled_manifest.erl | 243 +++++++++++++++++++++++++++----------- src/leveled_pclerk.erl | 4 +- src/leveled_penciller.erl | 21 ++-- 4 files changed, 186 insertions(+), 84 deletions(-) diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index 10ffcc7..6dbbff4 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -282,6 +282,8 @@ turn_to_string(Item) -> % Compare a key against a query key, only comparing elements that are non-null % in the Query key. This is used for comparing against end keys in queries. +endkey_passed(all, _) -> + false; endkey_passed({EK1, null, null, null}, {CK1, _, _, _}) -> EK1 < CK1; endkey_passed({EK1, EK2, null, null}, {CK1, CK2, _, _}) -> diff --git a/src/leveled_manifest.erl b/src/leveled_manifest.erl index f7efed9..8da2512 100644 --- a/src/leveled_manifest.erl +++ b/src/leveled_manifest.erl @@ -52,12 +52,15 @@ pointer_convert/2 ]). +-export([ + filepath/2 + ]). + -include_lib("eunit/include/eunit.hrl"). -define(MANIFEST_FILEX, "man"). -define(MANIFEST_FP, "ledger_manifest"). -define(MAX_LEVELS, 8). --define(END_KEY, {null, null, null, null}). -record(manifest, {table, % A Multi-Version ETS table for lookup @@ -71,7 +74,7 @@ % An array of level counts to speed up compation work assessment snapshots :: list(), % A list of snaphots (i.e. clones) - delete_sqn :: integer()|infinity + delete_sqn :: integer() % The lowest SQN of any clone }). @@ -80,7 +83,7 @@ %%%============================================================================ new_manifest() -> - Table = ets:new(manifest, [ordered_set]), + Table = ets:new(manifest, [ordered_set, public]), new_manifest(Table). open_manifest(RootPath) -> @@ -101,9 +104,8 @@ open_manifest(RootPath) -> ValidManSQNs = lists:reverse(lists:sort(lists:foldl(ExtractSQNFun, [], Filenames))), - {ManSQN, Table} = open_manifestfile(RootPath, ValidManSQNs), - Manifest = new_manifest(Table), - Manifest#manifest{manifest_sqn = ManSQN}. + {ManSQN, Manifest} = open_manifestfile(RootPath, ValidManSQNs), + Manifest#manifest{manifest_sqn = ManSQN, delete_sqn = ManSQN}. copy_manifest(Manifest) -> % Copy the manifest ensuring anything only the master process should care @@ -162,7 +164,7 @@ insert_manifest_entry(Manifest, ManSQN, Level, Entry) -> remove_manifest_entry(Manifest, ManSQN, Level, Entry) -> Key = {Level, Entry#manifest_entry.end_key, Entry#manifest_entry.filename}, - [{Key, Value0}] = ets:lookup(Manifest, Key), + [{Key, Value0}] = ets:lookup(Manifest#manifest.table, Key), {StartKey, {active, ActiveSQN}, {tomb, infinity}} = Value0, Value1 = {StartKey, {active, ActiveSQN}, {tomb, ManSQN}}, true = ets:insert(Manifest#manifest.table, {Key, Value1}), @@ -205,7 +207,13 @@ key_lookup(Manifest, Level, Key) -> range_lookup(Manifest, Level, StartKey, EndKey) -> MapFun = fun({{_Level, _LastKey, FN}, FirstKey}) -> - {next, dict:fetch(FN, Manifest#manifest.pidmap), FirstKey} + {Pid, _SQN} = dict:fetch(FN, Manifest#manifest.pidmap), + case FirstKey < StartKey of + true -> + {next, Pid, StartKey}; + false -> + {next, Pid, FirstKey} + end end, range_lookup(Manifest, Level, StartKey, EndKey, MapFun). @@ -245,7 +253,7 @@ mergefile_selector(Manifest, Level) -> Level, {all, 0}, all, - ?END_KEY, + all, [], Manifest#manifest.manifest_sqn), {{Level, LastKey, FN}, @@ -267,14 +275,15 @@ release_snapshot(Manifest, Pid) -> fun({P, SQN, TS}, {Acc, MinSQN}) -> case P of Pid -> - Acc; + {Acc, MinSQN}; _ -> {[{P, SQN, TS}|Acc], min(SQN, MinSQN)} end end, - {SnapList0, DeleteSQN} = lists:foldl(FilterFun, - {[], infinity}, - Manifest#manifest.snapshots), + {SnapList0, + DeleteSQN} = lists:foldl(FilterFun, + {[], Manifest#manifest.manifest_sqn}, + Manifest#manifest.snapshots), leveled_log:log("P0004", [SnapList0]), Manifest#manifest{snapshots = SnapList0, delete_sqn = DeleteSQN}. @@ -292,7 +301,7 @@ ready_to_delete(Manifest, Filename) -> check_for_work(Manifest, Thresholds) -> CheckLevelFun = fun({Level, MaxCount}, {AccL, AccC}) -> - case dict:fetch(Level, Manifest#manifest.level_counts) of + case array:get(Level, Manifest#manifest.level_counts) of LC when LC > MaxCount -> {[Level|AccL], AccC + LC - MaxCount}; _ -> @@ -335,7 +344,7 @@ new_manifest(Table) -> pidmap = dict:new(), level_counts = array:new([{size, ?MAX_LEVELS + 1}, {default, 0}]), snapshots = [], - delete_sqn = infinity + delete_sqn = 0 }. range_lookup(Manifest, Level, StartKey, EndKey, MapFun) -> @@ -430,7 +439,7 @@ open_manifestfile(RootPath, [TopManSQN|Rest]) -> open_manifestfile(RootPath, Rest); {ok, Table} -> leveled_log:log("P0012", [TopManSQN]), - {TopManSQN, Table} + {TopManSQN, new_manifest(Table)} end. key_lookup(Manifest, Level, KeyToFind, ManSQN, GC) -> @@ -483,88 +492,178 @@ key_lookup(Manifest, Level, {LastKey, LastFN}, KeyToFind, ManSQN, GC) -> -ifdef(TEST). -rangequery_manifest_test() -> +initial_setup() -> E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, - filename="Z1"}, + filename="Z1", + owner="pid_z1"}, E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"}, end_key={o, "Bucket1", "K71", null}, - filename="Z2"}, + filename="Z2", + owner="pid_z2"}, E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null}, end_key={o, "Bucket1", "K993", null}, - filename="Z3"}, + filename="Z3", + owner="pid_z3"}, E4 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, end_key={i, "Bucket1", {"Idx1", "Fld7"}, "K93"}, - filename="Z4"}, + filename="Z4", + owner="pid_z4"}, E5 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld7"}, "K97"}, end_key={o, "Bucket1", "K78", null}, - filename="Z5"}, + filename="Z5", + owner="pid_z5"}, E6 = #manifest_entry{start_key={o, "Bucket1", "K81", null}, end_key={o, "Bucket1", "K996", null}, - filename="Z6"}, - - Manifest = new_manifest(), - insert_manifest_entry(Manifest, 1, 1, E1), - insert_manifest_entry(Manifest, 1, 1, E2), - insert_manifest_entry(Manifest, 1, 1, E3), - insert_manifest_entry(Manifest, 1, 2, E4), - insert_manifest_entry(Manifest, 1, 2, E5), - insert_manifest_entry(Manifest, 1, 2, E6), - - SK1 = {o, "Bucket1", "K711", null}, - EK1 = {o, "Bucket1", "K999", null}, - RL1_1 = range_lookup(Manifest, 1, SK1, EK1), - ?assertMatch(["Z3"], RL1_1), - RL1_2 = range_lookup(Manifest, 2, SK1, EK1), - ?assertMatch(["Z5", "Z6"], RL1_2), - SK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, - EK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, - RL2_1 = range_lookup(Manifest, 1, SK2, EK2), - ?assertMatch(["Z1"], RL2_1), - RL2_2 = range_lookup(Manifest, 2, SK2, EK2), - ?assertMatch(["Z5"], RL2_2), - - SK3 = {o, "Bucket1", "K994", null}, - EK3 = {o, "Bucket1", "K995", null}, - RL3_1 = range_lookup(Manifest, 1, SK3, EK3), - ?assertMatch([], RL3_1), - RL3_2 = range_lookup(Manifest, 2, SK3, EK3), - ?assertMatch(["Z6"], RL3_2), + filename="Z6", + owner="pid_z6"}, + Man0 = new_manifest(), + % insert_manifest_entry(Manifest, ManSQN, Level, Entry) + Man1 = insert_manifest_entry(Man0, 1, 1, E1), + Man2 = insert_manifest_entry(Man1, 1, 1, E2), + Man3 = insert_manifest_entry(Man2, 1, 1, E3), + Man4 = insert_manifest_entry(Man3, 1, 2, E4), + Man5 = insert_manifest_entry(Man4, 1, 2, E5), + Man6 = insert_manifest_entry(Man5, 1, 2, E6), + {Man0, Man1, Man2, Man3, Man4, Man5, Man6}. + +changeup_setup(Man6) -> + E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, + end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, + filename="Z1", + owner="pid_z1"}, + E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"}, + end_key={o, "Bucket1", "K71", null}, + filename="Z2", + owner="pid_z2"}, + E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null}, + end_key={o, "Bucket1", "K993", null}, + filename="Z3", + owner="pid_z3"}, + E1_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld4"}, "K8"}, end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K62"}, + owner="pid_y1", filename="Y1"}, E2_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K67"}, - end_key={o, "Bucket1", "K45", null}, - filename="Y2"}, + end_key={o, "Bucket1", "K45", null}, + owner="pid_y2", + filename="Y2"}, E3_2 = #manifest_entry{start_key={o, "Bucket1", "K47", null}, end_key={o, "Bucket1", "K812", null}, + owner="pid_y3", filename="Y3"}, E4_2 = #manifest_entry{start_key={o, "Bucket1", "K815", null}, end_key={o, "Bucket1", "K998", null}, + owner="pid_y4", filename="Y4"}, - insert_manifest_entry(Manifest, 2, 1, E1_2), - insert_manifest_entry(Manifest, 2, 1, E2_2), - insert_manifest_entry(Manifest, 2, 1, E3_2), - insert_manifest_entry(Manifest, 2, 1, E4_2), - remove_manifest_entry(Manifest, 2, 1, E1), - remove_manifest_entry(Manifest, 2, 1, E2), - remove_manifest_entry(Manifest, 2, 1, E3), + Man7 = insert_manifest_entry(Man6, 2, 1, E1_2), + Man8 = insert_manifest_entry(Man7, 2, 1, E2_2), + Man9 = insert_manifest_entry(Man8, 2, 1, E3_2), + Man10 = insert_manifest_entry(Man9, 2, 1, E4_2), + % remove_manifest_entry(Manifest, ManSQN, Level, Entry) + Man11 = remove_manifest_entry(Man10, 2, 1, E1), + Man12 = remove_manifest_entry(Man11, 2, 1, E2), + Man13 = remove_manifest_entry(Man12, 2, 1, E3), + {Man7, Man8, Man9, Man10, Man11, Man12, Man13}. + +keylookup_manifest_test() -> + {Man0, Man1, Man2, Man3, _Man4, _Man5, Man6} = initial_setup(), + LK1_1 = {o, "Bucket1", "K711", null}, + LK1_2 = {o, "Bucket1", "K70", null}, + LK1_3 = {o, "Bucket1", "K71", null}, + LK1_4 = {o, "Bucket1", "K75", null}, + LK1_5 = {o, "Bucket1", "K76", null}, - RL1_1A = range_lookup(Manifest, 1, SK1, EK1), - ?assertMatch(["Z3"], RL1_1A), - RL2_1A = range_lookup(Manifest, 1, SK2, EK2), - ?assertMatch(["Z1"], RL2_1A), - RL3_1A = range_lookup(Manifest, 1, SK3, EK3), + ?assertMatch(false, key_lookup(Man0, 1, LK1_1)), + ?assertMatch(false, key_lookup(Man1, 1, LK1_1)), + ?assertMatch(false, key_lookup(Man2, 1, LK1_1)), + ?assertMatch(false, key_lookup(Man3, 1, LK1_1)), + ?assertMatch(false, key_lookup(Man6, 1, LK1_1)), + + ?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_2)), + ?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_3)), + ?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_4)), + ?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_5)), + + ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_2)), + ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_3)), + ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_4)), + ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_5)), + + {_Man7, _Man8, _Man9, _Man10, _Man11, _Man12, + Man13} = changeup_setup(Man6), + + ?assertMatch(false, key_lookup(Man0, 1, LK1_1)), + ?assertMatch(false, key_lookup(Man1, 1, LK1_1)), + ?assertMatch(false, key_lookup(Man2, 1, LK1_1)), + ?assertMatch(false, key_lookup(Man3, 1, LK1_1)), + ?assertMatch(false, key_lookup(Man6, 1, LK1_1)), + + ?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_2)), + ?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_3)), + io:format("Commencing failing test:~n"), + ?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_4)), + ?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_5)), + + ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_2)), + ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_3)), + ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_4)), + ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_5)), + + ?assertMatch("pid_y3", key_lookup(Man13, 1, LK1_4)), + ?assertMatch("pid_z5", key_lookup(Man13, 2, LK1_4)). + + +rangequery_manifest_test() -> + {_Man0, _Man1, _Man2, _Man3, _Man4, _Man5, Man6} = initial_setup(), + + SK1 = {o, "Bucket1", "K711", null}, + EK1 = {o, "Bucket1", "K999", null}, + RL1_1 = range_lookup(Man6, 1, SK1, EK1), + ?assertMatch([{next, "pid_z3", {o, "Bucket1", "K75", null}}], RL1_1), + RL1_2 = range_lookup(Man6, 2, SK1, EK1), + ?assertMatch([{next, "pid_z5", {o, "Bucket1", "K711", null}}, + {next, "pid_z6", {o, "Bucket1", "K81", null}}], + RL1_2), + SK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, + EK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, + RL2_1 = range_lookup(Man6, 1, SK2, EK2), + ?assertMatch([{next, "pid_z1", {i, "Bucket1", {"Idx1", "Fld8"}, null}}], + RL2_1), + RL2_2 = range_lookup(Man6, 2, SK2, EK2), + ?assertMatch([{next, "pid_z5", {i, "Bucket1", {"Idx1", "Fld8"}, null}}], + RL2_2), + + SK3 = {o, "Bucket1", "K994", null}, + EK3 = {o, "Bucket1", "K995", null}, + RL3_1 = range_lookup(Man6, 1, SK3, EK3), + ?assertMatch([], RL3_1), + RL3_2 = range_lookup(Man6, 2, SK3, EK3), + ?assertMatch([{next, "pid_z6", {o, "Bucket1", "K994", null}}], RL3_2), + + {_Man7, _Man8, _Man9, _Man10, _Man11, _Man12, + Man13} = changeup_setup(Man6), + + % Results unchanged despiter ES table change if using old manifest + RL1_1A = range_lookup(Man6, 1, SK1, EK1), + ?assertMatch([{next, "pid_z3", {o, "Bucket1", "K75", null}}], RL1_1A), + RL2_1A = range_lookup(Man6, 1, SK2, EK2), + ?assertMatch([{next, "pid_z1", {i, "Bucket1", {"Idx1", "Fld8"}, null}}], + RL2_1A), + RL3_1A = range_lookup(Man6, 1, SK3, EK3), ?assertMatch([], RL3_1A), - - RL1_1B = range_lookup(Manifest, 1, SK1, EK1), - ?assertMatch(["Y3", "Y4"], RL1_1B), - RL2_1B = range_lookup(Manifest, 1, SK2, EK2), - ?assertMatch(["Y1"], RL2_1B), - RL3_1B = range_lookup(Manifest, 1, SK3, EK3), - ?assertMatch(["Y4"], RL3_1B). + + RL1_1B = range_lookup(Man13, 1, SK1, EK1), + ?assertMatch([{next, "pid_y3", {o, "Bucket1", "K711", null}}, + {next, "pid_y4", {o, "Bucket1", "K815", null}}], RL1_1B), + RL2_1B = range_lookup(Man13, 1, SK2, EK2), + ?assertMatch([{next, "pid_y1", {i, "Bucket1", {"Idx1", "Fld8"}, null}}], + RL2_1B), + RL3_1B = range_lookup(Man13, 1, SK3, EK3), + ?assertMatch([{next, "pid_y4", {o, "Bucket1", "K994", null}}], RL3_1B). -endif. \ No newline at end of file diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index a0ddbcd..0d45833 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -111,7 +111,7 @@ requestandhandle_work(State) -> Manifest, State#state.root_path), leveled_log:log("PC007", []), - ok = leveled_penciller:pcl_commitmanifestchange(State#state.owner, + ok = leveled_penciller:pcl_manifestchange(State#state.owner, UpdManifest), ok = leveled_manifest:save_manifest(UpdManifest, State#state.root_path), @@ -171,7 +171,7 @@ perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN) -> SinkPointerList = leveled_manifest:pointer_convert(Manifest, SinkList), MaxSQN = leveled_sst:sst_getmaxsequencenumber(Src#manifest_entry.owner), SinkLevel = SrcLevel + 1, - SinkBasement = leveled_basement:is_basement(Manifest, SinkLevel), + SinkBasement = leveled_manifest:is_basement(Manifest, SinkLevel), Man0 = do_merge(SrcList, SinkPointerList, SinkLevel, SinkBasement, RootPath, NewSQN, MaxSQN, diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index d4423cb..6969cf5 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -179,7 +179,7 @@ pcl_fetchnextkey/5, pcl_checksequencenumber/3, pcl_workforclerk/1, - pcl_confirmmanifestchange/2, + pcl_manifestchange/2, pcl_confirml0complete/4, pcl_confirmdelete/2, pcl_close/1, @@ -291,7 +291,7 @@ pcl_checksequencenumber(Pid, Key, SQN) -> pcl_workforclerk(Pid) -> gen_server:call(Pid, work_for_clerk, infinity). -pcl_confirmmanifestchange(Pid, Manifest) -> +pcl_manifestchange(Pid, Manifest) -> gen_server:cast(Pid, {manifest_change, Manifest}). pcl_confirml0complete(Pid, FN, StartKey, EndKey) -> @@ -548,7 +548,7 @@ terminate(Reason, State) -> L0 = leveled_manifest:key_lookup(State#state.manifest, 0, all), case {State#state.levelzero_pending, L0} of {false, false} -> - L0Pid = roll_memory(State, true), + L0Pid = roll_memory(State, true), ok = leveled_sst:sst_close(L0Pid); StatusTuple -> leveled_log:log("P0010", [StatusTuple]) @@ -842,10 +842,9 @@ find_nextkey(QueryArray, LCnt, {BestKeyLevel, BestKV}, LCnt + 1, {BKL, BKV}, StartKey, EndKey, Width); - {{next, ManifestEntry, _SK}, BKL, BKV} -> + {{next, Owner, _SK}, BKL, BKV} -> % The first key at this level is pointer to a file - need to query % the file to expand this level out before proceeding - Owner = ManifestEntry#manifest_entry.owner, Pointer = {next, Owner, StartKey, EndKey}, UpdList = leveled_sst:expand_list_by_pointer(Pointer, RestOfKeys, @@ -1036,7 +1035,7 @@ generate_randomkeys(Count, SQN, Acc) -> clean_testdir(RootPath) -> - clean_subdir(filepath(RootPath, manifest)), + clean_subdir(leveled_manifest:filepath(RootPath, manifest)), clean_subdir(filepath(RootPath, files)). clean_subdir(DirPath) -> @@ -1186,10 +1185,6 @@ simple_server_test() -> 1)), ok = pcl_close(PclSnap), - % Ignore a fake pending mnaifest on startup - ok = file:write_file(RootPath ++ "/" ++ ?MANIFEST_FP ++ "nonzero_99.pnd", - term_to_binary("Hello")), - {ok, PclSnap2} = pcl_start(SnapOpts), leveled_bookie:load_snapshot(PclSnap2, leveled_bookie:empty_ledgercache()), ?assertMatch(false, pcl_checksequencenumber(PclSnap2, @@ -1204,6 +1199,12 @@ simple_server_test() -> "Key0001", null}, 4005)), + + io:format("Snap2 B2 K2 ~w~n", + [pcl_fetch(PclSnap2, {o, "Bucket0002", "Key0002", null})]), + io:format("r B2 K2 ~w~n", + [pcl_fetch(PCLr, {o, "Bucket0002", "Key0002", null})]), + ?assertMatch(true, pcl_checksequencenumber(PclSnap2, {o, "Bucket0002", From 13c81f0ed134a2fb8ce4ba2fcc1b3d7ca230ce13 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 14 Jan 2017 19:41:09 +0000 Subject: [PATCH 06/37] Basic working Some basic tests working - but still outstanding issues. --- src/leveled_log.erl | 2 ++ src/leveled_manifest.erl | 34 ++++++++++++++++++++++++++-------- src/leveled_pclerk.erl | 13 +++++++------ src/leveled_penciller.erl | 28 +++++++++++++++------------- src/leveled_sst.erl | 1 + 5 files changed, 51 insertions(+), 27 deletions(-) diff --git a/src/leveled_log.erl b/src/leveled_log.erl index f5829be..091e454 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -251,6 +251,8 @@ {info, "Completed creation of ~s at level ~w with max sqn ~w"}}, {"SST09", {warn, "Read request exposes slot with bad CRC"}}, + {"SST10", + {info, "Expansion sought to support pointer to pid ~w status ~w"}}, {"CDB01", {info, "Opening file for writing with filename ~s"}}, diff --git a/src/leveled_manifest.erl b/src/leveled_manifest.erl index 8da2512..894d344 100644 --- a/src/leveled_manifest.erl +++ b/src/leveled_manifest.erl @@ -49,7 +49,8 @@ is_basement/2, dump_pidmap/1, levelzero_present/1, - pointer_convert/2 + pointer_convert/2, + delete_confirmed/2 ]). -export([ @@ -220,7 +221,7 @@ range_lookup(Manifest, Level, StartKey, EndKey) -> merge_lookup(Manifest, Level, StartKey, EndKey) -> MapFun = fun({{_Level, LastKey, FN}, FirstKey}) -> - Owner = dict:fetch(FN, Manifest#manifest.pidmap), + {Owner, _DelSQN} = dict:fetch(FN, Manifest#manifest.pidmap), #manifest_entry{filename = FN, owner = Owner, start_key = FirstKey, @@ -231,10 +232,9 @@ merge_lookup(Manifest, Level, StartKey, EndKey) -> pointer_convert(Manifest, EntryList) -> MapFun = fun(Entry) -> - {next, - dict:fetch(Entry#manifest_entry.filename, - Manifest#manifest.pidmap), - all} + {Pid, _DelSQN} = dict:fetch(Entry#manifest_entry.filename, + Manifest#manifest.pidmap), + {next, Pid, all} end, lists:map(MapFun, EntryList). @@ -333,6 +333,11 @@ levelzero_present(Manifest) -> true end. +delete_confirmed(Manifest, Filename) -> + PidMap = dict:erase(Filename, Manifest#manifest.pidmap), + % Would be better to clear ETS at this point rather than on lookup? + Manifest#manifest{pidmap = PidMap}. + %%%============================================================================ %%% Internal Functions %%%============================================================================ @@ -418,7 +423,9 @@ ready_to_delete(SnapList, FileDeleteSQN, Now) -> lists:foldl(FilterFun, true, SnapList). filepath(RootPath, manifest) -> - RootPath ++ "/" ++ ?MANIFEST_FP ++ "/". + MFP = RootPath ++ "/" ++ ?MANIFEST_FP ++ "/", + filelib:ensure_dir(MFP), + MFP. filepath(RootPath, NewMSN, current_manifest) -> filepath(RootPath, manifest) ++ "nonzero_" @@ -455,7 +462,7 @@ key_lookup(Manifest, Level, {LastKey, LastFN}, KeyToFind, ManSQN, GC) -> Active = (ManSQN >= ActiveSQN) and (ManSQN < TombSQN), case Active of true -> - InRange = KeyToFind >= FirstKey, + InRange = (KeyToFind >= FirstKey) or (KeyToFind == all), case InRange of true -> NextFN; @@ -665,5 +672,16 @@ rangequery_manifest_test() -> RL3_1B = range_lookup(Man13, 1, SK3, EK3), ?assertMatch([{next, "pid_y4", {o, "Bucket1", "K994", null}}], RL3_1B). +levelzero_present_test() -> + E0 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, + end_key={o, "Bucket1", "Key996", null}, + filename="Z0", + owner="pid_z0"}, + + Man0 = new_manifest(), + ?assertMatch(false, levelzero_present(Man0)), + % insert_manifest_entry(Manifest, ManSQN, Level, Entry) + Man1 = insert_manifest_entry(Man0, 1, 0, E0), + ?assertMatch(true, levelzero_present(Man1)). -endif. \ No newline at end of file diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 0d45833..8976761 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -184,7 +184,8 @@ perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN) -> Entry) end, Man1 = lists:foldl(RemoveFun, Man0, SinkList), - leveled_manifest:remove_manifest_entry(Man1, NewSQN, SrcLevel, Src). + Man2 = leveled_manifest:remove_manifest_entry(Man1, NewSQN, SrcLevel, Src), + {Man2, [Src|SinkList]}. do_merge([], [], SinkLevel, _SinkB, _RP, NewSQN, _MaxSQN, Counter, Man0) -> leveled_log:log("PC011", [NewSQN, SinkLevel, Counter]), @@ -273,12 +274,12 @@ merge_file_test() -> Man0 = leveled_manifest:new_manifest(), Man1 = leveled_manifest:insert_manifest_entry(Man0, 1, 2, E1), - Man2 = leveled_manifest:insert_manifest_entry(Man1, 1, 2, E1), - Man3 = leveled_manifest:insert_manifest_entry(Man2, 1, 2, E1), - Man4 = leveled_manifest:insert_manifest_entry(Man3, 1, 2, E1), - Man5 = leveled_manifest:insert_manifest_entry(Man4, 2, 1, E1), + Man2 = leveled_manifest:insert_manifest_entry(Man1, 1, 2, E2), + Man3 = leveled_manifest:insert_manifest_entry(Man2, 1, 2, E3), + Man4 = leveled_manifest:insert_manifest_entry(Man3, 1, 2, E4), + Man5 = leveled_manifest:insert_manifest_entry(Man4, 2, 1, E5), - Man6 = perform_merge(Man5, E1, [E2, E3, E4, E5], 1, "../test", 3), + {Man6, _Dels} = perform_merge(Man5, E1, [E2, E3, E4, E5], 1, "../test", 3), ?assertMatch(3, leveled_manifest:get_manifest_sqn(Man6)). diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 6969cf5..a0fe4cb 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -498,7 +498,9 @@ handle_cast({confirm_delete, Filename}, State=#state{is_snapshot=Snap}) {true, Pid} -> leveled_log:log("P0005", [Filename]), ok = leveled_sst:sst_deleteconfirmed(Pid), - {noreply, State}; + Man0 = leveled_manifest:delete_confirmed(State#state.manifest, + Filename), + {noreply, State#state{manifest=Man0}}; {false, _Pid} -> {noreply, State} end; @@ -545,9 +547,10 @@ terminate(Reason, State) -> ok = leveled_pclerk:clerk_close(State#state.clerk), leveled_log:log("P0008", [Reason]), - L0 = leveled_manifest:key_lookup(State#state.manifest, 0, all), - case {State#state.levelzero_pending, L0} of - {false, false} -> + L0_Present = leveled_manifest:key_lookup(State#state.manifest, 0, all), + L0_Left = State#state.levelzero_size > 0, + case {State#state.levelzero_pending, L0_Present, L0_Left} of + {false, false, true} -> L0Pid = roll_memory(State, true), ok = leveled_sst:sst_close(L0Pid); StatusTuple -> @@ -719,7 +722,7 @@ roll_memory(State, true) -> Constructor. levelzero_filename(State) -> - ManSQN = leveled_manifest:get_manifest_sqn(State#state.manifest), + ManSQN = leveled_manifest:get_manifest_sqn(State#state.manifest) + 1, FileName = State#state.root_path ++ "/" ++ ?FILES_FP ++ "/" ++ integer_to_list(ManSQN) ++ "_0_0", @@ -1000,7 +1003,9 @@ keyfolder({[{IMMKey, IMMVal}|NxIMMiterator], SSTiterator}, KeyRange, filepath(RootPath, files) -> - RootPath ++ "/" ++ ?FILES_FP. + FP = RootPath ++ "/" ++ ?FILES_FP, + filelib:ensure_dir(FP ++ "/"), + FP. filepath(RootPath, NewMSN, new_merge_files) -> filepath(RootPath, files) ++ "/" ++ integer_to_list(NewMSN). @@ -1135,11 +1140,13 @@ simple_server_test() -> ?assertMatch(Key2, pcl_fetch(PCLr, {o,"Bucket0002", "Key0002", null})), ?assertMatch(Key3, pcl_fetch(PCLr, {o,"Bucket0003", "Key0003", null})), ?assertMatch(Key4, pcl_fetch(PCLr, {o,"Bucket0004", "Key0004", null})), + SnapOpts = #penciller_options{start_snapshot = true, source_penciller = PCLr}, {ok, PclSnap} = pcl_start(SnapOpts), leveled_bookie:load_snapshot(PclSnap, leveled_bookie:empty_ledgercache()), + ?assertMatch(Key1, pcl_fetch(PclSnap, {o,"Bucket0001", "Key0001", null})), ?assertMatch(Key2, pcl_fetch(PclSnap, {o,"Bucket0002", "Key0002", null})), ?assertMatch(Key3, pcl_fetch(PclSnap, {o,"Bucket0003", "Key0003", null})), @@ -1171,6 +1178,7 @@ simple_server_test() -> % Add some more keys and confirm that check sequence number still % sees the old version in the previous snapshot, but will see the new version % in a new snapshot + Key1A_Pre = {{o,"Bucket0001", "Key0001", null}, {4005, {active, infinity}, null}}, Key1A = add_missing_hash(Key1A_Pre), @@ -1184,7 +1192,7 @@ simple_server_test() -> null}, 1)), ok = pcl_close(PclSnap), - + {ok, PclSnap2} = pcl_start(SnapOpts), leveled_bookie:load_snapshot(PclSnap2, leveled_bookie:empty_ledgercache()), ?assertMatch(false, pcl_checksequencenumber(PclSnap2, @@ -1199,12 +1207,6 @@ simple_server_test() -> "Key0001", null}, 4005)), - - io:format("Snap2 B2 K2 ~w~n", - [pcl_fetch(PclSnap2, {o, "Bucket0002", "Key0002", null})]), - io:format("r B2 K2 ~w~n", - [pcl_fetch(PCLr, {o, "Bucket0002", "Key0002", null})]), - ?assertMatch(true, pcl_checksequencenumber(PclSnap2, {o, "Bucket0002", diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index 62aa904..ec8f8b9 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -1203,6 +1203,7 @@ expand_list_by_pointer({pointer, SSTPid, Slot, StartKey, EndKey}, Tail, Width) - ExpPointers = leveled_sst:sst_getslots(SSTPid, AccPointers), lists:append(ExpPointers, AccTail); expand_list_by_pointer({next, SSTPid, StartKey, EndKey}, Tail, Width) -> + leveled_log:log("SST10", [SSTPid, is_pid(SSTPid)]), ExpPointer = leveled_sst:sst_getkvrange(SSTPid, StartKey, EndKey, Width), ExpPointer ++ Tail. From 85c03a61f9ee5ab42da0fd0ce632b645afee0f2a Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 14 Jan 2017 20:11:01 +0000 Subject: [PATCH 07/37] Log changes --- src/leveled_log.erl | 3 +-- src/leveled_manifest.erl | 15 ++++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/leveled_log.erl b/src/leveled_log.erl index 091e454..f8498f5 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -65,8 +65,7 @@ {"P0004", {info, "Remaining ledger snapshots are ~w"}}, {"P0005", - {info, "Delete confirmed as file ~s is removed from " ++ - "unreferenced files"}}, + {info, "Delete confirmed as file ~s is removed from Manifest"}}, {"P0006", {info, "Orphaned reply after timeout on L0 file write ~s"}}, {"P0007", diff --git a/src/leveled_manifest.erl b/src/leveled_manifest.erl index 894d344..d308cbc 100644 --- a/src/leveled_manifest.erl +++ b/src/leveled_manifest.erl @@ -45,12 +45,12 @@ add_snapshot/3, release_snapshot/2, ready_to_delete/2, + delete_confirmed/2, check_for_work/2, is_basement/2, dump_pidmap/1, levelzero_present/1, - pointer_convert/2, - delete_confirmed/2 + pointer_convert/2 ]). -export([ @@ -298,6 +298,12 @@ ready_to_delete(Manifest, Filename) -> P} end. +delete_confirmed(Manifest, Filename) -> + PidMap = dict:erase(Filename, Manifest#manifest.pidmap), + % Would be better to clear ETS at this point rather than on lookup? + Manifest#manifest{pidmap = PidMap}. + + check_for_work(Manifest, Thresholds) -> CheckLevelFun = fun({Level, MaxCount}, {AccL, AccC}) -> @@ -333,11 +339,6 @@ levelzero_present(Manifest) -> true end. -delete_confirmed(Manifest, Filename) -> - PidMap = dict:erase(Filename, Manifest#manifest.pidmap), - % Would be better to clear ETS at this point rather than on lookup? - Manifest#manifest{pidmap = PidMap}. - %%%============================================================================ %%% Internal Functions %%%============================================================================ From 86af99e5116ba3034ac466180da7f8d9dc60cf63 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 14 Jan 2017 20:19:08 +0000 Subject: [PATCH 08/37] Set timeout correctly --- src/leveled_penciller.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index a0fe4cb..a9f9520 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -212,7 +212,7 @@ -define(COIN_SIDECOUNT, 5). -define(SLOW_FETCH, 20000). -define(ITERATOR_SCANWIDTH, 4). --define(SNAPSHOT_TIMEOUT, 3600). +-define(SNAPSHOT_TIMEOUT, 3600 * 1000). -record(state, {manifest, % a manifest record from the leveled_manifest module persisted_sqn = 0 :: integer(), % The highest SQN persisted From 8f0a096dd2ab7b72b7269a8b65626304e7a6c437 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 14 Jan 2017 20:57:43 +0000 Subject: [PATCH 09/37] More work on timeout --- src/leveled_manifest.erl | 7 +++++-- src/leveled_penciller.erl | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/leveled_manifest.erl b/src/leveled_manifest.erl index d308cbc..cd6fc12 100644 --- a/src/leveled_manifest.erl +++ b/src/leveled_manifest.erl @@ -265,7 +265,9 @@ mergefile_selector(Manifest, Level) -> end_key = LastKey}. add_snapshot(Manifest, Pid, Timeout) -> - SnapEntry = {Pid, Manifest#manifest.manifest_sqn, Timeout}, + {MegaNow, SecNow, _} = os:timestamp(), + TimeToTimeout = MegaNow * 1000000 + SecNow + Timeout, + SnapEntry = {Pid, Manifest#manifest.manifest_sqn, TimeToTimeout}, SnapList0 = [SnapEntry|Manifest#manifest.snapshots], MinDelSQN = min(Manifest#manifest.delete_sqn, Manifest#manifest.manifest_sqn), Manifest#manifest{snapshots = SnapList0, delete_sqn = MinDelSQN}. @@ -292,9 +294,10 @@ ready_to_delete(Manifest, Filename) -> {P, infinity} -> {false, P}; {P, DeleteSQN} -> + {MegaNow, SecNow, _} = os:timestamp(), {ready_to_delete(Manifest#manifest.snapshots, DeleteSQN, - os:timestamp()), + MegaNow * 1000000 + SecNow), P} end. diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index a9f9520..a0fe4cb 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -212,7 +212,7 @@ -define(COIN_SIDECOUNT, 5). -define(SLOW_FETCH, 20000). -define(ITERATOR_SCANWIDTH, 4). --define(SNAPSHOT_TIMEOUT, 3600 * 1000). +-define(SNAPSHOT_TIMEOUT, 3600). -record(state, {manifest, % a manifest record from the leveled_manifest module persisted_sqn = 0 :: integer(), % The highest SQN persisted From dcf3afc05644cb5bda73520d904f44e85d6e293a Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 14 Jan 2017 21:19:51 +0000 Subject: [PATCH 10/37] Log basement setting when creating files --- src/leveled_log.erl | 3 ++- src/leveled_pclerk.erl | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/leveled_log.erl b/src/leveled_log.erl index f8498f5..5ebd976 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -149,7 +149,8 @@ {"PC011", {info, "Merge completed with MSN=~w to Level=~w and FileCounter=~w"}}, {"PC012", - {info, "File to be created as part of MSN=~w Filename=~s"}}, + {info, "File to be created as part of MSN=~w Filename=~s " + ++ "IsBasement=~w"}}, {"PC013", {warn, "Merge resulted in empty file ~s"}}, {"PC015", diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 8976761..10b710b 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -193,7 +193,7 @@ do_merge([], [], SinkLevel, _SinkB, _RP, NewSQN, _MaxSQN, Counter, Man0) -> do_merge(KL1, KL2, SinkLevel, SinkB, RP, NewSQN, MaxSQN, Counter, Man0) -> FileName = lists:flatten(io_lib:format(RP ++ "_~w_~w.sst", [SinkLevel, Counter])), - leveled_log:log("PC012", [NewSQN, FileName]), + leveled_log:log("PC012", [NewSQN, FileName, SinkB]), TS1 = os:timestamp(), case leveled_sst:sst_new(FileName, KL1, KL2, SinkB, SinkLevel, MaxSQN) of empty -> From ea171cb13b937aae0296cad155645ad6da663340 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 14 Jan 2017 22:03:57 +0000 Subject: [PATCH 11/37] Attempt to stop race - between level zero and ongoing merge --- src/leveled_penciller.erl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index a0fe4cb..04fa77c 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -235,8 +235,8 @@ source_penciller :: pid(), levelzero_astree :: list(), - ongoing_work = [] :: list(), - work_backlog = false :: boolean(), + work_ongoing = false :: boolean(), % i.e. compaction work + work_backlog = false :: boolean(), % i.e. compaction work head_timing :: tuple()}). @@ -443,13 +443,13 @@ handle_call(work_for_clerk, _From, State) -> [TL|_Tail] = WL, {reply, {TL, State#state.manifest}, - State#state{work_backlog=true}}; + State#state{work_backlog=true, work_ongoing=true}}; N -> leveled_log:log("P0024", [N, false]), [TL|_Tail] = WL, {reply, {TL, State#state.manifest}, - State#state{work_backlog=false}} + State#state{work_backlog=false, work_ongoing=true}} end end; handle_call(get_startup_sqn, _From, State) -> @@ -485,7 +485,7 @@ handle_call(doom, _From, State) -> {stop, normal, {ok, [ManifestFP, FilesFP]}, State}. handle_cast({manifest_change, NewManifest}, State) -> - {noreply, State#state{manifest = NewManifest}}; + {noreply, State#state{manifest = NewManifest, work_ongoing=false}}; handle_cast({release_snapshot, Snapshot}, State) -> Manifest0 = leveled_manifest:release_snapshot(State#state.manifest, Snapshot), @@ -676,9 +676,10 @@ update_levelzero(L0Size, {PushedTree, PushedIdx, MinSQN, MaxSQN}, false -> true end, + NoPendingManifestChange = not State#state.work_ongoing, JitterCheck = RandomFactor or CacheMuchTooBig, - case {CacheTooBig, L0Free, JitterCheck} of - {true, true, true} -> + case {CacheTooBig, L0Free, JitterCheck, NoPendingManifestChange} of + {true, true, true, true} -> L0Constructor = roll_memory(UpdState, false), leveled_log:log_timer("P0031", [], SW), UpdState#state{levelzero_pending=true, From 7b0b3e9b83ce928e4349d52ed5925798987c4598 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 14 Jan 2017 22:26:26 +0000 Subject: [PATCH 12/37] Add logging of Manifest SQN at startup --- src/leveled_log.erl | 2 ++ src/leveled_penciller.erl | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/leveled_log.erl b/src/leveled_log.erl index 5ebd976..fe9b448 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -126,6 +126,8 @@ {"P0034", {warn, "Snapshot with pid ~w timed out and so deletion will " ++ "continue regardless"}}, + {"P0035", + {info, "Startup with Manifest SQN of ~w~n"}}, {"PC001", {info, "Penciller's clerk ~w started with owner ~w"}}, {"PC002", diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 04fa77c..06fb75e 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -610,7 +610,7 @@ start_from_file(PCLopts) -> SQNFun), leveled_log:log("P0014", [MaxSQN]), ManSQN = leveled_manifest:get_manifest_sqn(Manifest1), - + leveled_log:log("P0035", [ManSQN]), %% Find any L0 files L0FN = filepath(RootPath, ManSQN, new_merge_files) ++ "_0_0.sst", case filelib:is_file(L0FN) of From 0c01a90f96e08c1cafea6ac2bf00590cca41f14b Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 14 Jan 2017 22:31:45 +0000 Subject: [PATCH 13/37] Manifest SQN at startup An L0 file outside the Manifest will be written at a Manifest SQN one higher - so need to look at stratup for one at one higher --- src/leveled_penciller.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 06fb75e..d506264 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -612,7 +612,7 @@ start_from_file(PCLopts) -> ManSQN = leveled_manifest:get_manifest_sqn(Manifest1), leveled_log:log("P0035", [ManSQN]), %% Find any L0 files - L0FN = filepath(RootPath, ManSQN, new_merge_files) ++ "_0_0.sst", + L0FN = filepath(RootPath, ManSQN + 1, new_merge_files) ++ "_0_0.sst", case filelib:is_file(L0FN) of true -> leveled_log:log("P0015", [L0FN]), From 9577d24be09ce87954a3172d4c79ce31c69370dc Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 14 Jan 2017 22:59:04 +0000 Subject: [PATCH 14/37] Wait on close of penciller clerk The clerk never calls the Penciller, so cannot deadlock. Will wait for a merge to be complete --- src/leveled_pclerk.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 10b710b..1233a1b 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -62,7 +62,7 @@ clerk_prompt(Pid) -> gen_server:cast(Pid, prompt). clerk_close(Pid) -> - gen_server:cast(Pid, close). + gen_server:call(Pid, close, 20000). %%%============================================================================ %%% gen_server callbacks @@ -72,12 +72,13 @@ init([]) -> {ok, #state{}}. handle_call({load, Owner, RootPath}, _From, State) -> - {reply, ok, State#state{owner=Owner, root_path=RootPath}, ?MIN_TIMEOUT}. + {reply, ok, State#state{owner=Owner, root_path=RootPath}, ?MIN_TIMEOUT}; +handle_call(close, _From, State) -> + {stop, normal, ok, State}. handle_cast(prompt, State) -> - handle_info(timeout, State); -handle_cast(close, State) -> - {stop, normal, State}. + handle_info(timeout, State). + handle_info(timeout, State) -> case requestandhandle_work(State) of @@ -89,7 +90,6 @@ handle_info(timeout, State) -> {noreply, State, ?MIN_TIMEOUT} end. - terminate(Reason, _State) -> leveled_log:log("PC005", [self(), Reason]). From f868be80862b5333a88b35bd8f0cb47c05b105ee Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Sun, 15 Jan 2017 00:52:43 +0000 Subject: [PATCH 15/37] Change Penciller to Clerk handoff correct the previous change to make sure that deletes are not confirmed when work is outstanding, but also make sure that the clerk only ever casts the Penciller so no deadlocks can ensue. --- src/leveled_pclerk.erl | 51 +++++++++++------------- src/leveled_penciller.erl | 81 ++++++++++++++++++++++----------------- 2 files changed, 69 insertions(+), 63 deletions(-) diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 1233a1b..c255943 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -37,12 +37,13 @@ -export([ clerk_new/2, clerk_prompt/1, + clerk_push/2, clerk_close/1 ]). -include_lib("eunit/include/eunit.hrl"). --define(MAX_TIMEOUT, 1000). +-define(MAX_TIMEOUT, 2000). -define(MIN_TIMEOUT, 200). -record(state, {owner :: pid(), @@ -61,6 +62,9 @@ clerk_new(Owner, Manifest) -> clerk_prompt(Pid) -> gen_server:cast(Pid, prompt). +clerk_push(Pid, Work) -> + gen_server:cast(Pid, {push_work, Work}). + clerk_close(Pid) -> gen_server:call(Pid, close, 20000). @@ -77,18 +81,14 @@ handle_call(close, _From, State) -> {stop, normal, ok, State}. handle_cast(prompt, State) -> - handle_info(timeout, State). - + handle_info(timeout, State); +handle_cast({push_work, Work}, State) -> + handle_work(Work, State), + {noreply, State, ?MIN_TIMEOUT}. handle_info(timeout, State) -> - case requestandhandle_work(State) of - false -> - {noreply, State, ?MAX_TIMEOUT}; - true -> - % No timeout now as will wait for call to return manifest - % change - {noreply, State, ?MIN_TIMEOUT} - end. + request_work(State), + {noreply, State, ?MAX_TIMEOUT}. terminate(Reason, _State) -> leveled_log:log("PC005", [self(), Reason]). @@ -101,24 +101,19 @@ code_change(_OldVsn, State, _Extra) -> %%% Internal functions %%%============================================================================ -requestandhandle_work(State) -> - case leveled_penciller:pcl_workforclerk(State#state.owner) of - none -> - leveled_log:log("PC006", []), - false; - {SrcLevel, Manifest} -> - {UpdManifest, EntriesToDelete} = merge(SrcLevel, - Manifest, - State#state.root_path), - leveled_log:log("PC007", []), - ok = leveled_penciller:pcl_manifestchange(State#state.owner, - UpdManifest), - ok = leveled_manifest:save_manifest(UpdManifest, - State#state.root_path), - ok = notify_deletions(EntriesToDelete, State#state.owner), - true - end. +request_work(State) -> + ok = leveled_penciller:pcl_workforclerk(State#state.owner). +handle_work({SrcLevel, Manifest}, State) -> + {UpdManifest, EntriesToDelete} = merge(SrcLevel, + Manifest, + State#state.root_path), + leveled_log:log("PC007", []), + ok = leveled_penciller:pcl_manifestchange(State#state.owner, + UpdManifest), + ok = leveled_manifest:save_manifest(UpdManifest, + State#state.root_path), + ok = notify_deletions(EntriesToDelete, State#state.owner). merge(SrcLevel, Manifest, RootPath) -> Src = leveled_manifest:mergefile_selector(Manifest, SrcLevel), diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index d506264..b340f7d 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -289,7 +289,7 @@ pcl_checksequencenumber(Pid, Key, SQN) -> end. pcl_workforclerk(Pid) -> - gen_server:call(Pid, work_for_clerk, infinity). + gen_server:cast(Pid, work_for_clerk). pcl_manifestchange(Pid, Manifest) -> gen_server:cast(Pid, {manifest_change, Manifest}). @@ -428,30 +428,6 @@ handle_call({fetch_keys, StartKey, EndKey, AccFun, InitAcc, MaxKeys}, MaxKeys), {reply, Acc, State#state{levelzero_astree = L0AsList}}; -handle_call(work_for_clerk, _From, State) -> - case State#state.levelzero_pending of - true -> - {reply, none, State}; - false -> - {WL, WC} = leveled_manifest:check_for_work(State#state.manifest, - ?LEVEL_SCALEFACTOR), - case WC of - 0 -> - {reply, none, State#state{work_backlog=false}}; - N when N > ?WORKQUEUE_BACKLOG_TOLERANCE -> - leveled_log:log("P0024", [N, true]), - [TL|_Tail] = WL, - {reply, - {TL, State#state.manifest}, - State#state{work_backlog=true, work_ongoing=true}}; - N -> - leveled_log:log("P0024", [N, false]), - [TL|_Tail] = WL, - {reply, - {TL, State#state.manifest}, - State#state{work_backlog=false, work_ongoing=true}} - end - end; handle_call(get_startup_sqn, _From, State) -> {reply, State#state.persisted_sqn, State}; handle_call({register_snapshot, Snapshot}, _From, State) -> @@ -493,15 +469,24 @@ handle_cast({release_snapshot, Snapshot}, State) -> {noreply, State#state{manifest=Manifest0}}; handle_cast({confirm_delete, Filename}, State=#state{is_snapshot=Snap}) when Snap == false -> - R2D = leveled_manifest:ready_to_delete(State#state.manifest, Filename), - case R2D of - {true, Pid} -> - leveled_log:log("P0005", [Filename]), - ok = leveled_sst:sst_deleteconfirmed(Pid), - Man0 = leveled_manifest:delete_confirmed(State#state.manifest, - Filename), - {noreply, State#state{manifest=Man0}}; - {false, _Pid} -> + case State#state.work_ongoing of + false -> + R2D = leveled_manifest:ready_to_delete(State#state.manifest, + Filename), + case R2D of + {true, Pid} -> + leveled_log:log("P0005", [Filename]), + ok = leveled_sst:sst_deleteconfirmed(Pid), + M0 = leveled_manifest:delete_confirmed(State#state.manifest, + Filename), + {noreply, State#state{manifest=M0}}; + {false, _Pid} -> + {noreply, State} + end; + true -> + % If there is ongoing work, then we can't safely update the pidmap + % as any change will be reverted when the manifest is passed back + % from the Clerk {noreply, State} end; handle_cast({levelzero_complete, FN, StartKey, EndKey}, State) -> @@ -524,7 +509,33 @@ handle_cast({levelzero_complete, FN, StartKey, EndKey}, State) -> levelzero_constructor=undefined, levelzero_size=0, manifest=UpdMan, - persisted_sqn=State#state.ledger_sqn}}. + persisted_sqn=State#state.ledger_sqn}}; +handle_cast(work_for_clerk, State) -> + case State#state.levelzero_pending of + true -> + {noreply, State}; + false -> + {WL, WC} = leveled_manifest:check_for_work(State#state.manifest, + ?LEVEL_SCALEFACTOR), + case WC of + 0 -> + {noreply, State#state{work_backlog=false}}; + N when N > ?WORKQUEUE_BACKLOG_TOLERANCE -> + leveled_log:log("P0024", [N, true]), + [TL|_Tail] = WL, + ok = leveled_pclerk:clerk_push(State#state.clerk, + {TL, State#state.manifest}), + {noreply, + State#state{work_backlog=true, work_ongoing=true}}; + N -> + leveled_log:log("P0024", [N, false]), + [TL|_Tail] = WL, + ok = leveled_pclerk:clerk_push(State#state.clerk, + {TL, State#state.manifest}), + {noreply, + State#state{work_backlog=false, work_ongoing=true}} + end + end. handle_info(_Info, State) -> From 8e92a2c5638f010b0bcd075974d0ca8835170c66 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sun, 15 Jan 2017 01:47:23 +0000 Subject: [PATCH 16/37] Fix pclerk unit test --- src/leveled_pclerk.erl | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index c255943..ad9afbf 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -261,18 +261,34 @@ merge_file_test() -> 2, KL4_L2, undefined), - E1 = #manifest_entry{owner = PidL1_1, filename = "../test/KL1_L1.sst"}, - E2 = #manifest_entry{owner = PidL2_1, filename = "../test/KL1_L2.sst"}, - E3 = #manifest_entry{owner = PidL2_2, filename = "../test/KL2_L2.sst"}, - E4 = #manifest_entry{owner = PidL2_3, filename = "../test/KL3_L2.sst"}, - E5 = #manifest_entry{owner = PidL2_4, filename = "../test/KL4_L2.sst"}, + + E1 = #manifest_entry{owner = PidL1_1, + filename = "../test/KL1_L1.sst", + end_key = lists:last(KL1_L1), + start_key = lists:nth(1, KL1_L1)}, + E2 = #manifest_entry{owner = PidL2_1, + filename = "../test/KL1_L2.sst", + end_key = lists:last(KL1_L2), + start_key = lists:nth(1, KL1_L2)}, + E3 = #manifest_entry{owner = PidL2_2, + filename = "../test/KL2_L2.sst", + end_key = lists:last(KL2_L2), + start_key = lists:nth(1, KL2_L2)}, + E4 = #manifest_entry{owner = PidL2_3, + filename = "../test/KL3_L2.sst", + end_key = lists:last(KL3_L2), + start_key = lists:nth(1, KL3_L2)}, + E5 = #manifest_entry{owner = PidL2_4, + filename = "../test/KL4_L2.sst", + end_key = lists:last(KL4_L2), + start_key = lists:nth(1, KL4_L2)}, Man0 = leveled_manifest:new_manifest(), - Man1 = leveled_manifest:insert_manifest_entry(Man0, 1, 2, E1), - Man2 = leveled_manifest:insert_manifest_entry(Man1, 1, 2, E2), - Man3 = leveled_manifest:insert_manifest_entry(Man2, 1, 2, E3), - Man4 = leveled_manifest:insert_manifest_entry(Man3, 1, 2, E4), - Man5 = leveled_manifest:insert_manifest_entry(Man4, 2, 1, E5), + Man1 = leveled_manifest:insert_manifest_entry(Man0, 1, 2, E2), + Man2 = leveled_manifest:insert_manifest_entry(Man1, 1, 2, E3), + Man3 = leveled_manifest:insert_manifest_entry(Man2, 1, 2, E4), + Man4 = leveled_manifest:insert_manifest_entry(Man3, 1, 2, E5), + Man5 = leveled_manifest:insert_manifest_entry(Man4, 2, 1, E1), {Man6, _Dels} = perform_merge(Man5, E1, [E2, E3, E4, E5], 1, "../test", 3), From 13c5cc88992e2453dcc4d0297c1fde258a409aea Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sun, 15 Jan 2017 10:36:50 +0000 Subject: [PATCH 17/37] Add timing points Add some timing points to manifest updates --- src/leveled_log.erl | 4 ++++ src/leveled_pclerk.erl | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/leveled_log.erl b/src/leveled_log.erl index fe9b448..637e99f 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -159,6 +159,10 @@ {info, "File created"}}, {"PC016", {info, "Slow fetch from SFT ~w of ~w microseconds with result ~w"}}, + {"PC017", + {info, "Notified clerk of manifest change"}}, + {"PC018", + {info, "Saved manifest file"}}, {"I0001", {info, "Unexpected failure to fetch value for Key=~w SQN=~w " diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index ad9afbf..31d9861 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -109,10 +109,14 @@ handle_work({SrcLevel, Manifest}, State) -> Manifest, State#state.root_path), leveled_log:log("PC007", []), + SWMC = os:timestamp(), ok = leveled_penciller:pcl_manifestchange(State#state.owner, UpdManifest), + leveled_log:log_timer("PC017", [], SWMC), + SWSM = os:timestamp(), ok = leveled_manifest:save_manifest(UpdManifest, State#state.root_path), + leveled_log:log_timer("PC018", [], SWSM), ok = notify_deletions(EntriesToDelete, State#state.owner). merge(SrcLevel, Manifest, RootPath) -> From e6c4c9eff8b95e480b372aa0d4a5619ceb769434 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sun, 15 Jan 2017 11:04:26 +0000 Subject: [PATCH 18/37] Log deletions from mnaifest (via GC) --- src/leveled_log.erl | 5 ++++- src/leveled_manifest.erl | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/leveled_log.erl b/src/leveled_log.erl index 637e99f..525586d 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -127,7 +127,10 @@ {warn, "Snapshot with pid ~w timed out and so deletion will " ++ "continue regardless"}}, {"P0035", - {info, "Startup with Manifest SQN of ~w~n"}}, + {info, "Startup with Manifest SQN of ~w"}}, + {"P0036", + {info, "Garbage collection on mnaifest removes key for filename ~s"}}, + {"PC001", {info, "Penciller's clerk ~w started with owner ~w"}}, {"PC002", diff --git a/src/leveled_manifest.erl b/src/leveled_manifest.erl index cd6fc12..316a09b 100644 --- a/src/leveled_manifest.erl +++ b/src/leveled_manifest.erl @@ -480,6 +480,7 @@ key_lookup(Manifest, Level, {LastKey, LastFN}, KeyToFind, ManSQN, GC) -> {true, GC_SQN} -> case TombSQN < GC_SQN of true -> + leveled_log:log("P0036", [element(3, K)]), ets:delete(Manifest, K); false -> ok From d16defa59725abb331b43db20d96884c947686ca Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sun, 15 Jan 2017 11:29:19 +0000 Subject: [PATCH 19/37] Remove non-compatible OTP16 option --- src/leveled_manifest.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leveled_manifest.erl b/src/leveled_manifest.erl index 316a09b..8437731 100644 --- a/src/leveled_manifest.erl +++ b/src/leveled_manifest.erl @@ -143,7 +143,7 @@ save_manifest(Manifest, RootPath) -> FP = filepath(RootPath, Manifest#manifest.manifest_sqn, current_manifest), ets:tab2file(Manifest#manifest.table, FP, - [{extended_info, [md5sum]}, {sync, true}]). + [{extended_info, [md5sum]}]). insert_manifest_entry(Manifest, ManSQN, Level, Entry) -> From 38c7c9be9b24e11262eb5fabe7031da1eab1006c Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sun, 15 Jan 2017 12:41:16 +0000 Subject: [PATCH 20/37] Attempt to fix manifets issues ETS manifest is making eveyrthing SLOW. Marked basement. Perhaps fixed some GC issues. --- src/leveled_manifest.erl | 66 +++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/src/leveled_manifest.erl b/src/leveled_manifest.erl index 8437731..ba69309 100644 --- a/src/leveled_manifest.erl +++ b/src/leveled_manifest.erl @@ -75,8 +75,10 @@ % An array of level counts to speed up compation work assessment snapshots :: list(), % A list of snaphots (i.e. clones) - delete_sqn :: integer() + delete_sqn :: integer()|infinity, % The lowest SQN of any clone + basement :: integer() + % Currently the lowest level (the largest number) }). %%%============================================================================ @@ -128,8 +130,10 @@ load_manifest(Manifest, PidFun, SQNFun) -> AccMan#manifest.pidmap), LC = array:get(L, AccMan#manifest.level_counts), LC0 = array:set(L, LC + 1, AccMan#manifest.level_counts), + Basement = max(AccMan#manifest.basement, L), AccMan0 = AccMan#manifest{pidmap = PidMap0, - level_counts = LC0}, + level_counts = LC0, + basement = Basement}, SQN = SQNFun(Pid), MaxSQN0 = max(MaxSQN, SQN), {MaxSQN0, AccMan0}; @@ -159,9 +163,11 @@ insert_manifest_entry(Manifest, ManSQN, Level, Entry) -> LC = array:get(Level, Manifest#manifest.level_counts), LCArray0 = array:set(Level, LC + 1, Manifest#manifest.level_counts), MaxManSQN = max(ManSQN, Manifest#manifest.manifest_sqn), + Basement = max(Level, Manifest#manifest.basement), Manifest#manifest{pidmap = PidMap0, level_counts = LCArray0, - manifest_sqn = MaxManSQN}. + manifest_sqn = MaxManSQN, + basement = Basement}. remove_manifest_entry(Manifest, ManSQN, Level, Entry) -> Key = {Level, Entry#manifest_entry.end_key, Entry#manifest_entry.filename}, @@ -185,24 +191,31 @@ get_manifest_sqn(Manifest) -> Manifest#manifest.manifest_sqn. key_lookup(Manifest, Level, Key) -> - GC = - case Manifest#manifest.is_clone of - true -> - false; - false -> - {true, Manifest#manifest.delete_sqn} - end, - FN = key_lookup(Manifest#manifest.table, - Level, - Key, - Manifest#manifest.manifest_sqn, - GC), - case FN of - false -> + case Level > Manifest#manifest.basement of + true -> false; - _ -> - {Pid, _TombSQN} = dict:fetch(FN, Manifest#manifest.pidmap), - Pid + false -> + GC = + case Manifest#manifest.is_clone of + true -> + false; + false -> + {true, + min(Manifest#manifest.delete_sqn, + Manifest#manifest.manifest_sqn)} + end, + FN = key_lookup(Manifest#manifest.table, + Level, + Key, + Manifest#manifest.manifest_sqn, + GC), + case FN of + false -> + false; + _ -> + {Pid, _TombSQN} = dict:fetch(FN, Manifest#manifest.pidmap), + Pid + end end. range_lookup(Manifest, Level, StartKey, EndKey) -> @@ -284,7 +297,7 @@ release_snapshot(Manifest, Pid) -> end, {SnapList0, DeleteSQN} = lists:foldl(FilterFun, - {[], Manifest#manifest.manifest_sqn}, + {[], infinity}, Manifest#manifest.snapshots), leveled_log:log("P0004", [SnapList0]), Manifest#manifest{snapshots = SnapList0, delete_sqn = DeleteSQN}. @@ -320,16 +333,7 @@ check_for_work(Manifest, Thresholds) -> lists:foldl(CheckLevelFun, {[], 0}, Thresholds). is_basement(Manifest, Level) -> - CheckFun = - fun(L, Acc) -> - case array:get(L, Manifest#manifest.level_counts) of - 0 -> - Acc; - _N -> - false - end - end, - lists:foldl(CheckFun, true, lists:seq(Level + 1, ?MAX_LEVELS)). + Level >= Manifest#manifest.basement. dump_pidmap(Manifest) -> dict:to_list(Manifest#manifest.pidmap). From 72d16af2b107073da3fbdecf40b484ab9e400430 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 16 Jan 2017 23:27:42 +0000 Subject: [PATCH 21/37] Switch out implementation of manifest This gives a new manifest implementation that is an array of lists. Just basic unit testing of lookup insertion and removal in this module. The API was changed subtly, and so nothing broader will work at this stage --- rebar.lock | 1 - src/leveled_manifest.erl | 644 +++++++++++++++++---------------------- 2 files changed, 275 insertions(+), 370 deletions(-) delete mode 100644 rebar.lock diff --git a/rebar.lock b/rebar.lock deleted file mode 100644 index 57afcca..0000000 --- a/rebar.lock +++ /dev/null @@ -1 +0,0 @@ -[]. diff --git a/src/leveled_manifest.erl b/src/leveled_manifest.erl index ba69309..16629cd 100644 --- a/src/leveled_manifest.erl +++ b/src/leveled_manifest.erl @@ -3,27 +3,7 @@ %% The manifest is an ordered set of files for each level to be used to find %% which file is relevant for a given key or range lookup at a given level. %% -%% It is implemented as an ETS table, primarily to optimise lookup speed, but -%% also for the convenience of the tab2file format to support the persisting -%% of the manifest. However, it needs to be a multi-version table, as the -%% table may be accessed by both the real actor but also cloned actors too, -%% and they need to see state at different points in time. -%% -%% The ets table is an ordered set. The Key is a tuple of: -%% -%% {Level, LastKey, Filename} -%% -%% for the file. The manifest entry will have a value of: -%% -%% {FirstKey, {active, aSQN}, {tomb, tSQN}} -%% -%% When an item is added to the manifest it is added with aSQN set to the -%% manifets SQN which is intended to make this change current, and a tSQN -%% of infinity. When an item is removed the element is first altered so -%% that the tSQN is set to the next ManifestSQN. When the active -%% (non-cloned) actor reads items in the manifest it should also reap any -%% tombstone elements that have passed the lowest manifest SQN of any of -%% the registered clones. + -module(leveled_manifest). @@ -38,19 +18,16 @@ get_manifest_sqn/1, key_lookup/3, range_lookup/4, - merge_lookup/4, insert_manifest_entry/4, remove_manifest_entry/4, + switch_manifest_entry/4, mergefile_selector/2, add_snapshot/3, release_snapshot/2, ready_to_delete/2, - delete_confirmed/2, check_for_work/2, is_basement/2, - dump_pidmap/1, - levelzero_present/1, - pointer_convert/2 + levelzero_present/1 ]). -export([ @@ -63,20 +40,18 @@ -define(MANIFEST_FP, "ledger_manifest"). -define(MAX_LEVELS, 8). --record(manifest, {table, - % A Multi-Version ETS table for lookup - pidmap, - % A dictionary to map filenames to {Pid, DeleteSQN} +-record(manifest, {levels, + % an array of lists or trees representing the manifest manifest_sqn = 0 :: integer(), % The current manifest SQN - is_clone = false :: boolean(), - % Is this manifest held by a clone (i.e. snapshot) - level_counts, - % An array of level counts to speed up compation work assessment snapshots :: list(), % A list of snaphots (i.e. clones) - delete_sqn :: integer()|infinity, - % The lowest SQN of any clone + min_snapshot_sqn = 0 :: integer(), + % The smallest snapshot manifest SQN in the snapshot + % list + pending_deletes :: dict:dict(), + % a dictionary mapping keys (filenames) to SQN when + % the deletion was made basement :: integer() % Currently the lowest level (the largest number) }). @@ -86,8 +61,13 @@ %%%============================================================================ new_manifest() -> - Table = ets:new(manifest, [ordered_set, public]), - new_manifest(Table). + #manifest{ + levels = array:new([{size, ?MAX_LEVELS + 1}, {default, []}]), + manifest_sqn = 0, + snapshots = [], + pending_deletes = dict:new(), + basement = 0 + }. open_manifest(RootPath) -> % Open the manifest in the file path which has the highest SQN, and will @@ -107,149 +87,114 @@ open_manifest(RootPath) -> ValidManSQNs = lists:reverse(lists:sort(lists:foldl(ExtractSQNFun, [], Filenames))), - {ManSQN, Manifest} = open_manifestfile(RootPath, ValidManSQNs), - Manifest#manifest{manifest_sqn = ManSQN, delete_sqn = ManSQN}. + Manifest = open_manifestfile(RootPath, ValidManSQNs), + Manifest. copy_manifest(Manifest) -> % Copy the manifest ensuring anything only the master process should care % about is switched to undefined - #manifest{is_clone = true, - table = Manifest#manifest.table, - manifest_sqn = Manifest#manifest.manifest_sqn, - pidmap = Manifest#manifest.pidmap}. + Manifest#manifest{snapshots = undefined, pending_deletes = undefined}. load_manifest(Manifest, PidFun, SQNFun) -> - FlatManifest = ets:tab2list(Manifest#manifest.table), - InitiateFun = - fun({{L, _EK, FN}, {_SK, ActSt, DelSt}}, {MaxSQN, AccMan}) -> - case {ActSt, DelSt} of - {{active, _ActSQN}, {tomb, infinity}} -> - Pid = PidFun(FN), - PidMap0 = dict:store(FN, - {Pid, infinity}, - AccMan#manifest.pidmap), - LC = array:get(L, AccMan#manifest.level_counts), - LC0 = array:set(L, LC + 1, AccMan#manifest.level_counts), - Basement = max(AccMan#manifest.basement, L), - AccMan0 = AccMan#manifest{pidmap = PidMap0, - level_counts = LC0, - basement = Basement}, - SQN = SQNFun(Pid), - MaxSQN0 = max(MaxSQN, SQN), - {MaxSQN0, AccMan0}; - {_, {tomb, _TombSQN}} -> - {MaxSQN, AccMan} - end + UpdateLevelFun = + fun(LevelIdx, {AccMaxSQN, AccMan}) -> + L0 = array:get(LevelIdx, AccMan#manifest.levels), + {L1, SQN1} = load_level(LevelIdx, L0, PidFun, SQNFun), + {max(AccMaxSQN, SQN1), + AccMan#manifest{levels = array:set(LevelIdx, L1, AccMan)}} end, - lists:foldl(InitiateFun, {1, Manifest}, FlatManifest). - + lists:foldl(UpdateLevelFun, {0, Manifest}, + lists:seq(0, Manifest#manifest.basement)). + save_manifest(Manifest, RootPath) -> FP = filepath(RootPath, Manifest#manifest.manifest_sqn, current_manifest), - ets:tab2file(Manifest#manifest.table, - FP, - [{extended_info, [md5sum]}]). + ManBin = term_to_binary(Manifest), + CRC = erlang:crc32(ManBin), + ok = file:write_file(FP, <>). +insert_manifest_entry(Manifest, ManSQN, LevelIdx, Entry) -> + Levels = Manifest#manifest.levels, + Level = array:get(LevelIdx, Levels), + UpdLevel = add_entry(LevelIdx, Level, Entry), + Basement = max(LevelIdx, Manifest#manifest.basement), + Manifest#manifest{levels = array:set(LevelIdx, UpdLevel, Levels), + basement = Basement, + manifest_sqn = ManSQN}. -insert_manifest_entry(Manifest, ManSQN, Level, Entry) -> - Key = {Level, Entry#manifest_entry.end_key, Entry#manifest_entry.filename}, - Pid = Entry#manifest_entry.owner, - Value = {Entry#manifest_entry.start_key, - {active, ManSQN}, - {tomb, infinity}}, - true = ets:insert_new(Manifest#manifest.table, {Key, Value}), - PidMap0 = dict:store(Entry#manifest_entry.filename, - {Pid, infinity}, - Manifest#manifest.pidmap), - LC = array:get(Level, Manifest#manifest.level_counts), - LCArray0 = array:set(Level, LC + 1, Manifest#manifest.level_counts), - MaxManSQN = max(ManSQN, Manifest#manifest.manifest_sqn), - Basement = max(Level, Manifest#manifest.basement), - Manifest#manifest{pidmap = PidMap0, - level_counts = LCArray0, - manifest_sqn = MaxManSQN, - basement = Basement}. +remove_manifest_entry(Manifest, ManSQN, LevelIdx, Entry) -> + Levels = Manifest#manifest.levels, + Level = array:get(LevelIdx, Levels), + UpdLevel = remove_entry(LevelIdx, Level, Entry), + DelFun = + fun(E, Acc) -> + dict:store(E#manifest_entry.filename, ManSQN, Acc) + end, + Entries = + case is_list(Entry) of + true -> + Entry; + false -> + [Entry] + end, + PendingDeletes = lists:foldl(DelFun, + Manifest#manifest.pending_deletes, + Entries), + UpdLevels = array:set(LevelIdx, UpdLevel, Levels), + case is_empty(LevelIdx, UpdLevel) of + true -> + Manifest#manifest{levels = UpdLevels, + basement = get_basement(UpdLevels), + manifest_sqn = ManSQN, + pending_deletes = PendingDeletes}; + false -> + Manifest#manifest{levels = UpdLevels, + manifest_sqn = ManSQN, + pending_deletes = PendingDeletes} + end. -remove_manifest_entry(Manifest, ManSQN, Level, Entry) -> - Key = {Level, Entry#manifest_entry.end_key, Entry#manifest_entry.filename}, - [{Key, Value0}] = ets:lookup(Manifest#manifest.table, Key), - {StartKey, {active, ActiveSQN}, {tomb, infinity}} = Value0, - Value1 = {StartKey, {active, ActiveSQN}, {tomb, ManSQN}}, - true = ets:insert(Manifest#manifest.table, {Key, Value1}), - {Pid, infinity} = dict:fetch(Entry#manifest_entry.filename, - Manifest#manifest.pidmap), - PidMap0 = dict:store(Entry#manifest_entry.filename, - {Pid, ManSQN}, - Manifest#manifest.pidmap), - LC = array:get(Level, Manifest#manifest.level_counts), - LCArray0 = array:set(Level, LC - 1, Manifest#manifest.level_counts), - MaxManSQN = max(ManSQN, Manifest#manifest.manifest_sqn), - Manifest#manifest{pidmap = PidMap0, - level_counts = LCArray0, - manifest_sqn = MaxManSQN}. +switch_manifest_entry(Manifest, ManSQN, SrcLevel, Entry) -> + % Move to level below - so needs to be removed but not marked as a + % pending deletion + Levels = Manifest#manifest.levels, + Level = array:get(SrcLevel, Levels), + UpdLevel = remove_entry(SrcLevel, Level, Entry), + UpdLevels = array:set(SrcLevel, UpdLevel, Levels), + insert_manifest_entry(Manifest#manifest{levels = UpdLevels}, + ManSQN, + SrcLevel + 1, + Entry). get_manifest_sqn(Manifest) -> Manifest#manifest.manifest_sqn. -key_lookup(Manifest, Level, Key) -> - case Level > Manifest#manifest.basement of +key_lookup(Manifest, LevelIdx, Key) -> + case LevelIdx > Manifest#manifest.basement of true -> false; false -> - GC = - case Manifest#manifest.is_clone of - true -> - false; - false -> - {true, - min(Manifest#manifest.delete_sqn, - Manifest#manifest.manifest_sqn)} - end, - FN = key_lookup(Manifest#manifest.table, - Level, - Key, - Manifest#manifest.manifest_sqn, - GC), - case FN of - false -> - false; - _ -> - {Pid, _TombSQN} = dict:fetch(FN, Manifest#manifest.pidmap), - Pid - end + key_lookup_level(LevelIdx, + array:get(LevelIdx, Manifest#manifest.levels), + Key) end. -range_lookup(Manifest, Level, StartKey, EndKey) -> - MapFun = - fun({{_Level, _LastKey, FN}, FirstKey}) -> - {Pid, _SQN} = dict:fetch(FN, Manifest#manifest.pidmap), - case FirstKey < StartKey of - true -> - {next, Pid, StartKey}; - false -> - {next, Pid, FirstKey} - end +range_lookup(Manifest, LevelIdx, StartKey, EndKey) -> + Range = + case LevelIdx > Manifest#manifest.basement of + true -> + []; + false -> + range_lookup_level(LevelIdx, + array:get(LevelIdx, + Manifest#manifest.levels), + StartKey, + EndKey) end, - range_lookup(Manifest, Level, StartKey, EndKey, MapFun). + MakePointerFun = + fun(M) -> + {next, M, StartKey} + end, + lists:map(MakePointerFun, Range). -merge_lookup(Manifest, Level, StartKey, EndKey) -> - MapFun = - fun({{_Level, LastKey, FN}, FirstKey}) -> - {Owner, _DelSQN} = dict:fetch(FN, Manifest#manifest.pidmap), - #manifest_entry{filename = FN, - owner = Owner, - start_key = FirstKey, - end_key = LastKey} - end, - range_lookup(Manifest, Level, StartKey, EndKey, MapFun). - -pointer_convert(Manifest, EntryList) -> - MapFun = - fun(Entry) -> - {Pid, _DelSQN} = dict:fetch(Entry#manifest_entry.filename, - Manifest#manifest.pidmap), - {next, Pid, all} - end, - lists:map(MapFun, EntryList). %% An algorithm for discovering which files to merge .... %% We can find the most optimal file: @@ -261,174 +206,179 @@ pointer_convert(Manifest, EntryList) -> %% genuinely better - eventually every file has to be compacted. %% %% Hence, the initial implementation is to select files to merge at random -mergefile_selector(Manifest, Level) -> - KL = range_lookup(Manifest#manifest.table, - Level, - {all, 0}, - all, - all, - [], - Manifest#manifest.manifest_sqn), - {{Level, LastKey, FN}, - FirstKey} = lists:nth(random:uniform(length(KL)), KL), - {Owner, infinity} = dict:fetch(FN, Manifest#manifest.pidmap), - #manifest_entry{filename = FN, - owner = Owner, - start_key = FirstKey, - end_key = LastKey}. +mergefile_selector(Manifest, LevelIdx) -> + Level = array:get(LevelIdx, Manifest#manifest.levels), + lists:nth(random:uniform(length(Level), Level)). add_snapshot(Manifest, Pid, Timeout) -> {MegaNow, SecNow, _} = os:timestamp(), TimeToTimeout = MegaNow * 1000000 + SecNow + Timeout, SnapEntry = {Pid, Manifest#manifest.manifest_sqn, TimeToTimeout}, SnapList0 = [SnapEntry|Manifest#manifest.snapshots], - MinDelSQN = min(Manifest#manifest.delete_sqn, Manifest#manifest.manifest_sqn), - Manifest#manifest{snapshots = SnapList0, delete_sqn = MinDelSQN}. + ManSQN = Manifest#manifest.manifest_sqn, + case Manifest#manifest.min_snapshot_sqn of + 0 -> + + Manifest#manifest{snapshots = SnapList0, + min_snapshot_sqn = ManSQN}; + N -> + N0 = min(N, ManSQN), + Manifest#manifest{snapshots = SnapList0, min_snapshot_sqn = N0} + end. release_snapshot(Manifest, Pid) -> FilterFun = fun({P, SQN, TS}, {Acc, MinSQN}) -> case P of Pid -> - {Acc, MinSQN}; + {Acc, min(SQN, MinSQN)}; _ -> {[{P, SQN, TS}|Acc], min(SQN, MinSQN)} end end, - {SnapList0, - DeleteSQN} = lists:foldl(FilterFun, - {[], infinity}, - Manifest#manifest.snapshots), + {SnapList0, MinSnapSQN} = lists:foldl(FilterFun, + {[], infinity}, + Manifest#manifest.snapshots), leveled_log:log("P0004", [SnapList0]), - Manifest#manifest{snapshots = SnapList0, delete_sqn = DeleteSQN}. - -ready_to_delete(Manifest, Filename) -> - case dict:fetch(Filename, Manifest#manifest.pidmap) of - {P, infinity} -> - {false, P}; - {P, DeleteSQN} -> - {MegaNow, SecNow, _} = os:timestamp(), - {ready_to_delete(Manifest#manifest.snapshots, - DeleteSQN, - MegaNow * 1000000 + SecNow), - P} + case SnapList0 of + [] -> + Manifest#manifest{snapshots = SnapList0, + min_snapshot_sqn = 0}; + _ -> + Manifest#manifest{snapshots = SnapList0, + min_snapshot_sqn = MinSnapSQN} end. -delete_confirmed(Manifest, Filename) -> - PidMap = dict:erase(Filename, Manifest#manifest.pidmap), - % Would be better to clear ETS at this point rather than on lookup? - Manifest#manifest{pidmap = PidMap}. - +ready_to_delete(Manifest, Filename) -> + ChangeSQN = dict:fetch(Filename, Manifest#manifest.pending_deletes), + case Manifest#manifest.min_snapshot_sqn >= ChangeSQN of + true -> + % Every snapshot is looking at a version of history after this + % was removed + PDs = dict:erase(Filename, Manifest#manifest.pending_deletes), + {true, Manifest#manifest{pending_deletes = PDs}}; + false -> + {false, Manifest} + end. check_for_work(Manifest, Thresholds) -> CheckLevelFun = - fun({Level, MaxCount}, {AccL, AccC}) -> - case array:get(Level, Manifest#manifest.level_counts) of - LC when LC > MaxCount -> - {[Level|AccL], AccC + LC - MaxCount}; - _ -> - {AccL, AccC} + fun({LevelIdx, MaxCount}, {AccL, AccC}) -> + case LevelIdx > Manifest#manifest.basement of + true -> + {AccL, AccC}; + false -> + Level = array:get(LevelIdx, Manifest#manifest.levels), + S = size(LevelIdx, Level), + case S > MaxCount of + true -> + {[LevelIdx|AccL], AccC + S - MaxCount}; + false -> + {AccL, AccC} + end end end, - lists:foldl(CheckLevelFun, {[], 0}, Thresholds). + lists:foldr(CheckLevelFun, {[], 0}, Thresholds). is_basement(Manifest, Level) -> Level >= Manifest#manifest.basement. -dump_pidmap(Manifest) -> - dict:to_list(Manifest#manifest.pidmap). - levelzero_present(Manifest) -> - case key_lookup(Manifest, 0, all) of - false -> - false; - _ -> - true - end. + not is_empty(0, array:get(0, Manifest#manifest.levels)). %%%============================================================================ %%% Internal Functions %%%============================================================================ +%% All these internal functions that work on a level are also passed LeveIdx +%% even if this is not presently relevant. Currnetly levels are lists, but +%% future branches may make lower levels trees or skiplists to improve fetch +%% efficiency -new_manifest(Table) -> - #manifest{ - table = Table, - pidmap = dict:new(), - level_counts = array:new([{size, ?MAX_LEVELS + 1}, {default, 0}]), - snapshots = [], - delete_sqn = 0 - }. +load_level(_LevelIdx, Level, PidFun, SQNFun) -> + LevelLoadFun = + fun(ME, {L_Out, L_MaxSQN}) -> + FN = ME#manifest_entry.filename, + {ok, P, _Keys} = PidFun(FN), + SQN = SQNFun(P), + {[ME#manifest_entry{owner=P}|L_Out], max(SQN, L_MaxSQN)} + end, + lists:foldr(LevelLoadFun, {[], 0}, Level). -range_lookup(Manifest, Level, StartKey, EndKey, MapFun) -> - KL = range_lookup(Manifest#manifest.table, - Level, - {StartKey, 0}, - StartKey, - EndKey, - [], - Manifest#manifest.manifest_sqn), - lists:map(MapFun, KL). -range_lookup(Manifest, Level, {LastKey, LastFN}, SK, EK, Acc, ManSQN) -> - case ets:next(Manifest, {Level, LastKey, LastFN}) of - '$end_of_table' -> - Acc; - {Level, NextKey, NextFN} -> - [{K, V}] = ets:lookup(Manifest, {Level, NextKey, NextFN}), - {FirstKey, {active, ActiveSQN}, {tomb, TombSQN}} = V, - Active = (ManSQN >= ActiveSQN) and (ManSQN < TombSQN), - case Active of +is_empty(_LevelIdx, []) -> + true; +is_empty(_LevelIdx, _Level) -> + false. + +size(_LevelIdx, Level) -> + length(Level). + +add_entry(_LevelIdx, Level, Entries) when is_list(Entries) -> + lists:sort(Level ++ Entries); +add_entry(_LevelIdx, Level, Entry) -> + lists:sort([Entry|Level]). + +remove_entry(_LevelIdx, Level, Entries) when is_list(Entries) -> + % We're assuming we're removing a sorted sublist + RemLength = length(Entries), + RemStart = lists:nth(1, Entries), + remove_section(Level, RemStart#manifest_entry.start_key, RemLength - 1); +remove_entry(_LevelIdx, Level, Entry) -> + remove_section(Level, Entry#manifest_entry.start_key, 0). + +remove_section(Level, StartKey, Length) -> + PredFun = + fun(E) -> + E#manifest_entry.start_key < StartKey + end, + {Pre, Rest} = lists:splitwith(PredFun, Level), + Post = lists:nthtail(length(Rest) - Length, Rest), + Pre ++ Post. + + + +key_lookup_level(_LevelIdx, [], _Key) -> + false; +key_lookup_level(LevelIdx, [Entry|Rest], Key) -> + case Entry#manifest_entry.end_key >= Key of + true -> + case Key >= Entry#manifest_entry.start_key of true -> - PostEnd = leveled_codec:endkey_passed(EK, FirstKey), - case PostEnd of - true -> - Acc; - false -> - range_lookup(Manifest, - Level, - {NextKey, NextFN}, - SK, - EK, - Acc ++ [{K, FirstKey}], - ManSQN) - end; + Entry#manifest_entry.owner; false -> - range_lookup(Manifest, - Level, - {NextKey, NextFN}, - SK, - EK, - Acc, - ManSQN) + false end; - {OtherLevel, _, _} when OtherLevel > Level -> - Acc + false -> + key_lookup_level(LevelIdx, Rest, Key) end. -ready_to_delete(SnapList, FileDeleteSQN, Now) -> - FilterFun = - fun({P, SnapSQN, ExpiryTS}, Acc) -> - case Acc of +range_lookup_level(_LevelIdx, Level, QStartKey, QEndKey) -> + BeforeFun = + fun(M) -> + QStartKey > M#manifest_entry.end_key + end, + NotAfterFun = + fun(M) -> + not leveled_codec:endkey_passed(QEndKey, + M#manifest_entry.start_key) + end, + {_Before, MaybeIn} = lists:splitwith(BeforeFun, Level), + {In, _After} = lists:splitwith(NotAfterFun, MaybeIn), + In. + +get_basement(Levels) -> + GetBaseFun = + fun(L, Acc) -> + case is_empty(L, array:get(L, Levels)) of false -> - false; + max(L, Acc); true -> - case FileDeleteSQN < SnapSQN of - true -> - % Snapshot taken after the file deletion - true; - false -> - case Now > ExpiryTS of - true -> - leveled_log:log("P0034", [P]), - true; - false -> - false - end - end + Acc end end, - lists:foldl(FilterFun, true, SnapList). + lists:foldl(GetBaseFun, 0, lists:seq(0, ?MAX_LEVELS)). + filepath(RootPath, manifest) -> MFP = RootPath ++ "/" ++ ?MANIFEST_FP ++ "/", @@ -448,60 +398,17 @@ open_manifestfile(_RootPath, [0]) -> {0, new_manifest()}; open_manifestfile(RootPath, [TopManSQN|Rest]) -> CurrManFile = filepath(RootPath, TopManSQN, current_manifest), - case ets:file2tab(CurrManFile, [{verify,true}]) of - {error, Reason} -> - leveled_log:log("P0033", [CurrManFile, Reason]), - open_manifestfile(RootPath, Rest); - {ok, Table} -> + FileBin = file:read_file(CurrManFile), + <> = FileBin, + case erlang:crc32(BinaryOfTerm) of + CRC -> leveled_log:log("P0012", [TopManSQN]), - {TopManSQN, new_manifest(Table)} + binary_to_term(BinaryOfTerm); + _ -> + leveled_log:log("P0033", [CurrManFile, "crc wonky"]), + open_manifestfile(RootPath, Rest) end. -key_lookup(Manifest, Level, KeyToFind, ManSQN, GC) -> - key_lookup(Manifest, Level, {KeyToFind, any}, KeyToFind, ManSQN, GC). - -key_lookup(Manifest, Level, {LastKey, LastFN}, KeyToFind, ManSQN, GC) -> - case ets:next(Manifest, {Level, LastKey, LastFN}) of - '$end_of_table' -> - false; - {Level, NextKey, NextFN} -> - [{K, V}] = ets:lookup(Manifest, {Level, NextKey, NextFN}), - {FirstKey, {active, ActiveSQN}, {tomb, TombSQN}} = V, - Active = (ManSQN >= ActiveSQN) and (ManSQN < TombSQN), - case Active of - true -> - InRange = (KeyToFind >= FirstKey) or (KeyToFind == all), - case InRange of - true -> - NextFN; - false -> - false - end; - false -> - case GC of - false -> - ok; - {true, GC_SQN} -> - case TombSQN < GC_SQN of - true -> - leveled_log:log("P0036", [element(3, K)]), - ets:delete(Manifest, K); - false -> - ok - end - end, - key_lookup(Manifest, - Level, - {NextKey, NextFN}, - KeyToFind, - ManSQN, - GC) - end; - {OtherLevel, _, _} when OtherLevel > Level -> - false - end. - - %%%============================================================================ %%% Test %%%============================================================================ @@ -575,14 +482,16 @@ changeup_setup(Man6) -> owner="pid_y4", filename="Y4"}, - Man7 = insert_manifest_entry(Man6, 2, 1, E1_2), - Man8 = insert_manifest_entry(Man7, 2, 1, E2_2), - Man9 = insert_manifest_entry(Man8, 2, 1, E3_2), - Man10 = insert_manifest_entry(Man9, 2, 1, E4_2), + Man7 = remove_manifest_entry(Man6, 2, 1, E1), + Man8 = remove_manifest_entry(Man7, 2, 1, E2), + Man9 = remove_manifest_entry(Man8, 2, 1, E3), + + Man10 = insert_manifest_entry(Man9, 2, 1, E1_2), + Man11 = insert_manifest_entry(Man10, 2, 1, E2_2), + Man12 = insert_manifest_entry(Man11, 2, 1, E3_2), + Man13 = insert_manifest_entry(Man12, 2, 1, E4_2), % remove_manifest_entry(Manifest, ManSQN, Level, Entry) - Man11 = remove_manifest_entry(Man10, 2, 1, E1), - Man12 = remove_manifest_entry(Man11, 2, 1, E2), - Man13 = remove_manifest_entry(Man12, 2, 1, E3), + {Man7, Man8, Man9, Man10, Man11, Man12, Man13}. keylookup_manifest_test() -> @@ -620,7 +529,6 @@ keylookup_manifest_test() -> ?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_2)), ?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_3)), - io:format("Commencing failing test:~n"), ?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_4)), ?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_5)), @@ -636,50 +544,48 @@ keylookup_manifest_test() -> rangequery_manifest_test() -> {_Man0, _Man1, _Man2, _Man3, _Man4, _Man5, Man6} = initial_setup(), + PidMapFun = + fun(Pointer) -> + {next, ME, _SK} = Pointer, + ME#manifest_entry.owner + end, + SK1 = {o, "Bucket1", "K711", null}, EK1 = {o, "Bucket1", "K999", null}, - RL1_1 = range_lookup(Man6, 1, SK1, EK1), - ?assertMatch([{next, "pid_z3", {o, "Bucket1", "K75", null}}], RL1_1), - RL1_2 = range_lookup(Man6, 2, SK1, EK1), - ?assertMatch([{next, "pid_z5", {o, "Bucket1", "K711", null}}, - {next, "pid_z6", {o, "Bucket1", "K81", null}}], - RL1_2), + RL1_1 = lists:map(PidMapFun, range_lookup(Man6, 1, SK1, EK1)), + ?assertMatch(["pid_z3"], RL1_1), + RL1_2 = lists:map(PidMapFun, range_lookup(Man6, 2, SK1, EK1)), + ?assertMatch(["pid_z5", "pid_z6"], RL1_2), SK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, EK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, - RL2_1 = range_lookup(Man6, 1, SK2, EK2), - ?assertMatch([{next, "pid_z1", {i, "Bucket1", {"Idx1", "Fld8"}, null}}], - RL2_1), - RL2_2 = range_lookup(Man6, 2, SK2, EK2), - ?assertMatch([{next, "pid_z5", {i, "Bucket1", {"Idx1", "Fld8"}, null}}], - RL2_2), + RL2_1 = lists:map(PidMapFun, range_lookup(Man6, 1, SK2, EK2)), + ?assertMatch(["pid_z1"], RL2_1), + RL2_2 = lists:map(PidMapFun, range_lookup(Man6, 2, SK2, EK2)), + ?assertMatch(["pid_z5"], RL2_2), SK3 = {o, "Bucket1", "K994", null}, EK3 = {o, "Bucket1", "K995", null}, - RL3_1 = range_lookup(Man6, 1, SK3, EK3), + RL3_1 = lists:map(PidMapFun, range_lookup(Man6, 1, SK3, EK3)), ?assertMatch([], RL3_1), - RL3_2 = range_lookup(Man6, 2, SK3, EK3), - ?assertMatch([{next, "pid_z6", {o, "Bucket1", "K994", null}}], RL3_2), + RL3_2 = lists:map(PidMapFun, range_lookup(Man6, 2, SK3, EK3)), + ?assertMatch(["pid_z6"], RL3_2), {_Man7, _Man8, _Man9, _Man10, _Man11, _Man12, Man13} = changeup_setup(Man6), - % Results unchanged despiter ES table change if using old manifest - RL1_1A = range_lookup(Man6, 1, SK1, EK1), - ?assertMatch([{next, "pid_z3", {o, "Bucket1", "K75", null}}], RL1_1A), - RL2_1A = range_lookup(Man6, 1, SK2, EK2), - ?assertMatch([{next, "pid_z1", {i, "Bucket1", {"Idx1", "Fld8"}, null}}], - RL2_1A), - RL3_1A = range_lookup(Man6, 1, SK3, EK3), + RL1_1A = lists:map(PidMapFun, range_lookup(Man6, 1, SK1, EK1)), + ?assertMatch(["pid_z3"], RL1_1A), + RL2_1A = lists:map(PidMapFun, range_lookup(Man6, 1, SK2, EK2)), + ?assertMatch(["pid_z1"], RL2_1A), + RL3_1A = lists:map(PidMapFun, range_lookup(Man6, 1, SK3, EK3)), ?assertMatch([], RL3_1A), - RL1_1B = range_lookup(Man13, 1, SK1, EK1), - ?assertMatch([{next, "pid_y3", {o, "Bucket1", "K711", null}}, - {next, "pid_y4", {o, "Bucket1", "K815", null}}], RL1_1B), - RL2_1B = range_lookup(Man13, 1, SK2, EK2), - ?assertMatch([{next, "pid_y1", {i, "Bucket1", {"Idx1", "Fld8"}, null}}], - RL2_1B), - RL3_1B = range_lookup(Man13, 1, SK3, EK3), - ?assertMatch([{next, "pid_y4", {o, "Bucket1", "K994", null}}], RL3_1B). + RL1_1B = lists:map(PidMapFun, range_lookup(Man13, 1, SK1, EK1)), + ?assertMatch(["pid_y3", "pid_y4"], RL1_1B), + RL2_1B = lists:map(PidMapFun, range_lookup(Man13, 1, SK2, EK2)), + ?assertMatch(["pid_y1"], RL2_1B), + RL3_1B = lists:map(PidMapFun, range_lookup(Man13, 1, SK3, EK3)), + ?assertMatch(["pid_y4"], RL3_1B). levelzero_present_test() -> E0 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, From 9832ecc3690c4d483187d8245d5c6bb394210bae Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 17 Jan 2017 10:12:15 +0000 Subject: [PATCH 22/37] Manifest now back to a simple list This has refactored code with the implementation of the manifest isolated in to a seperate module, and the pure async relationship between penciller and their clerk. However, the manifest is just a simple list at each level. --- src/leveled_manifest.erl | 69 ++++++++++++++++++----------- src/leveled_pclerk.erl | 91 ++++++++++++++++++++++----------------- src/leveled_penciller.erl | 11 +++-- src/leveled_sst.erl | 9 ++-- 4 files changed, 108 insertions(+), 72 deletions(-) diff --git a/src/leveled_manifest.erl b/src/leveled_manifest.erl index 16629cd..f27e421 100644 --- a/src/leveled_manifest.erl +++ b/src/leveled_manifest.erl @@ -14,10 +14,12 @@ open_manifest/1, copy_manifest/1, load_manifest/3, + close_manifest/2, save_manifest/2, get_manifest_sqn/1, key_lookup/3, range_lookup/4, + merge_lookup/4, insert_manifest_entry/4, remove_manifest_entry/4, switch_manifest_entry/4, @@ -87,8 +89,7 @@ open_manifest(RootPath) -> ValidManSQNs = lists:reverse(lists:sort(lists:foldl(ExtractSQNFun, [], Filenames))), - Manifest = open_manifestfile(RootPath, ValidManSQNs), - Manifest. + open_manifestfile(RootPath, ValidManSQNs). copy_manifest(Manifest) -> % Copy the manifest ensuring anything only the master process should care @@ -100,12 +101,20 @@ load_manifest(Manifest, PidFun, SQNFun) -> fun(LevelIdx, {AccMaxSQN, AccMan}) -> L0 = array:get(LevelIdx, AccMan#manifest.levels), {L1, SQN1} = load_level(LevelIdx, L0, PidFun, SQNFun), - {max(AccMaxSQN, SQN1), - AccMan#manifest{levels = array:set(LevelIdx, L1, AccMan)}} + UpdLevels = array:set(LevelIdx, L1, AccMan#manifest.levels), + {max(AccMaxSQN, SQN1), AccMan#manifest{levels = UpdLevels}} end, lists:foldl(UpdateLevelFun, {0, Manifest}, lists:seq(0, Manifest#manifest.basement)). - + +close_manifest(Manifest, CloseEntryFun) -> + CloseLevelFun = + fun(LevelIdx) -> + Level = array:get(LevelIdx, Manifest#manifest.levels), + close_level(LevelIdx, Level, CloseEntryFun) + end, + lists:foreach(CloseLevelFun, lists:seq(0, Manifest#manifest.basement)). + save_manifest(Manifest, RootPath) -> FP = filepath(RootPath, Manifest#manifest.manifest_sqn, current_manifest), ManBin = term_to_binary(Manifest), @@ -176,24 +185,21 @@ key_lookup(Manifest, LevelIdx, Key) -> array:get(LevelIdx, Manifest#manifest.levels), Key) end. - + range_lookup(Manifest, LevelIdx, StartKey, EndKey) -> - Range = - case LevelIdx > Manifest#manifest.basement of - true -> - []; - false -> - range_lookup_level(LevelIdx, - array:get(LevelIdx, - Manifest#manifest.levels), - StartKey, - EndKey) - end, MakePointerFun = fun(M) -> {next, M, StartKey} end, - lists:map(MakePointerFun, Range). + range_lookup_int(Manifest, LevelIdx, StartKey, EndKey, MakePointerFun). + +merge_lookup(Manifest, LevelIdx, StartKey, EndKey) -> + MakePointerFun = + fun(M) -> + {next, M, all} + end, + range_lookup_int(Manifest, LevelIdx, StartKey, EndKey, MakePointerFun). + %% An algorithm for discovering which files to merge .... @@ -208,7 +214,7 @@ range_lookup(Manifest, LevelIdx, StartKey, EndKey) -> %% Hence, the initial implementation is to select files to merge at random mergefile_selector(Manifest, LevelIdx) -> Level = array:get(LevelIdx, Manifest#manifest.levels), - lists:nth(random:uniform(length(Level), Level)). + lists:nth(random:uniform(length(Level)), Level). add_snapshot(Manifest, Pid, Timeout) -> {MegaNow, SecNow, _} = os:timestamp(), @@ -299,12 +305,14 @@ load_level(_LevelIdx, Level, PidFun, SQNFun) -> LevelLoadFun = fun(ME, {L_Out, L_MaxSQN}) -> FN = ME#manifest_entry.filename, - {ok, P, _Keys} = PidFun(FN), + P = PidFun(FN), SQN = SQNFun(P), {[ME#manifest_entry{owner=P}|L_Out], max(SQN, L_MaxSQN)} end, lists:foldr(LevelLoadFun, {[], 0}, Level). +close_level(_LevelIdx, Level, CloseEntryFun) -> + lists:foreach(CloseEntryFun, Level). is_empty(_LevelIdx, []) -> true; @@ -337,7 +345,6 @@ remove_section(Level, StartKey, Length) -> Pre ++ Post. - key_lookup_level(_LevelIdx, [], _Key) -> false; key_lookup_level(LevelIdx, [Entry|Rest], Key) -> @@ -353,6 +360,20 @@ key_lookup_level(LevelIdx, [Entry|Rest], Key) -> key_lookup_level(LevelIdx, Rest, Key) end. +range_lookup_int(Manifest, LevelIdx, StartKey, EndKey, MakePointerFun) -> + Range = + case LevelIdx > Manifest#manifest.basement of + true -> + []; + false -> + range_lookup_level(LevelIdx, + array:get(LevelIdx, + Manifest#manifest.levels), + StartKey, + EndKey) + end, + lists:map(MakePointerFun, Range). + range_lookup_level(_LevelIdx, Level, QStartKey, QEndKey) -> BeforeFun = fun(M) -> @@ -392,13 +413,13 @@ filepath(RootPath, NewMSN, current_manifest) -> open_manifestfile(_RootPath, []) -> leveled_log:log("P0013", []), - {0, new_manifest()}; + new_manifest(); open_manifestfile(_RootPath, [0]) -> leveled_log:log("P0013", []), - {0, new_manifest()}; + new_manifest(); open_manifestfile(RootPath, [TopManSQN|Rest]) -> CurrManFile = filepath(RootPath, TopManSQN, current_manifest), - FileBin = file:read_file(CurrManFile), + {ok, FileBin} = file:read_file(CurrManFile), <> = FileBin, case erlang:crc32(BinaryOfTerm) of CRC -> diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 31d9861..ddf4935 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -38,7 +38,8 @@ clerk_new/2, clerk_prompt/1, clerk_push/2, - clerk_close/1 + clerk_close/1, + clerk_promptdeletions/2 ]). -include_lib("eunit/include/eunit.hrl"). @@ -47,7 +48,8 @@ -define(MIN_TIMEOUT, 200). -record(state, {owner :: pid(), - root_path :: string()}). + root_path :: string(), + pending_deletions = dict:new() :: dict:dict()}). %%%============================================================================ %%% API @@ -62,6 +64,9 @@ clerk_new(Owner, Manifest) -> clerk_prompt(Pid) -> gen_server:cast(Pid, prompt). +clerk_promptdeletions(Pid, ManifestSQN) -> + gen_server:cast(Pid, {prompt_deletions, ManifestSQN}). + clerk_push(Pid, Work) -> gen_server:cast(Pid, {push_work, Work}). @@ -83,8 +88,14 @@ handle_call(close, _From, State) -> handle_cast(prompt, State) -> handle_info(timeout, State); handle_cast({push_work, Work}, State) -> - handle_work(Work, State), - {noreply, State, ?MIN_TIMEOUT}. + {ManifestSQN, Deletions} = handle_work(Work, State), + PDs = dict:store(ManifestSQN, Deletions, State#state.pending_deletions), + {noreply, State#state{pending_deletions = PDs}, ?MAX_TIMEOUT}; +handle_cast({prompt_deletions, ManifestSQN}, State) -> + Deletions = dict:fetch(ManifestSQN, State#state.pending_deletions), + ok = notify_deletions(Deletions, State#state.owner), + UpdDeletions = dict:erase(ManifestSQN, State#state.pending_deletions), + {noreply, State#state{pending_deletions = UpdDeletions}, ?MIN_TIMEOUT}. handle_info(timeout, State) -> request_work(State), @@ -117,7 +128,7 @@ handle_work({SrcLevel, Manifest}, State) -> ok = leveled_manifest:save_manifest(UpdManifest, State#state.root_path), leveled_log:log_timer("PC018", [], SWSM), - ok = notify_deletions(EntriesToDelete, State#state.owner). + {leveled_manifest:get_manifest_sqn(UpdManifest), EntriesToDelete}. merge(SrcLevel, Manifest, RootPath) -> Src = leveled_manifest:mergefile_selector(Manifest, SrcLevel), @@ -130,21 +141,13 @@ merge(SrcLevel, Manifest, RootPath) -> leveled_log:log("PC008", [SrcLevel, Candidates]), case Candidates of 0 -> - %% If no overlapping candiates, manifest change only required - %% - %% TODO: need to think still about simply renaming when at - %% lower level leveled_log:log("PC009", [Src#manifest_entry.filename, SrcLevel + 1]), - Man0 = leveled_manifest:remove_manifest_entry(Manifest, + Man0 = leveled_manifest:switch_manifest_entry(Manifest, NewSQN, SrcLevel, Src), - Man1 = leveled_manifest:insert_manifest_entry(Man0, - NewSQN, - SrcLevel + 1, - Src), - {Man1, []}; + {Man0, []}; _ -> FilePath = leveled_penciller:filepath(RootPath, NewSQN, @@ -166,53 +169,59 @@ notify_deletions([Head|Tail], Penciller) -> perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN) -> leveled_log:log("PC010", [Src#manifest_entry.filename, NewSQN]), - SrcList = [{next, Src#manifest_entry.owner, all}], - SinkPointerList = leveled_manifest:pointer_convert(Manifest, SinkList), + SrcList = [{next, Src, all}], MaxSQN = leveled_sst:sst_getmaxsequencenumber(Src#manifest_entry.owner), SinkLevel = SrcLevel + 1, SinkBasement = leveled_manifest:is_basement(Manifest, SinkLevel), - Man0 = do_merge(SrcList, SinkPointerList, - SinkLevel, SinkBasement, - RootPath, NewSQN, MaxSQN, - 0, Manifest), - RemoveFun = - fun(Entry, AccMan) -> - leveled_manifest:remove_manifest_entry(AccMan, + Additions = do_merge(SrcList, SinkList, + SinkLevel, SinkBasement, + RootPath, NewSQN, MaxSQN, + []), + RevertPointerFun = + fun({next, ME, _SK}) -> + ME + end, + Man0 = leveled_manifest:insert_manifest_entry(Manifest, NewSQN, SinkLevel, - Entry) - end, - Man1 = lists:foldl(RemoveFun, Man0, SinkList), - Man2 = leveled_manifest:remove_manifest_entry(Man1, NewSQN, SrcLevel, Src), + Additions), + Man1 = leveled_manifest:remove_manifest_entry(Man0, + NewSQN, + SinkLevel, + lists:map(RevertPointerFun, + SinkList)), + Man2 = leveled_manifest:remove_manifest_entry(Man1, + NewSQN, + SrcLevel, + Src), {Man2, [Src|SinkList]}. -do_merge([], [], SinkLevel, _SinkB, _RP, NewSQN, _MaxSQN, Counter, Man0) -> - leveled_log:log("PC011", [NewSQN, SinkLevel, Counter]), - Man0; -do_merge(KL1, KL2, SinkLevel, SinkB, RP, NewSQN, MaxSQN, Counter, Man0) -> +do_merge([], [], SinkLevel, _SinkB, _RP, NewSQN, _MaxSQN, Additions) -> + leveled_log:log("PC011", [NewSQN, SinkLevel, length(Additions)]), + Additions; +do_merge(KL1, KL2, SinkLevel, SinkB, RP, NewSQN, MaxSQN, Additions) -> FileName = lists:flatten(io_lib:format(RP ++ "_~w_~w.sst", - [SinkLevel, Counter])), + [SinkLevel, length(Additions)])), leveled_log:log("PC012", [NewSQN, FileName, SinkB]), TS1 = os:timestamp(), case leveled_sst:sst_new(FileName, KL1, KL2, SinkB, SinkLevel, MaxSQN) of empty -> leveled_log:log("PC013", [FileName]), - Man0; + do_merge([], [], + SinkLevel, SinkB, + RP, NewSQN, MaxSQN, + Additions); {ok, Pid, Reply} -> {{KL1Rem, KL2Rem}, SmallestKey, HighestKey} = Reply, Entry = #manifest_entry{start_key=SmallestKey, end_key=HighestKey, owner=Pid, filename=FileName}, - Man1 = leveled_manifest:insert_manifest_entry(Man0, - NewSQN, - SinkLevel, - Entry), leveled_log:log_timer("PC015", [], TS1), do_merge(KL1Rem, KL2Rem, SinkLevel, SinkB, RP, NewSQN, MaxSQN, - Counter + 1, Man1) + Additions ++ [Entry]) end. @@ -294,7 +303,9 @@ merge_file_test() -> Man4 = leveled_manifest:insert_manifest_entry(Man3, 1, 2, E5), Man5 = leveled_manifest:insert_manifest_entry(Man4, 2, 1, E1), - {Man6, _Dels} = perform_merge(Man5, E1, [E2, E3, E4, E5], 1, "../test", 3), + PointerList = lists:map(fun(ME) -> {next, ME, all} end, + [E2, E3, E4, E5]), + {Man6, _Dels} = perform_merge(Man5, E1, PointerList, 1, "../test", 3), ?assertMatch(3, leveled_manifest:get_manifest_sqn(Man6)). diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index b340f7d..889bf50 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -461,6 +461,8 @@ handle_call(doom, _From, State) -> {stop, normal, {ok, [ManifestFP, FilesFP]}, State}. handle_cast({manifest_change, NewManifest}, State) -> + NewManSQN = leveled_manifest:get_manifest_sqn(NewManifest), + ok = leveled_pclerk:clerk_promptdeletions(State#state.clerk, NewManSQN), {noreply, State#state{manifest = NewManifest, work_ongoing=false}}; handle_cast({release_snapshot, Snapshot}, State) -> Manifest0 = leveled_manifest:release_snapshot(State#state.manifest, @@ -569,10 +571,11 @@ terminate(Reason, State) -> end, % Tidy shutdown of individual files - lists:foreach(fun({_FN, {Pid, _DSQN}}) -> - ok = leveled_sst:sst_close(Pid) - end, - leveled_manifest:dump_pidmap(State#state.manifest)), + EntryCloseFun = + fun(ME) -> + ok = leveled_sst:sst_close(ME#manifest_entry.owner) + end, + leveled_manifest:close_manifest(State#state.manifest, EntryCloseFun), leveled_log:log("P0011", []), ok. diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index ec8f8b9..712d74f 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -1175,8 +1175,8 @@ maybe_expand_pointer([{pointer, SSTPid, Slot, StartKey, all}|Tail]) -> expand_list_by_pointer({pointer, SSTPid, Slot, StartKey, all}, Tail, ?MERGE_SCANWIDTH); -maybe_expand_pointer([{next, SSTPid, StartKey}|Tail]) -> - expand_list_by_pointer({next, SSTPid, StartKey, all}, +maybe_expand_pointer([{next, ManEntry, StartKey}|Tail]) -> + expand_list_by_pointer({next, ManEntry, StartKey, all}, Tail, ?MERGE_SCANWIDTH); maybe_expand_pointer(List) -> @@ -1202,8 +1202,9 @@ expand_list_by_pointer({pointer, SSTPid, Slot, StartKey, EndKey}, Tail, Width) - {AccPointers, AccTail} = lists:foldl(FoldFun, InitAcc, Tail), ExpPointers = leveled_sst:sst_getslots(SSTPid, AccPointers), lists:append(ExpPointers, AccTail); -expand_list_by_pointer({next, SSTPid, StartKey, EndKey}, Tail, Width) -> - leveled_log:log("SST10", [SSTPid, is_pid(SSTPid)]), +expand_list_by_pointer({next, ManEntry, StartKey, EndKey}, Tail, Width) -> + SSTPid = ManEntry#manifest_entry.owner, + leveled_log:log("SST10", [SSTPid, is_process_alive(SSTPid)]), ExpPointer = leveled_sst:sst_getkvrange(SSTPid, StartKey, EndKey, Width), ExpPointer ++ Tail. From c32fd3fb4ca589888d8ef7b0f73e5ea029366ab6 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 17 Jan 2017 10:14:40 +0000 Subject: [PATCH 23/37] Change to use manifest_entry not straight PID in unit test --- src/leveled_sst.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index 712d74f..7af877a 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -1442,8 +1442,8 @@ merge_test() -> ?assertMatch(ExpFK2, FK2), ?assertMatch(ExpLK1, LK1), ?assertMatch(ExpLK2, LK2), - ML1 = [{next, P1, FK1}], - ML2 = [{next, P2, FK2}], + ML1 = [{next, #manifest_entry{owner = P1}, FK1}], + ML2 = [{next, #manifest_entry{owner = P2}, FK2}], {ok, P3, {{Rem1, Rem2}, FK3, LK3}} = sst_new("../test/level2_merge", ML1, ML2, From 59dc451e7066f69038af02af98236202304ffda4 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 17 Jan 2017 10:21:29 +0000 Subject: [PATCH 24/37] Address confusion over type in DeletionsList --- src/leveled_pclerk.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index ddf4935..8000dad 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -181,6 +181,7 @@ perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN) -> fun({next, ME, _SK}) -> ME end, + SinkManifestList = lists:map(RevertPointerFun, SinkList), Man0 = leveled_manifest:insert_manifest_entry(Manifest, NewSQN, SinkLevel, @@ -188,13 +189,12 @@ perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN) -> Man1 = leveled_manifest:remove_manifest_entry(Man0, NewSQN, SinkLevel, - lists:map(RevertPointerFun, - SinkList)), + SinkManifestList), Man2 = leveled_manifest:remove_manifest_entry(Man1, NewSQN, SrcLevel, Src), - {Man2, [Src|SinkList]}. + {Man2, [Src|SinkManifestList]}. do_merge([], [], SinkLevel, _SinkB, _RP, NewSQN, _MaxSQN, Additions) -> leveled_log:log("PC011", [NewSQN, SinkLevel, length(Additions)]), From 5adfb5c5ef8002f5d27332e3616368ad9c948b96 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 17 Jan 2017 10:32:15 +0000 Subject: [PATCH 25/37] Extra logging for troubleshooting --- src/leveled_log.erl | 2 ++ src/leveled_manifest.erl | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/leveled_log.erl b/src/leveled_log.erl index 525586d..9c62469 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -166,6 +166,8 @@ {info, "Notified clerk of manifest change"}}, {"PC018", {info, "Saved manifest file"}}, + {"PC019", + {info, "After ~s level ~w is ~w"}}, {"I0001", {info, "Unexpected failure to fetch value for Key=~w SQN=~w " diff --git a/src/leveled_manifest.erl b/src/leveled_manifest.erl index f27e421..8d5e192 100644 --- a/src/leveled_manifest.erl +++ b/src/leveled_manifest.erl @@ -125,6 +125,7 @@ insert_manifest_entry(Manifest, ManSQN, LevelIdx, Entry) -> Levels = Manifest#manifest.levels, Level = array:get(LevelIdx, Levels), UpdLevel = add_entry(LevelIdx, Level, Entry), + leveled_log:log("PC019", ["insert", LevelIdx, UpdLevel]), Basement = max(LevelIdx, Manifest#manifest.basement), Manifest#manifest{levels = array:set(LevelIdx, UpdLevel, Levels), basement = Basement, @@ -134,6 +135,7 @@ remove_manifest_entry(Manifest, ManSQN, LevelIdx, Entry) -> Levels = Manifest#manifest.levels, Level = array:get(LevelIdx, Levels), UpdLevel = remove_entry(LevelIdx, Level, Entry), + leveled_log:log("PC019", ["remove", LevelIdx, UpdLevel]), DelFun = fun(E, Acc) -> dict:store(E#manifest_entry.filename, ManSQN, Acc) From ec08d1ab9784970d82db7e85a0596ab01fc8f123 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 17 Jan 2017 10:37:46 +0000 Subject: [PATCH 26/37] Must remove before we insert - cannot safely if overlapping with insertions --- src/leveled_pclerk.erl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 8000dad..983327a 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -182,14 +182,15 @@ perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN) -> ME end, SinkManifestList = lists:map(RevertPointerFun, SinkList), - Man0 = leveled_manifest:insert_manifest_entry(Manifest, - NewSQN, - SinkLevel, - Additions), - Man1 = leveled_manifest:remove_manifest_entry(Man0, + Man0 = leveled_manifest:remove_manifest_entry(Manifest, NewSQN, SinkLevel, SinkManifestList), + Man1 = leveled_manifest:insert_manifest_entry(Man0, + NewSQN, + SinkLevel, + Additions), + Man2 = leveled_manifest:remove_manifest_entry(Man1, NewSQN, SrcLevel, From 5076187545df05f28f22905c456c470bd57489be Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 17 Jan 2017 10:39:31 +0000 Subject: [PATCH 27/37] Reduce log level of troubleshooting log --- src/leveled_log.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leveled_log.erl b/src/leveled_log.erl index 9c62469..5250a7e 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -154,7 +154,7 @@ {"PC011", {info, "Merge completed with MSN=~w to Level=~w and FileCounter=~w"}}, {"PC012", - {info, "File to be created as part of MSN=~w Filename=~s " + {debug, "File to be created as part of MSN=~w Filename=~s " ++ "IsBasement=~w"}}, {"PC013", {warn, "Merge resulted in empty file ~s"}}, From 14ebf94e568aa8b7e6640b4388851c3b9e39c2fc Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 17 Jan 2017 10:53:13 +0000 Subject: [PATCH 28/37] Correct botched manifest entry removal --- src/leveled_log.erl | 4 ++-- src/leveled_manifest.erl | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/leveled_log.erl b/src/leveled_log.erl index 5250a7e..d2c8a3b 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -154,7 +154,7 @@ {"PC011", {info, "Merge completed with MSN=~w to Level=~w and FileCounter=~w"}}, {"PC012", - {debug, "File to be created as part of MSN=~w Filename=~s " + {info, "File to be created as part of MSN=~w Filename=~s " ++ "IsBasement=~w"}}, {"PC013", {warn, "Merge resulted in empty file ~s"}}, @@ -167,7 +167,7 @@ {"PC018", {info, "Saved manifest file"}}, {"PC019", - {info, "After ~s level ~w is ~w"}}, + {debug, "After ~s level ~w is ~w"}}, {"I0001", {info, "Unexpected failure to fetch value for Key=~w SQN=~w " diff --git a/src/leveled_manifest.erl b/src/leveled_manifest.erl index 8d5e192..7150c37 100644 --- a/src/leveled_manifest.erl +++ b/src/leveled_manifest.erl @@ -332,18 +332,18 @@ add_entry(_LevelIdx, Level, Entry) -> remove_entry(_LevelIdx, Level, Entries) when is_list(Entries) -> % We're assuming we're removing a sorted sublist RemLength = length(Entries), - RemStart = lists:nth(1, Entries), - remove_section(Level, RemStart#manifest_entry.start_key, RemLength - 1); + [RemStart|_Tail] = Entries, + remove_section(Level, RemStart#manifest_entry.start_key, RemLength); remove_entry(_LevelIdx, Level, Entry) -> - remove_section(Level, Entry#manifest_entry.start_key, 0). + remove_section(Level, Entry#manifest_entry.start_key, 1). -remove_section(Level, StartKey, Length) -> +remove_section(Level, SectionStartKey, SectionLength) -> PredFun = fun(E) -> - E#manifest_entry.start_key < StartKey + E#manifest_entry.start_key < SectionStartKey end, {Pre, Rest} = lists:splitwith(PredFun, Level), - Post = lists:nthtail(length(Rest) - Length, Rest), + Post = lists:nthtail(SectionLength, Rest), Pre ++ Post. From c99c50ce6e1a99c08712d2497eca223f01f6df31 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 17 Jan 2017 11:18:58 +0000 Subject: [PATCH 29/37] Fix-up message exchange on confirm delete --- src/leveled_penciller.erl | 16 +++++++--------- src/leveled_sst.erl | 3 ++- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 889bf50..7eb6b5d 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -181,7 +181,7 @@ pcl_workforclerk/1, pcl_manifestchange/2, pcl_confirml0complete/4, - pcl_confirmdelete/2, + pcl_confirmdelete/3, pcl_close/1, pcl_doom/1, pcl_registersnapshot/2, @@ -297,8 +297,8 @@ pcl_manifestchange(Pid, Manifest) -> pcl_confirml0complete(Pid, FN, StartKey, EndKey) -> gen_server:cast(Pid, {levelzero_complete, FN, StartKey, EndKey}). -pcl_confirmdelete(Pid, FileName) -> - gen_server:cast(Pid, {confirm_delete, FileName}). +pcl_confirmdelete(Pid, FileName, FilePid) -> + gen_server:cast(Pid, {confirm_delete, FileName, FilePid}). pcl_getstartupsequencenumber(Pid) -> gen_server:call(Pid, get_startup_sqn, infinity). @@ -469,20 +469,18 @@ handle_cast({release_snapshot, Snapshot}, State) -> Snapshot), leveled_log:log("P0003", [Snapshot]), {noreply, State#state{manifest=Manifest0}}; -handle_cast({confirm_delete, Filename}, State=#state{is_snapshot=Snap}) +handle_cast({confirm_delete, Filename, FilePid}, State=#state{is_snapshot=Snap}) when Snap == false -> case State#state.work_ongoing of false -> R2D = leveled_manifest:ready_to_delete(State#state.manifest, Filename), case R2D of - {true, Pid} -> + {true, M0} -> leveled_log:log("P0005", [Filename]), - ok = leveled_sst:sst_deleteconfirmed(Pid), - M0 = leveled_manifest:delete_confirmed(State#state.manifest, - Filename), + ok = leveled_sst:sst_deleteconfirmed(FilePid), {noreply, State#state{manifest=M0}}; - {false, _Pid} -> + {false, _M0} -> {noreply, State} end; true -> diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index 7af877a..c9102d1 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -363,7 +363,8 @@ delete_pending(close, _From, State) -> delete_pending(timeout, State) -> ok = leveled_penciller:pcl_confirmdelete(State#state.penciller, - State#state.filename), + State#state.filename, + self()), {next_state, delete_pending, State, ?DELETE_TIMEOUT}; delete_pending(close, State) -> leveled_log:log("SST07", [State#state.filename]), From 2e734ff501b08aa9aa9901e38815df0d6c95dfb7 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 17 Jan 2017 14:02:21 +0000 Subject: [PATCH 30/37] Corrected snapshot releases --- src/leveled_manifest.erl | 64 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/src/leveled_manifest.erl b/src/leveled_manifest.erl index 7150c37..c668f63 100644 --- a/src/leveled_manifest.erl +++ b/src/leveled_manifest.erl @@ -239,7 +239,7 @@ release_snapshot(Manifest, Pid) -> fun({P, SQN, TS}, {Acc, MinSQN}) -> case P of Pid -> - {Acc, min(SQN, MinSQN)}; + {Acc, MinSQN}; _ -> {[{P, SQN, TS}|Acc], min(SQN, MinSQN)} end @@ -259,13 +259,17 @@ release_snapshot(Manifest, Pid) -> ready_to_delete(Manifest, Filename) -> ChangeSQN = dict:fetch(Filename, Manifest#manifest.pending_deletes), - case Manifest#manifest.min_snapshot_sqn >= ChangeSQN of - true -> + case Manifest#manifest.min_snapshot_sqn of + 0 -> + % no shapshots + PDs = dict:erase(Filename, Manifest#manifest.pending_deletes), + {true, Manifest#manifest{pending_deletes = PDs}}; + N when N >= ChangeSQN -> % Every snapshot is looking at a version of history after this % was removed PDs = dict:erase(Filename, Manifest#manifest.pending_deletes), {true, Manifest#manifest{pending_deletes = PDs}}; - false -> + _N -> {false, Manifest} end. @@ -622,4 +626,56 @@ levelzero_present_test() -> Man1 = insert_manifest_entry(Man0, 1, 0, E0), ?assertMatch(true, levelzero_present(Man1)). +snapshot_release_test() -> + Man6 = element(7, initial_setup()), + E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, + end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, + filename="Z1", + owner="pid_z1"}, + E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"}, + end_key={o, "Bucket1", "K71", null}, + filename="Z2", + owner="pid_z2"}, + E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null}, + end_key={o, "Bucket1", "K993", null}, + filename="Z3", + owner="pid_z3"}, + + Man7 = add_snapshot(Man6, "pid_a1", 3600), + Man8 = remove_manifest_entry(Man7, 2, 1, E1), + Man9 = add_snapshot(Man8, "pid_a2", 3600), + Man10 = remove_manifest_entry(Man9, 3, 1, E2), + Man11 = add_snapshot(Man10, "pid_a3", 3600), + Man12 = remove_manifest_entry(Man11, 4, 1, E3), + Man13 = add_snapshot(Man12, "pid_a4", 3600), + + ?assertMatch(false, element(1, ready_to_delete(Man8, "Z1"))), + ?assertMatch(false, element(1, ready_to_delete(Man10, "Z2"))), + ?assertMatch(false, element(1, ready_to_delete(Man12, "Z3"))), + + Man14 = release_snapshot(Man13, "pid_a1"), + ?assertMatch(false, element(1, ready_to_delete(Man14, "Z2"))), + ?assertMatch(false, element(1, ready_to_delete(Man14, "Z3"))), + {Bool14, Man15} = ready_to_delete(Man14, "Z1"), + ?assertMatch(true, Bool14), + + %This doesn't change anything - released snaphsot not the min + Man16 = release_snapshot(Man15, "pid_a4"), + ?assertMatch(false, element(1, ready_to_delete(Man16, "Z2"))), + ?assertMatch(false, element(1, ready_to_delete(Man16, "Z3"))), + + Man17 = release_snapshot(Man16, "pid_a2"), + ?assertMatch(false, element(1, ready_to_delete(Man17, "Z3"))), + {Bool17, Man18} = ready_to_delete(Man17, "Z2"), + ?assertMatch(true, Bool17), + + Man19 = release_snapshot(Man18, "pid_a3"), + + io:format("MinSnapSQN ~w~n", [Man19#manifest.min_snapshot_sqn]), + + {Bool19, _Man20} = ready_to_delete(Man19, "Z3"), + ?assertMatch(true, Bool19). + + + -endif. \ No newline at end of file From 7086717765840a6fa4586ff00bc117004669e6fd Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 17 Jan 2017 14:11:50 +0000 Subject: [PATCH 31/37] Rename to pmanifest There may be a seperate imanifest in the future --- src/leveled_pclerk.erl | 34 ++++++++--------- src/leveled_penciller.erl | 38 +++++++++---------- ...led_manifest.erl => leveled_pmanifest.erl} | 2 +- 3 files changed, 37 insertions(+), 37 deletions(-) rename src/{leveled_manifest.erl => leveled_pmanifest.erl} (99%) diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 983327a..1bc776b 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -125,15 +125,15 @@ handle_work({SrcLevel, Manifest}, State) -> UpdManifest), leveled_log:log_timer("PC017", [], SWMC), SWSM = os:timestamp(), - ok = leveled_manifest:save_manifest(UpdManifest, + ok = leveled_pmanifest:save_manifest(UpdManifest, State#state.root_path), leveled_log:log_timer("PC018", [], SWSM), - {leveled_manifest:get_manifest_sqn(UpdManifest), EntriesToDelete}. + {leveled_pmanifest:get_manifest_sqn(UpdManifest), EntriesToDelete}. merge(SrcLevel, Manifest, RootPath) -> - Src = leveled_manifest:mergefile_selector(Manifest, SrcLevel), - NewSQN = leveled_manifest:get_manifest_sqn(Manifest) + 1, - SinkList = leveled_manifest:merge_lookup(Manifest, + Src = leveled_pmanifest:mergefile_selector(Manifest, SrcLevel), + NewSQN = leveled_pmanifest:get_manifest_sqn(Manifest) + 1, + SinkList = leveled_pmanifest:merge_lookup(Manifest, SrcLevel + 1, Src#manifest_entry.start_key, Src#manifest_entry.end_key), @@ -143,7 +143,7 @@ merge(SrcLevel, Manifest, RootPath) -> 0 -> leveled_log:log("PC009", [Src#manifest_entry.filename, SrcLevel + 1]), - Man0 = leveled_manifest:switch_manifest_entry(Manifest, + Man0 = leveled_pmanifest:switch_manifest_entry(Manifest, NewSQN, SrcLevel, Src), @@ -172,7 +172,7 @@ perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN) -> SrcList = [{next, Src, all}], MaxSQN = leveled_sst:sst_getmaxsequencenumber(Src#manifest_entry.owner), SinkLevel = SrcLevel + 1, - SinkBasement = leveled_manifest:is_basement(Manifest, SinkLevel), + SinkBasement = leveled_pmanifest:is_basement(Manifest, SinkLevel), Additions = do_merge(SrcList, SinkList, SinkLevel, SinkBasement, RootPath, NewSQN, MaxSQN, @@ -182,16 +182,16 @@ perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN) -> ME end, SinkManifestList = lists:map(RevertPointerFun, SinkList), - Man0 = leveled_manifest:remove_manifest_entry(Manifest, + Man0 = leveled_pmanifest:remove_manifest_entry(Manifest, NewSQN, SinkLevel, SinkManifestList), - Man1 = leveled_manifest:insert_manifest_entry(Man0, + Man1 = leveled_pmanifest:insert_manifest_entry(Man0, NewSQN, SinkLevel, Additions), - Man2 = leveled_manifest:remove_manifest_entry(Man1, + Man2 = leveled_pmanifest:remove_manifest_entry(Man1, NewSQN, SrcLevel, Src), @@ -297,18 +297,18 @@ merge_file_test() -> end_key = lists:last(KL4_L2), start_key = lists:nth(1, KL4_L2)}, - Man0 = leveled_manifest:new_manifest(), - Man1 = leveled_manifest:insert_manifest_entry(Man0, 1, 2, E2), - Man2 = leveled_manifest:insert_manifest_entry(Man1, 1, 2, E3), - Man3 = leveled_manifest:insert_manifest_entry(Man2, 1, 2, E4), - Man4 = leveled_manifest:insert_manifest_entry(Man3, 1, 2, E5), - Man5 = leveled_manifest:insert_manifest_entry(Man4, 2, 1, E1), + Man0 = leveled_pmanifest:new_manifest(), + Man1 = leveled_pmanifest:insert_manifest_entry(Man0, 1, 2, E2), + Man2 = leveled_pmanifest:insert_manifest_entry(Man1, 1, 2, E3), + Man3 = leveled_pmanifest:insert_manifest_entry(Man2, 1, 2, E4), + Man4 = leveled_pmanifest:insert_manifest_entry(Man3, 1, 2, E5), + Man5 = leveled_pmanifest:insert_manifest_entry(Man4, 2, 1, E1), PointerList = lists:map(fun(ME) -> {next, ME, all} end, [E2, E3, E4, E5]), {Man6, _Dels} = perform_merge(Man5, E1, PointerList, 1, "../test", 3), - ?assertMatch(3, leveled_manifest:get_manifest_sqn(Man6)). + ?assertMatch(3, leveled_pmanifest:get_manifest_sqn(Man6)). coverage_cheat_test() -> {ok, _State1} = code_change(null, #state{}, null). diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 7eb6b5d..9a6daf3 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -329,7 +329,7 @@ init([PCLopts]) -> {undefined, true} -> SrcPenciller = PCLopts#penciller_options.source_penciller, {ok, State} = pcl_registersnapshot(SrcPenciller, self()), - ManifestClone = leveled_manifest:copy_manifest(State#state.manifest), + ManifestClone = leveled_pmanifest:copy_manifest(State#state.manifest), leveled_log:log("P0001", [self()]), {ok, State#state{is_snapshot=true, source_penciller=SrcPenciller, @@ -411,7 +411,7 @@ handle_call({fetch_keys, StartKey, EndKey, AccFun, InitAcc, MaxKeys}, SetupFoldFun = fun(Level, Acc) -> - Pointers = leveled_manifest:range_lookup(State#state.manifest, + Pointers = leveled_pmanifest:range_lookup(State#state.manifest, Level, StartKey, EndKey), @@ -431,7 +431,7 @@ handle_call({fetch_keys, StartKey, EndKey, AccFun, InitAcc, MaxKeys}, handle_call(get_startup_sqn, _From, State) -> {reply, State#state.persisted_sqn, State}; handle_call({register_snapshot, Snapshot}, _From, State) -> - Manifest0 = leveled_manifest:add_snapshot(State#state.manifest, + Manifest0 = leveled_pmanifest:add_snapshot(State#state.manifest, Snapshot, ?SNAPSHOT_TIMEOUT), {reply, {ok, State}, State#state{manifest = Manifest0}}; @@ -461,11 +461,11 @@ handle_call(doom, _From, State) -> {stop, normal, {ok, [ManifestFP, FilesFP]}, State}. handle_cast({manifest_change, NewManifest}, State) -> - NewManSQN = leveled_manifest:get_manifest_sqn(NewManifest), + NewManSQN = leveled_pmanifest:get_manifest_sqn(NewManifest), ok = leveled_pclerk:clerk_promptdeletions(State#state.clerk, NewManSQN), {noreply, State#state{manifest = NewManifest, work_ongoing=false}}; handle_cast({release_snapshot, Snapshot}, State) -> - Manifest0 = leveled_manifest:release_snapshot(State#state.manifest, + Manifest0 = leveled_pmanifest:release_snapshot(State#state.manifest, Snapshot), leveled_log:log("P0003", [Snapshot]), {noreply, State#state{manifest=Manifest0}}; @@ -473,7 +473,7 @@ handle_cast({confirm_delete, Filename, FilePid}, State=#state{is_snapshot=Snap}) when Snap == false -> case State#state.work_ongoing of false -> - R2D = leveled_manifest:ready_to_delete(State#state.manifest, + R2D = leveled_pmanifest:ready_to_delete(State#state.manifest, Filename), case R2D of {true, M0} -> @@ -495,8 +495,8 @@ handle_cast({levelzero_complete, FN, StartKey, EndKey}, State) -> end_key=EndKey, owner=State#state.levelzero_constructor, filename=FN}, - ManifestSQN = leveled_manifest:get_manifest_sqn(State#state.manifest) + 1, - UpdMan = leveled_manifest:insert_manifest_entry(State#state.manifest, + ManifestSQN = leveled_pmanifest:get_manifest_sqn(State#state.manifest) + 1, + UpdMan = leveled_pmanifest:insert_manifest_entry(State#state.manifest, ManifestSQN, 0, ManEntry), @@ -515,7 +515,7 @@ handle_cast(work_for_clerk, State) -> true -> {noreply, State}; false -> - {WL, WC} = leveled_manifest:check_for_work(State#state.manifest, + {WL, WC} = leveled_pmanifest:check_for_work(State#state.manifest, ?LEVEL_SCALEFACTOR), case WC of 0 -> @@ -558,7 +558,7 @@ terminate(Reason, State) -> ok = leveled_pclerk:clerk_close(State#state.clerk), leveled_log:log("P0008", [Reason]), - L0_Present = leveled_manifest:key_lookup(State#state.manifest, 0, all), + L0_Present = leveled_pmanifest:key_lookup(State#state.manifest, 0, all), L0_Left = State#state.levelzero_size > 0, case {State#state.levelzero_pending, L0_Present, L0_Left} of {false, false, true} -> @@ -573,7 +573,7 @@ terminate(Reason, State) -> fun(ME) -> ok = leveled_sst:sst_close(ME#manifest_entry.owner) end, - leveled_manifest:close_manifest(State#state.manifest, EntryCloseFun), + leveled_pmanifest:close_manifest(State#state.manifest, EntryCloseFun), leveled_log:log("P0011", []), ok. @@ -610,18 +610,18 @@ start_from_file(PCLopts) -> levelzero_index=leveled_pmem:new_index()}, %% Open manifest - Manifest0 = leveled_manifest:open_manifest(RootPath), + Manifest0 = leveled_pmanifest:open_manifest(RootPath), OpenFun = fun(FN) -> {ok, Pid, {_FK, _LK}} = leveled_sst:sst_open(FN), Pid end, SQNFun = fun leveled_sst:sst_getmaxsequencenumber/1, - {MaxSQN, Manifest1} = leveled_manifest:load_manifest(Manifest0, + {MaxSQN, Manifest1} = leveled_pmanifest:load_manifest(Manifest0, OpenFun, SQNFun), leveled_log:log("P0014", [MaxSQN]), - ManSQN = leveled_manifest:get_manifest_sqn(Manifest1), + ManSQN = leveled_pmanifest:get_manifest_sqn(Manifest1), leveled_log:log("P0035", [ManSQN]), %% Find any L0 files L0FN = filepath(RootPath, ManSQN + 1, new_merge_files) ++ "_0_0.sst", @@ -636,7 +636,7 @@ start_from_file(PCLopts) -> end_key = L0EndKey, filename = L0FN, owner = L0Pid}, - Manifest2 = leveled_manifest:insert_manifest_entry(Manifest1, + Manifest2 = leveled_pmanifest:insert_manifest_entry(Manifest1, ManSQN + 1, 0, L0Entry), @@ -675,7 +675,7 @@ update_levelzero(L0Size, {PushedTree, PushedIdx, MinSQN, MaxSQN}, ledger_sqn=UpdMaxSQN}, CacheTooBig = NewL0Size > State#state.levelzero_maxcachesize, CacheMuchTooBig = NewL0Size > ?SUPER_MAX_TABLE_SIZE, - L0Free = not leveled_manifest:levelzero_present(State#state.manifest), + L0Free = not leveled_pmanifest:levelzero_present(State#state.manifest), RandomFactor = case State#state.levelzero_cointoss of true -> @@ -735,7 +735,7 @@ roll_memory(State, true) -> Constructor. levelzero_filename(State) -> - ManSQN = leveled_manifest:get_manifest_sqn(State#state.manifest) + 1, + ManSQN = leveled_pmanifest:get_manifest_sqn(State#state.manifest) + 1, FileName = State#state.root_path ++ "/" ++ ?FILES_FP ++ "/" ++ integer_to_list(ManSQN) ++ "_0_0", @@ -770,7 +770,7 @@ fetch_mem(Key, Hash, Manifest, L0Cache, L0Index) -> fetch(_Key, _Hash, _Manifest, ?MAX_LEVELS + 1, _FetchFun) -> {not_present, basement}; fetch(Key, Hash, Manifest, Level, FetchFun) -> - case leveled_manifest:key_lookup(Manifest, Level, Key) of + case leveled_pmanifest:key_lookup(Manifest, Level, Key) of false -> fetch(Key, Hash, Manifest, Level + 1, FetchFun); FP -> @@ -1053,7 +1053,7 @@ generate_randomkeys(Count, SQN, Acc) -> clean_testdir(RootPath) -> - clean_subdir(leveled_manifest:filepath(RootPath, manifest)), + clean_subdir(leveled_pmanifest:filepath(RootPath, manifest)), clean_subdir(filepath(RootPath, files)). clean_subdir(DirPath) -> diff --git a/src/leveled_manifest.erl b/src/leveled_pmanifest.erl similarity index 99% rename from src/leveled_manifest.erl rename to src/leveled_pmanifest.erl index c668f63..e16c137 100644 --- a/src/leveled_manifest.erl +++ b/src/leveled_pmanifest.erl @@ -5,7 +5,7 @@ %% --module(leveled_manifest). +-module(leveled_pmanifest). -include("include/leveled.hrl"). From 850a524ad3fdb788d1e09d6209398bfd4dfc2277 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 17 Jan 2017 16:54:54 +0000 Subject: [PATCH 32/37] Close Pending Deletes on shutdown Pending deletes are not in the Manifest, and so were not being closed on shutdown --- src/leveled_pmanifest.erl | 18 +++++++++++++----- test/end_to_end/basic_SUITE.erl | 12 ++++++------ 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/leveled_pmanifest.erl b/src/leveled_pmanifest.erl index e16c137..9fb015d 100644 --- a/src/leveled_pmanifest.erl +++ b/src/leveled_pmanifest.erl @@ -52,8 +52,8 @@ % The smallest snapshot manifest SQN in the snapshot % list pending_deletes :: dict:dict(), - % a dictionary mapping keys (filenames) to SQN when - % the deletion was made + % a dictionary mapping keys (filenames) to SQN when the + % deletion was made, and the original Manifest Entry basement :: integer() % Currently the lowest level (the largest number) }). @@ -113,7 +113,13 @@ close_manifest(Manifest, CloseEntryFun) -> Level = array:get(LevelIdx, Manifest#manifest.levels), close_level(LevelIdx, Level, CloseEntryFun) end, - lists:foreach(CloseLevelFun, lists:seq(0, Manifest#manifest.basement)). + lists:foreach(CloseLevelFun, lists:seq(0, Manifest#manifest.basement)), + + ClosePDFun = + fun({_FN, {_SQN, ME}}) -> + CloseEntryFun(ME) + end, + lists:foreach(ClosePDFun, dict:to_list(Manifest#manifest.pending_deletes)). save_manifest(Manifest, RootPath) -> FP = filepath(RootPath, Manifest#manifest.manifest_sqn, current_manifest), @@ -138,7 +144,9 @@ remove_manifest_entry(Manifest, ManSQN, LevelIdx, Entry) -> leveled_log:log("PC019", ["remove", LevelIdx, UpdLevel]), DelFun = fun(E, Acc) -> - dict:store(E#manifest_entry.filename, ManSQN, Acc) + dict:store(E#manifest_entry.filename, + {ManSQN, E}, + Acc) end, Entries = case is_list(Entry) of @@ -258,7 +266,7 @@ release_snapshot(Manifest, Pid) -> end. ready_to_delete(Manifest, Filename) -> - ChangeSQN = dict:fetch(Filename, Manifest#manifest.pending_deletes), + {ChangeSQN, _ME} = dict:fetch(Filename, Manifest#manifest.pending_deletes), case Manifest#manifest.min_snapshot_sqn of 0 -> % no shapshots diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index d6b743a..4bc25d9 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -12,12 +12,12 @@ ]). all() -> [ - simple_put_fetch_head_delete, - many_put_fetch_head, - journal_compaction, - fetchput_snapshot, - load_and_count, - load_and_count_withdelete, + % simple_put_fetch_head_delete, + % many_put_fetch_head, + % journal_compaction, + % fetchput_snapshot, + % load_and_count, + % load_and_count_withdelete, space_clear_ondelete ]. From c098eca06eaffc1cd880fbad2eead5a469ffc4d5 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 17 Jan 2017 16:55:35 +0000 Subject: [PATCH 33/37] Put tests back in --- test/end_to_end/basic_SUITE.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 4bc25d9..d6b743a 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -12,12 +12,12 @@ ]). all() -> [ - % simple_put_fetch_head_delete, - % many_put_fetch_head, - % journal_compaction, - % fetchput_snapshot, - % load_and_count, - % load_and_count_withdelete, + simple_put_fetch_head_delete, + many_put_fetch_head, + journal_compaction, + fetchput_snapshot, + load_and_count, + load_and_count_withdelete, space_clear_ondelete ]. From 629888659258e9dc2ad9e41b3d14eaee3bb389dd Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 17 Jan 2017 17:15:04 +0000 Subject: [PATCH 34/37] Empty non-persisted parts of record on save Some of the manifest should not be preserved between restarts - so empty these parts on save --- src/leveled_pmanifest.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/leveled_pmanifest.erl b/src/leveled_pmanifest.erl index 9fb015d..adde690 100644 --- a/src/leveled_pmanifest.erl +++ b/src/leveled_pmanifest.erl @@ -123,7 +123,9 @@ close_manifest(Manifest, CloseEntryFun) -> save_manifest(Manifest, RootPath) -> FP = filepath(RootPath, Manifest#manifest.manifest_sqn, current_manifest), - ManBin = term_to_binary(Manifest), + ManBin = term_to_binary(Manifest#manifest{snapshots = [], + pending_deletes = dict:new(), + min_snapshot_sqn = 0}), CRC = erlang:crc32(ManBin), ok = file:write_file(FP, <>). From 5345242e277eb658247cd472818328e96ec582cc Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 17 Jan 2017 19:00:30 +0000 Subject: [PATCH 35/37] Take out type definition for OTP 16 --- src/leveled_pmanifest.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leveled_pmanifest.erl b/src/leveled_pmanifest.erl index adde690..e33d58d 100644 --- a/src/leveled_pmanifest.erl +++ b/src/leveled_pmanifest.erl @@ -51,7 +51,7 @@ min_snapshot_sqn = 0 :: integer(), % The smallest snapshot manifest SQN in the snapshot % list - pending_deletes :: dict:dict(), + pending_deletes, % OTP16 does not like defining type % a dictionary mapping keys (filenames) to SQN when the % deletion was made, and the original Manifest Entry basement :: integer() From 59c0c06b6027707d64ec71a20a98091030779700 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 17 Jan 2017 19:01:59 +0000 Subject: [PATCH 36/37] OTP16 type again --- src/leveled_pclerk.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 1bc776b..66b7c74 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -49,7 +49,8 @@ -record(state, {owner :: pid(), root_path :: string(), - pending_deletions = dict:new() :: dict:dict()}). + pending_deletions = dict:new() % OTP 16 does not like type + }). %%%============================================================================ %%% API From f97e02975aa77f0af465e13b334db4fe46a950fa Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 17 Jan 2017 21:09:42 +0000 Subject: [PATCH 37/37] Add comments --- src/leveled_pmanifest.erl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/leveled_pmanifest.erl b/src/leveled_pmanifest.erl index e33d58d..9fa50ef 100644 --- a/src/leveled_pmanifest.erl +++ b/src/leveled_pmanifest.erl @@ -3,6 +3,16 @@ %% The manifest is an ordered set of files for each level to be used to find %% which file is relevant for a given key or range lookup at a given level. %% +%% This implementation is incomplete, in that it just uses a plain list at +%% each level. This is fine for short-lived volume tests, but as the deeper +%% levels are used there will be an exponential penalty. +%% +%% The originial intention was to swap out this implementation for a +%% multi-version ETS table - but that became complex. So one of two changes +%% are pending: +%% - Use a single version ES cache for lower levels (and not allow snapshots to +%% access the cache) +%% - Use a skiplist like enhanced list at lower levels. -module(leveled_pmanifest).