From 76bdd833461d8d2d68c8601f706e442e896d09d2 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 14 Jan 2017 16:36:05 +0000 Subject: [PATCH] Manifest refactor - STILL BROKEN Some working tests now, but sitll broken --- src/leveled_codec.erl | 2 + src/leveled_manifest.erl | 243 +++++++++++++++++++++++++++----------- src/leveled_pclerk.erl | 4 +- src/leveled_penciller.erl | 21 ++-- 4 files changed, 186 insertions(+), 84 deletions(-) diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index 10ffcc7..6dbbff4 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -282,6 +282,8 @@ turn_to_string(Item) -> % Compare a key against a query key, only comparing elements that are non-null % in the Query key. This is used for comparing against end keys in queries. +endkey_passed(all, _) -> + false; endkey_passed({EK1, null, null, null}, {CK1, _, _, _}) -> EK1 < CK1; endkey_passed({EK1, EK2, null, null}, {CK1, CK2, _, _}) -> diff --git a/src/leveled_manifest.erl b/src/leveled_manifest.erl index f7efed9..8da2512 100644 --- a/src/leveled_manifest.erl +++ b/src/leveled_manifest.erl @@ -52,12 +52,15 @@ pointer_convert/2 ]). +-export([ + filepath/2 + ]). + -include_lib("eunit/include/eunit.hrl"). -define(MANIFEST_FILEX, "man"). -define(MANIFEST_FP, "ledger_manifest"). -define(MAX_LEVELS, 8). --define(END_KEY, {null, null, null, null}). -record(manifest, {table, % A Multi-Version ETS table for lookup @@ -71,7 +74,7 @@ % An array of level counts to speed up compation work assessment snapshots :: list(), % A list of snaphots (i.e. clones) - delete_sqn :: integer()|infinity + delete_sqn :: integer() % The lowest SQN of any clone }). @@ -80,7 +83,7 @@ %%%============================================================================ new_manifest() -> - Table = ets:new(manifest, [ordered_set]), + Table = ets:new(manifest, [ordered_set, public]), new_manifest(Table). open_manifest(RootPath) -> @@ -101,9 +104,8 @@ open_manifest(RootPath) -> ValidManSQNs = lists:reverse(lists:sort(lists:foldl(ExtractSQNFun, [], Filenames))), - {ManSQN, Table} = open_manifestfile(RootPath, ValidManSQNs), - Manifest = new_manifest(Table), - Manifest#manifest{manifest_sqn = ManSQN}. + {ManSQN, Manifest} = open_manifestfile(RootPath, ValidManSQNs), + Manifest#manifest{manifest_sqn = ManSQN, delete_sqn = ManSQN}. copy_manifest(Manifest) -> % Copy the manifest ensuring anything only the master process should care @@ -162,7 +164,7 @@ insert_manifest_entry(Manifest, ManSQN, Level, Entry) -> remove_manifest_entry(Manifest, ManSQN, Level, Entry) -> Key = {Level, Entry#manifest_entry.end_key, Entry#manifest_entry.filename}, - [{Key, Value0}] = ets:lookup(Manifest, Key), + [{Key, Value0}] = ets:lookup(Manifest#manifest.table, Key), {StartKey, {active, ActiveSQN}, {tomb, infinity}} = Value0, Value1 = {StartKey, {active, ActiveSQN}, {tomb, ManSQN}}, true = ets:insert(Manifest#manifest.table, {Key, Value1}), @@ -205,7 +207,13 @@ key_lookup(Manifest, Level, Key) -> range_lookup(Manifest, Level, StartKey, EndKey) -> MapFun = fun({{_Level, _LastKey, FN}, FirstKey}) -> - {next, dict:fetch(FN, Manifest#manifest.pidmap), FirstKey} + {Pid, _SQN} = dict:fetch(FN, Manifest#manifest.pidmap), + case FirstKey < StartKey of + true -> + {next, Pid, StartKey}; + false -> + {next, Pid, FirstKey} + end end, range_lookup(Manifest, Level, StartKey, EndKey, MapFun). @@ -245,7 +253,7 @@ mergefile_selector(Manifest, Level) -> Level, {all, 0}, all, - ?END_KEY, + all, [], Manifest#manifest.manifest_sqn), {{Level, LastKey, FN}, @@ -267,14 +275,15 @@ release_snapshot(Manifest, Pid) -> fun({P, SQN, TS}, {Acc, MinSQN}) -> case P of Pid -> - Acc; + {Acc, MinSQN}; _ -> {[{P, SQN, TS}|Acc], min(SQN, MinSQN)} end end, - {SnapList0, DeleteSQN} = lists:foldl(FilterFun, - {[], infinity}, - Manifest#manifest.snapshots), + {SnapList0, + DeleteSQN} = lists:foldl(FilterFun, + {[], Manifest#manifest.manifest_sqn}, + Manifest#manifest.snapshots), leveled_log:log("P0004", [SnapList0]), Manifest#manifest{snapshots = SnapList0, delete_sqn = DeleteSQN}. @@ -292,7 +301,7 @@ ready_to_delete(Manifest, Filename) -> check_for_work(Manifest, Thresholds) -> CheckLevelFun = fun({Level, MaxCount}, {AccL, AccC}) -> - case dict:fetch(Level, Manifest#manifest.level_counts) of + case array:get(Level, Manifest#manifest.level_counts) of LC when LC > MaxCount -> {[Level|AccL], AccC + LC - MaxCount}; _ -> @@ -335,7 +344,7 @@ new_manifest(Table) -> pidmap = dict:new(), level_counts = array:new([{size, ?MAX_LEVELS + 1}, {default, 0}]), snapshots = [], - delete_sqn = infinity + delete_sqn = 0 }. range_lookup(Manifest, Level, StartKey, EndKey, MapFun) -> @@ -430,7 +439,7 @@ open_manifestfile(RootPath, [TopManSQN|Rest]) -> open_manifestfile(RootPath, Rest); {ok, Table} -> leveled_log:log("P0012", [TopManSQN]), - {TopManSQN, Table} + {TopManSQN, new_manifest(Table)} end. key_lookup(Manifest, Level, KeyToFind, ManSQN, GC) -> @@ -483,88 +492,178 @@ key_lookup(Manifest, Level, {LastKey, LastFN}, KeyToFind, ManSQN, GC) -> -ifdef(TEST). -rangequery_manifest_test() -> +initial_setup() -> E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, - filename="Z1"}, + filename="Z1", + owner="pid_z1"}, E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"}, end_key={o, "Bucket1", "K71", null}, - filename="Z2"}, + filename="Z2", + owner="pid_z2"}, E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null}, end_key={o, "Bucket1", "K993", null}, - filename="Z3"}, + filename="Z3", + owner="pid_z3"}, E4 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, end_key={i, "Bucket1", {"Idx1", "Fld7"}, "K93"}, - filename="Z4"}, + filename="Z4", + owner="pid_z4"}, E5 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld7"}, "K97"}, end_key={o, "Bucket1", "K78", null}, - filename="Z5"}, + filename="Z5", + owner="pid_z5"}, E6 = #manifest_entry{start_key={o, "Bucket1", "K81", null}, end_key={o, "Bucket1", "K996", null}, - filename="Z6"}, - - Manifest = new_manifest(), - insert_manifest_entry(Manifest, 1, 1, E1), - insert_manifest_entry(Manifest, 1, 1, E2), - insert_manifest_entry(Manifest, 1, 1, E3), - insert_manifest_entry(Manifest, 1, 2, E4), - insert_manifest_entry(Manifest, 1, 2, E5), - insert_manifest_entry(Manifest, 1, 2, E6), - - SK1 = {o, "Bucket1", "K711", null}, - EK1 = {o, "Bucket1", "K999", null}, - RL1_1 = range_lookup(Manifest, 1, SK1, EK1), - ?assertMatch(["Z3"], RL1_1), - RL1_2 = range_lookup(Manifest, 2, SK1, EK1), - ?assertMatch(["Z5", "Z6"], RL1_2), - SK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, - EK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, - RL2_1 = range_lookup(Manifest, 1, SK2, EK2), - ?assertMatch(["Z1"], RL2_1), - RL2_2 = range_lookup(Manifest, 2, SK2, EK2), - ?assertMatch(["Z5"], RL2_2), - - SK3 = {o, "Bucket1", "K994", null}, - EK3 = {o, "Bucket1", "K995", null}, - RL3_1 = range_lookup(Manifest, 1, SK3, EK3), - ?assertMatch([], RL3_1), - RL3_2 = range_lookup(Manifest, 2, SK3, EK3), - ?assertMatch(["Z6"], RL3_2), + filename="Z6", + owner="pid_z6"}, + Man0 = new_manifest(), + % insert_manifest_entry(Manifest, ManSQN, Level, Entry) + Man1 = insert_manifest_entry(Man0, 1, 1, E1), + Man2 = insert_manifest_entry(Man1, 1, 1, E2), + Man3 = insert_manifest_entry(Man2, 1, 1, E3), + Man4 = insert_manifest_entry(Man3, 1, 2, E4), + Man5 = insert_manifest_entry(Man4, 1, 2, E5), + Man6 = insert_manifest_entry(Man5, 1, 2, E6), + {Man0, Man1, Man2, Man3, Man4, Man5, Man6}. + +changeup_setup(Man6) -> + E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, + end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, + filename="Z1", + owner="pid_z1"}, + E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"}, + end_key={o, "Bucket1", "K71", null}, + filename="Z2", + owner="pid_z2"}, + E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null}, + end_key={o, "Bucket1", "K993", null}, + filename="Z3", + owner="pid_z3"}, + E1_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld4"}, "K8"}, end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K62"}, + owner="pid_y1", filename="Y1"}, E2_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K67"}, - end_key={o, "Bucket1", "K45", null}, - filename="Y2"}, + end_key={o, "Bucket1", "K45", null}, + owner="pid_y2", + filename="Y2"}, E3_2 = #manifest_entry{start_key={o, "Bucket1", "K47", null}, end_key={o, "Bucket1", "K812", null}, + owner="pid_y3", filename="Y3"}, E4_2 = #manifest_entry{start_key={o, "Bucket1", "K815", null}, end_key={o, "Bucket1", "K998", null}, + owner="pid_y4", filename="Y4"}, - insert_manifest_entry(Manifest, 2, 1, E1_2), - insert_manifest_entry(Manifest, 2, 1, E2_2), - insert_manifest_entry(Manifest, 2, 1, E3_2), - insert_manifest_entry(Manifest, 2, 1, E4_2), - remove_manifest_entry(Manifest, 2, 1, E1), - remove_manifest_entry(Manifest, 2, 1, E2), - remove_manifest_entry(Manifest, 2, 1, E3), + Man7 = insert_manifest_entry(Man6, 2, 1, E1_2), + Man8 = insert_manifest_entry(Man7, 2, 1, E2_2), + Man9 = insert_manifest_entry(Man8, 2, 1, E3_2), + Man10 = insert_manifest_entry(Man9, 2, 1, E4_2), + % remove_manifest_entry(Manifest, ManSQN, Level, Entry) + Man11 = remove_manifest_entry(Man10, 2, 1, E1), + Man12 = remove_manifest_entry(Man11, 2, 1, E2), + Man13 = remove_manifest_entry(Man12, 2, 1, E3), + {Man7, Man8, Man9, Man10, Man11, Man12, Man13}. + +keylookup_manifest_test() -> + {Man0, Man1, Man2, Man3, _Man4, _Man5, Man6} = initial_setup(), + LK1_1 = {o, "Bucket1", "K711", null}, + LK1_2 = {o, "Bucket1", "K70", null}, + LK1_3 = {o, "Bucket1", "K71", null}, + LK1_4 = {o, "Bucket1", "K75", null}, + LK1_5 = {o, "Bucket1", "K76", null}, - RL1_1A = range_lookup(Manifest, 1, SK1, EK1), - ?assertMatch(["Z3"], RL1_1A), - RL2_1A = range_lookup(Manifest, 1, SK2, EK2), - ?assertMatch(["Z1"], RL2_1A), - RL3_1A = range_lookup(Manifest, 1, SK3, EK3), + ?assertMatch(false, key_lookup(Man0, 1, LK1_1)), + ?assertMatch(false, key_lookup(Man1, 1, LK1_1)), + ?assertMatch(false, key_lookup(Man2, 1, LK1_1)), + ?assertMatch(false, key_lookup(Man3, 1, LK1_1)), + ?assertMatch(false, key_lookup(Man6, 1, LK1_1)), + + ?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_2)), + ?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_3)), + ?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_4)), + ?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_5)), + + ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_2)), + ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_3)), + ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_4)), + ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_5)), + + {_Man7, _Man8, _Man9, _Man10, _Man11, _Man12, + Man13} = changeup_setup(Man6), + + ?assertMatch(false, key_lookup(Man0, 1, LK1_1)), + ?assertMatch(false, key_lookup(Man1, 1, LK1_1)), + ?assertMatch(false, key_lookup(Man2, 1, LK1_1)), + ?assertMatch(false, key_lookup(Man3, 1, LK1_1)), + ?assertMatch(false, key_lookup(Man6, 1, LK1_1)), + + ?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_2)), + ?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_3)), + io:format("Commencing failing test:~n"), + ?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_4)), + ?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_5)), + + ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_2)), + ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_3)), + ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_4)), + ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_5)), + + ?assertMatch("pid_y3", key_lookup(Man13, 1, LK1_4)), + ?assertMatch("pid_z5", key_lookup(Man13, 2, LK1_4)). + + +rangequery_manifest_test() -> + {_Man0, _Man1, _Man2, _Man3, _Man4, _Man5, Man6} = initial_setup(), + + SK1 = {o, "Bucket1", "K711", null}, + EK1 = {o, "Bucket1", "K999", null}, + RL1_1 = range_lookup(Man6, 1, SK1, EK1), + ?assertMatch([{next, "pid_z3", {o, "Bucket1", "K75", null}}], RL1_1), + RL1_2 = range_lookup(Man6, 2, SK1, EK1), + ?assertMatch([{next, "pid_z5", {o, "Bucket1", "K711", null}}, + {next, "pid_z6", {o, "Bucket1", "K81", null}}], + RL1_2), + SK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, + EK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, + RL2_1 = range_lookup(Man6, 1, SK2, EK2), + ?assertMatch([{next, "pid_z1", {i, "Bucket1", {"Idx1", "Fld8"}, null}}], + RL2_1), + RL2_2 = range_lookup(Man6, 2, SK2, EK2), + ?assertMatch([{next, "pid_z5", {i, "Bucket1", {"Idx1", "Fld8"}, null}}], + RL2_2), + + SK3 = {o, "Bucket1", "K994", null}, + EK3 = {o, "Bucket1", "K995", null}, + RL3_1 = range_lookup(Man6, 1, SK3, EK3), + ?assertMatch([], RL3_1), + RL3_2 = range_lookup(Man6, 2, SK3, EK3), + ?assertMatch([{next, "pid_z6", {o, "Bucket1", "K994", null}}], RL3_2), + + {_Man7, _Man8, _Man9, _Man10, _Man11, _Man12, + Man13} = changeup_setup(Man6), + + % Results unchanged despiter ES table change if using old manifest + RL1_1A = range_lookup(Man6, 1, SK1, EK1), + ?assertMatch([{next, "pid_z3", {o, "Bucket1", "K75", null}}], RL1_1A), + RL2_1A = range_lookup(Man6, 1, SK2, EK2), + ?assertMatch([{next, "pid_z1", {i, "Bucket1", {"Idx1", "Fld8"}, null}}], + RL2_1A), + RL3_1A = range_lookup(Man6, 1, SK3, EK3), ?assertMatch([], RL3_1A), - - RL1_1B = range_lookup(Manifest, 1, SK1, EK1), - ?assertMatch(["Y3", "Y4"], RL1_1B), - RL2_1B = range_lookup(Manifest, 1, SK2, EK2), - ?assertMatch(["Y1"], RL2_1B), - RL3_1B = range_lookup(Manifest, 1, SK3, EK3), - ?assertMatch(["Y4"], RL3_1B). + + RL1_1B = range_lookup(Man13, 1, SK1, EK1), + ?assertMatch([{next, "pid_y3", {o, "Bucket1", "K711", null}}, + {next, "pid_y4", {o, "Bucket1", "K815", null}}], RL1_1B), + RL2_1B = range_lookup(Man13, 1, SK2, EK2), + ?assertMatch([{next, "pid_y1", {i, "Bucket1", {"Idx1", "Fld8"}, null}}], + RL2_1B), + RL3_1B = range_lookup(Man13, 1, SK3, EK3), + ?assertMatch([{next, "pid_y4", {o, "Bucket1", "K994", null}}], RL3_1B). -endif. \ No newline at end of file diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index a0ddbcd..0d45833 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -111,7 +111,7 @@ requestandhandle_work(State) -> Manifest, State#state.root_path), leveled_log:log("PC007", []), - ok = leveled_penciller:pcl_commitmanifestchange(State#state.owner, + ok = leveled_penciller:pcl_manifestchange(State#state.owner, UpdManifest), ok = leveled_manifest:save_manifest(UpdManifest, State#state.root_path), @@ -171,7 +171,7 @@ perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN) -> SinkPointerList = leveled_manifest:pointer_convert(Manifest, SinkList), MaxSQN = leveled_sst:sst_getmaxsequencenumber(Src#manifest_entry.owner), SinkLevel = SrcLevel + 1, - SinkBasement = leveled_basement:is_basement(Manifest, SinkLevel), + SinkBasement = leveled_manifest:is_basement(Manifest, SinkLevel), Man0 = do_merge(SrcList, SinkPointerList, SinkLevel, SinkBasement, RootPath, NewSQN, MaxSQN, diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index d4423cb..6969cf5 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -179,7 +179,7 @@ pcl_fetchnextkey/5, pcl_checksequencenumber/3, pcl_workforclerk/1, - pcl_confirmmanifestchange/2, + pcl_manifestchange/2, pcl_confirml0complete/4, pcl_confirmdelete/2, pcl_close/1, @@ -291,7 +291,7 @@ pcl_checksequencenumber(Pid, Key, SQN) -> pcl_workforclerk(Pid) -> gen_server:call(Pid, work_for_clerk, infinity). -pcl_confirmmanifestchange(Pid, Manifest) -> +pcl_manifestchange(Pid, Manifest) -> gen_server:cast(Pid, {manifest_change, Manifest}). pcl_confirml0complete(Pid, FN, StartKey, EndKey) -> @@ -548,7 +548,7 @@ terminate(Reason, State) -> L0 = leveled_manifest:key_lookup(State#state.manifest, 0, all), case {State#state.levelzero_pending, L0} of {false, false} -> - L0Pid = roll_memory(State, true), + L0Pid = roll_memory(State, true), ok = leveled_sst:sst_close(L0Pid); StatusTuple -> leveled_log:log("P0010", [StatusTuple]) @@ -842,10 +842,9 @@ find_nextkey(QueryArray, LCnt, {BestKeyLevel, BestKV}, LCnt + 1, {BKL, BKV}, StartKey, EndKey, Width); - {{next, ManifestEntry, _SK}, BKL, BKV} -> + {{next, Owner, _SK}, BKL, BKV} -> % The first key at this level is pointer to a file - need to query % the file to expand this level out before proceeding - Owner = ManifestEntry#manifest_entry.owner, Pointer = {next, Owner, StartKey, EndKey}, UpdList = leveled_sst:expand_list_by_pointer(Pointer, RestOfKeys, @@ -1036,7 +1035,7 @@ generate_randomkeys(Count, SQN, Acc) -> clean_testdir(RootPath) -> - clean_subdir(filepath(RootPath, manifest)), + clean_subdir(leveled_manifest:filepath(RootPath, manifest)), clean_subdir(filepath(RootPath, files)). clean_subdir(DirPath) -> @@ -1186,10 +1185,6 @@ simple_server_test() -> 1)), ok = pcl_close(PclSnap), - % Ignore a fake pending mnaifest on startup - ok = file:write_file(RootPath ++ "/" ++ ?MANIFEST_FP ++ "nonzero_99.pnd", - term_to_binary("Hello")), - {ok, PclSnap2} = pcl_start(SnapOpts), leveled_bookie:load_snapshot(PclSnap2, leveled_bookie:empty_ledgercache()), ?assertMatch(false, pcl_checksequencenumber(PclSnap2, @@ -1204,6 +1199,12 @@ simple_server_test() -> "Key0001", null}, 4005)), + + io:format("Snap2 B2 K2 ~w~n", + [pcl_fetch(PclSnap2, {o, "Bucket0002", "Key0002", null})]), + io:format("r B2 K2 ~w~n", + [pcl_fetch(PCLr, {o, "Bucket0002", "Key0002", null})]), + ?assertMatch(true, pcl_checksequencenumber(PclSnap2, {o, "Bucket0002",