Manifest refactor - STILL BROKEN

Some working tests now, but sitll broken
This commit is contained in:
martinsumner 2017-01-14 16:36:05 +00:00
parent 0204a23a58
commit 76bdd83346
4 changed files with 186 additions and 84 deletions

View file

@ -282,6 +282,8 @@ turn_to_string(Item) ->
% Compare a key against a query key, only comparing elements that are non-null % Compare a key against a query key, only comparing elements that are non-null
% in the Query key. This is used for comparing against end keys in queries. % in the Query key. This is used for comparing against end keys in queries.
endkey_passed(all, _) ->
false;
endkey_passed({EK1, null, null, null}, {CK1, _, _, _}) -> endkey_passed({EK1, null, null, null}, {CK1, _, _, _}) ->
EK1 < CK1; EK1 < CK1;
endkey_passed({EK1, EK2, null, null}, {CK1, CK2, _, _}) -> endkey_passed({EK1, EK2, null, null}, {CK1, CK2, _, _}) ->

View file

@ -52,12 +52,15 @@
pointer_convert/2 pointer_convert/2
]). ]).
-export([
filepath/2
]).
-include_lib("eunit/include/eunit.hrl"). -include_lib("eunit/include/eunit.hrl").
-define(MANIFEST_FILEX, "man"). -define(MANIFEST_FILEX, "man").
-define(MANIFEST_FP, "ledger_manifest"). -define(MANIFEST_FP, "ledger_manifest").
-define(MAX_LEVELS, 8). -define(MAX_LEVELS, 8).
-define(END_KEY, {null, null, null, null}).
-record(manifest, {table, -record(manifest, {table,
% A Multi-Version ETS table for lookup % A Multi-Version ETS table for lookup
@ -71,7 +74,7 @@
% An array of level counts to speed up compation work assessment % An array of level counts to speed up compation work assessment
snapshots :: list(), snapshots :: list(),
% A list of snaphots (i.e. clones) % A list of snaphots (i.e. clones)
delete_sqn :: integer()|infinity delete_sqn :: integer()
% The lowest SQN of any clone % The lowest SQN of any clone
}). }).
@ -80,7 +83,7 @@
%%%============================================================================ %%%============================================================================
new_manifest() -> new_manifest() ->
Table = ets:new(manifest, [ordered_set]), Table = ets:new(manifest, [ordered_set, public]),
new_manifest(Table). new_manifest(Table).
open_manifest(RootPath) -> open_manifest(RootPath) ->
@ -101,9 +104,8 @@ open_manifest(RootPath) ->
ValidManSQNs = lists:reverse(lists:sort(lists:foldl(ExtractSQNFun, ValidManSQNs = lists:reverse(lists:sort(lists:foldl(ExtractSQNFun,
[], [],
Filenames))), Filenames))),
{ManSQN, Table} = open_manifestfile(RootPath, ValidManSQNs), {ManSQN, Manifest} = open_manifestfile(RootPath, ValidManSQNs),
Manifest = new_manifest(Table), Manifest#manifest{manifest_sqn = ManSQN, delete_sqn = ManSQN}.
Manifest#manifest{manifest_sqn = ManSQN}.
copy_manifest(Manifest) -> copy_manifest(Manifest) ->
% Copy the manifest ensuring anything only the master process should care % Copy the manifest ensuring anything only the master process should care
@ -162,7 +164,7 @@ insert_manifest_entry(Manifest, ManSQN, Level, Entry) ->
remove_manifest_entry(Manifest, ManSQN, Level, Entry) -> remove_manifest_entry(Manifest, ManSQN, Level, Entry) ->
Key = {Level, Entry#manifest_entry.end_key, Entry#manifest_entry.filename}, Key = {Level, Entry#manifest_entry.end_key, Entry#manifest_entry.filename},
[{Key, Value0}] = ets:lookup(Manifest, Key), [{Key, Value0}] = ets:lookup(Manifest#manifest.table, Key),
{StartKey, {active, ActiveSQN}, {tomb, infinity}} = Value0, {StartKey, {active, ActiveSQN}, {tomb, infinity}} = Value0,
Value1 = {StartKey, {active, ActiveSQN}, {tomb, ManSQN}}, Value1 = {StartKey, {active, ActiveSQN}, {tomb, ManSQN}},
true = ets:insert(Manifest#manifest.table, {Key, Value1}), true = ets:insert(Manifest#manifest.table, {Key, Value1}),
@ -205,7 +207,13 @@ key_lookup(Manifest, Level, Key) ->
range_lookup(Manifest, Level, StartKey, EndKey) -> range_lookup(Manifest, Level, StartKey, EndKey) ->
MapFun = MapFun =
fun({{_Level, _LastKey, FN}, FirstKey}) -> fun({{_Level, _LastKey, FN}, FirstKey}) ->
{next, dict:fetch(FN, Manifest#manifest.pidmap), FirstKey} {Pid, _SQN} = dict:fetch(FN, Manifest#manifest.pidmap),
case FirstKey < StartKey of
true ->
{next, Pid, StartKey};
false ->
{next, Pid, FirstKey}
end
end, end,
range_lookup(Manifest, Level, StartKey, EndKey, MapFun). range_lookup(Manifest, Level, StartKey, EndKey, MapFun).
@ -245,7 +253,7 @@ mergefile_selector(Manifest, Level) ->
Level, Level,
{all, 0}, {all, 0},
all, all,
?END_KEY, all,
[], [],
Manifest#manifest.manifest_sqn), Manifest#manifest.manifest_sqn),
{{Level, LastKey, FN}, {{Level, LastKey, FN},
@ -267,14 +275,15 @@ release_snapshot(Manifest, Pid) ->
fun({P, SQN, TS}, {Acc, MinSQN}) -> fun({P, SQN, TS}, {Acc, MinSQN}) ->
case P of case P of
Pid -> Pid ->
Acc; {Acc, MinSQN};
_ -> _ ->
{[{P, SQN, TS}|Acc], min(SQN, MinSQN)} {[{P, SQN, TS}|Acc], min(SQN, MinSQN)}
end end
end, end,
{SnapList0, DeleteSQN} = lists:foldl(FilterFun, {SnapList0,
{[], infinity}, DeleteSQN} = lists:foldl(FilterFun,
Manifest#manifest.snapshots), {[], Manifest#manifest.manifest_sqn},
Manifest#manifest.snapshots),
leveled_log:log("P0004", [SnapList0]), leveled_log:log("P0004", [SnapList0]),
Manifest#manifest{snapshots = SnapList0, delete_sqn = DeleteSQN}. Manifest#manifest{snapshots = SnapList0, delete_sqn = DeleteSQN}.
@ -292,7 +301,7 @@ ready_to_delete(Manifest, Filename) ->
check_for_work(Manifest, Thresholds) -> check_for_work(Manifest, Thresholds) ->
CheckLevelFun = CheckLevelFun =
fun({Level, MaxCount}, {AccL, AccC}) -> fun({Level, MaxCount}, {AccL, AccC}) ->
case dict:fetch(Level, Manifest#manifest.level_counts) of case array:get(Level, Manifest#manifest.level_counts) of
LC when LC > MaxCount -> LC when LC > MaxCount ->
{[Level|AccL], AccC + LC - MaxCount}; {[Level|AccL], AccC + LC - MaxCount};
_ -> _ ->
@ -335,7 +344,7 @@ new_manifest(Table) ->
pidmap = dict:new(), pidmap = dict:new(),
level_counts = array:new([{size, ?MAX_LEVELS + 1}, {default, 0}]), level_counts = array:new([{size, ?MAX_LEVELS + 1}, {default, 0}]),
snapshots = [], snapshots = [],
delete_sqn = infinity delete_sqn = 0
}. }.
range_lookup(Manifest, Level, StartKey, EndKey, MapFun) -> range_lookup(Manifest, Level, StartKey, EndKey, MapFun) ->
@ -430,7 +439,7 @@ open_manifestfile(RootPath, [TopManSQN|Rest]) ->
open_manifestfile(RootPath, Rest); open_manifestfile(RootPath, Rest);
{ok, Table} -> {ok, Table} ->
leveled_log:log("P0012", [TopManSQN]), leveled_log:log("P0012", [TopManSQN]),
{TopManSQN, Table} {TopManSQN, new_manifest(Table)}
end. end.
key_lookup(Manifest, Level, KeyToFind, ManSQN, GC) -> key_lookup(Manifest, Level, KeyToFind, ManSQN, GC) ->
@ -483,88 +492,178 @@ key_lookup(Manifest, Level, {LastKey, LastFN}, KeyToFind, ManSQN, GC) ->
-ifdef(TEST). -ifdef(TEST).
rangequery_manifest_test() -> initial_setup() ->
E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"},
end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"},
filename="Z1"}, filename="Z1",
owner="pid_z1"},
E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"}, E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"},
end_key={o, "Bucket1", "K71", null}, end_key={o, "Bucket1", "K71", null},
filename="Z2"}, filename="Z2",
owner="pid_z2"},
E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null}, E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null},
end_key={o, "Bucket1", "K993", null}, end_key={o, "Bucket1", "K993", null},
filename="Z3"}, filename="Z3",
owner="pid_z3"},
E4 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, E4 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"},
end_key={i, "Bucket1", {"Idx1", "Fld7"}, "K93"}, end_key={i, "Bucket1", {"Idx1", "Fld7"}, "K93"},
filename="Z4"}, filename="Z4",
owner="pid_z4"},
E5 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld7"}, "K97"}, E5 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld7"}, "K97"},
end_key={o, "Bucket1", "K78", null}, end_key={o, "Bucket1", "K78", null},
filename="Z5"}, filename="Z5",
owner="pid_z5"},
E6 = #manifest_entry{start_key={o, "Bucket1", "K81", null}, E6 = #manifest_entry{start_key={o, "Bucket1", "K81", null},
end_key={o, "Bucket1", "K996", null}, end_key={o, "Bucket1", "K996", null},
filename="Z6"}, filename="Z6",
owner="pid_z6"},
Manifest = new_manifest(),
insert_manifest_entry(Manifest, 1, 1, E1),
insert_manifest_entry(Manifest, 1, 1, E2),
insert_manifest_entry(Manifest, 1, 1, E3),
insert_manifest_entry(Manifest, 1, 2, E4),
insert_manifest_entry(Manifest, 1, 2, E5),
insert_manifest_entry(Manifest, 1, 2, E6),
SK1 = {o, "Bucket1", "K711", null},
EK1 = {o, "Bucket1", "K999", null},
RL1_1 = range_lookup(Manifest, 1, SK1, EK1),
?assertMatch(["Z3"], RL1_1),
RL1_2 = range_lookup(Manifest, 2, SK1, EK1),
?assertMatch(["Z5", "Z6"], RL1_2),
SK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null},
EK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null},
RL2_1 = range_lookup(Manifest, 1, SK2, EK2),
?assertMatch(["Z1"], RL2_1),
RL2_2 = range_lookup(Manifest, 2, SK2, EK2),
?assertMatch(["Z5"], RL2_2),
SK3 = {o, "Bucket1", "K994", null},
EK3 = {o, "Bucket1", "K995", null},
RL3_1 = range_lookup(Manifest, 1, SK3, EK3),
?assertMatch([], RL3_1),
RL3_2 = range_lookup(Manifest, 2, SK3, EK3),
?assertMatch(["Z6"], RL3_2),
Man0 = new_manifest(),
% insert_manifest_entry(Manifest, ManSQN, Level, Entry)
Man1 = insert_manifest_entry(Man0, 1, 1, E1),
Man2 = insert_manifest_entry(Man1, 1, 1, E2),
Man3 = insert_manifest_entry(Man2, 1, 1, E3),
Man4 = insert_manifest_entry(Man3, 1, 2, E4),
Man5 = insert_manifest_entry(Man4, 1, 2, E5),
Man6 = insert_manifest_entry(Man5, 1, 2, E6),
{Man0, Man1, Man2, Man3, Man4, Man5, Man6}.
changeup_setup(Man6) ->
E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"},
end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"},
filename="Z1",
owner="pid_z1"},
E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"},
end_key={o, "Bucket1", "K71", null},
filename="Z2",
owner="pid_z2"},
E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null},
end_key={o, "Bucket1", "K993", null},
filename="Z3",
owner="pid_z3"},
E1_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld4"}, "K8"}, E1_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld4"}, "K8"},
end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K62"}, end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K62"},
owner="pid_y1",
filename="Y1"}, filename="Y1"},
E2_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K67"}, E2_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K67"},
end_key={o, "Bucket1", "K45", null}, end_key={o, "Bucket1", "K45", null},
filename="Y2"}, owner="pid_y2",
filename="Y2"},
E3_2 = #manifest_entry{start_key={o, "Bucket1", "K47", null}, E3_2 = #manifest_entry{start_key={o, "Bucket1", "K47", null},
end_key={o, "Bucket1", "K812", null}, end_key={o, "Bucket1", "K812", null},
owner="pid_y3",
filename="Y3"}, filename="Y3"},
E4_2 = #manifest_entry{start_key={o, "Bucket1", "K815", null}, E4_2 = #manifest_entry{start_key={o, "Bucket1", "K815", null},
end_key={o, "Bucket1", "K998", null}, end_key={o, "Bucket1", "K998", null},
owner="pid_y4",
filename="Y4"}, filename="Y4"},
insert_manifest_entry(Manifest, 2, 1, E1_2), Man7 = insert_manifest_entry(Man6, 2, 1, E1_2),
insert_manifest_entry(Manifest, 2, 1, E2_2), Man8 = insert_manifest_entry(Man7, 2, 1, E2_2),
insert_manifest_entry(Manifest, 2, 1, E3_2), Man9 = insert_manifest_entry(Man8, 2, 1, E3_2),
insert_manifest_entry(Manifest, 2, 1, E4_2), Man10 = insert_manifest_entry(Man9, 2, 1, E4_2),
remove_manifest_entry(Manifest, 2, 1, E1), % remove_manifest_entry(Manifest, ManSQN, Level, Entry)
remove_manifest_entry(Manifest, 2, 1, E2), Man11 = remove_manifest_entry(Man10, 2, 1, E1),
remove_manifest_entry(Manifest, 2, 1, E3), Man12 = remove_manifest_entry(Man11, 2, 1, E2),
Man13 = remove_manifest_entry(Man12, 2, 1, E3),
{Man7, Man8, Man9, Man10, Man11, Man12, Man13}.
keylookup_manifest_test() ->
{Man0, Man1, Man2, Man3, _Man4, _Man5, Man6} = initial_setup(),
LK1_1 = {o, "Bucket1", "K711", null},
LK1_2 = {o, "Bucket1", "K70", null},
LK1_3 = {o, "Bucket1", "K71", null},
LK1_4 = {o, "Bucket1", "K75", null},
LK1_5 = {o, "Bucket1", "K76", null},
RL1_1A = range_lookup(Manifest, 1, SK1, EK1), ?assertMatch(false, key_lookup(Man0, 1, LK1_1)),
?assertMatch(["Z3"], RL1_1A), ?assertMatch(false, key_lookup(Man1, 1, LK1_1)),
RL2_1A = range_lookup(Manifest, 1, SK2, EK2), ?assertMatch(false, key_lookup(Man2, 1, LK1_1)),
?assertMatch(["Z1"], RL2_1A), ?assertMatch(false, key_lookup(Man3, 1, LK1_1)),
RL3_1A = range_lookup(Manifest, 1, SK3, EK3), ?assertMatch(false, key_lookup(Man6, 1, LK1_1)),
?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_2)),
?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_3)),
?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_4)),
?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_5)),
?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_2)),
?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_3)),
?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_4)),
?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_5)),
{_Man7, _Man8, _Man9, _Man10, _Man11, _Man12,
Man13} = changeup_setup(Man6),
?assertMatch(false, key_lookup(Man0, 1, LK1_1)),
?assertMatch(false, key_lookup(Man1, 1, LK1_1)),
?assertMatch(false, key_lookup(Man2, 1, LK1_1)),
?assertMatch(false, key_lookup(Man3, 1, LK1_1)),
?assertMatch(false, key_lookup(Man6, 1, LK1_1)),
?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_2)),
?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_3)),
io:format("Commencing failing test:~n"),
?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_4)),
?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_5)),
?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_2)),
?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_3)),
?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_4)),
?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_5)),
?assertMatch("pid_y3", key_lookup(Man13, 1, LK1_4)),
?assertMatch("pid_z5", key_lookup(Man13, 2, LK1_4)).
rangequery_manifest_test() ->
{_Man0, _Man1, _Man2, _Man3, _Man4, _Man5, Man6} = initial_setup(),
SK1 = {o, "Bucket1", "K711", null},
EK1 = {o, "Bucket1", "K999", null},
RL1_1 = range_lookup(Man6, 1, SK1, EK1),
?assertMatch([{next, "pid_z3", {o, "Bucket1", "K75", null}}], RL1_1),
RL1_2 = range_lookup(Man6, 2, SK1, EK1),
?assertMatch([{next, "pid_z5", {o, "Bucket1", "K711", null}},
{next, "pid_z6", {o, "Bucket1", "K81", null}}],
RL1_2),
SK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null},
EK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null},
RL2_1 = range_lookup(Man6, 1, SK2, EK2),
?assertMatch([{next, "pid_z1", {i, "Bucket1", {"Idx1", "Fld8"}, null}}],
RL2_1),
RL2_2 = range_lookup(Man6, 2, SK2, EK2),
?assertMatch([{next, "pid_z5", {i, "Bucket1", {"Idx1", "Fld8"}, null}}],
RL2_2),
SK3 = {o, "Bucket1", "K994", null},
EK3 = {o, "Bucket1", "K995", null},
RL3_1 = range_lookup(Man6, 1, SK3, EK3),
?assertMatch([], RL3_1),
RL3_2 = range_lookup(Man6, 2, SK3, EK3),
?assertMatch([{next, "pid_z6", {o, "Bucket1", "K994", null}}], RL3_2),
{_Man7, _Man8, _Man9, _Man10, _Man11, _Man12,
Man13} = changeup_setup(Man6),
% Results unchanged despiter ES table change if using old manifest
RL1_1A = range_lookup(Man6, 1, SK1, EK1),
?assertMatch([{next, "pid_z3", {o, "Bucket1", "K75", null}}], RL1_1A),
RL2_1A = range_lookup(Man6, 1, SK2, EK2),
?assertMatch([{next, "pid_z1", {i, "Bucket1", {"Idx1", "Fld8"}, null}}],
RL2_1A),
RL3_1A = range_lookup(Man6, 1, SK3, EK3),
?assertMatch([], RL3_1A), ?assertMatch([], RL3_1A),
RL1_1B = range_lookup(Manifest, 1, SK1, EK1), RL1_1B = range_lookup(Man13, 1, SK1, EK1),
?assertMatch(["Y3", "Y4"], RL1_1B), ?assertMatch([{next, "pid_y3", {o, "Bucket1", "K711", null}},
RL2_1B = range_lookup(Manifest, 1, SK2, EK2), {next, "pid_y4", {o, "Bucket1", "K815", null}}], RL1_1B),
?assertMatch(["Y1"], RL2_1B), RL2_1B = range_lookup(Man13, 1, SK2, EK2),
RL3_1B = range_lookup(Manifest, 1, SK3, EK3), ?assertMatch([{next, "pid_y1", {i, "Bucket1", {"Idx1", "Fld8"}, null}}],
?assertMatch(["Y4"], RL3_1B). RL2_1B),
RL3_1B = range_lookup(Man13, 1, SK3, EK3),
?assertMatch([{next, "pid_y4", {o, "Bucket1", "K994", null}}], RL3_1B).
-endif. -endif.

View file

@ -111,7 +111,7 @@ requestandhandle_work(State) ->
Manifest, Manifest,
State#state.root_path), State#state.root_path),
leveled_log:log("PC007", []), leveled_log:log("PC007", []),
ok = leveled_penciller:pcl_commitmanifestchange(State#state.owner, ok = leveled_penciller:pcl_manifestchange(State#state.owner,
UpdManifest), UpdManifest),
ok = leveled_manifest:save_manifest(UpdManifest, ok = leveled_manifest:save_manifest(UpdManifest,
State#state.root_path), State#state.root_path),
@ -171,7 +171,7 @@ perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN) ->
SinkPointerList = leveled_manifest:pointer_convert(Manifest, SinkList), SinkPointerList = leveled_manifest:pointer_convert(Manifest, SinkList),
MaxSQN = leveled_sst:sst_getmaxsequencenumber(Src#manifest_entry.owner), MaxSQN = leveled_sst:sst_getmaxsequencenumber(Src#manifest_entry.owner),
SinkLevel = SrcLevel + 1, SinkLevel = SrcLevel + 1,
SinkBasement = leveled_basement:is_basement(Manifest, SinkLevel), SinkBasement = leveled_manifest:is_basement(Manifest, SinkLevel),
Man0 = do_merge(SrcList, SinkPointerList, Man0 = do_merge(SrcList, SinkPointerList,
SinkLevel, SinkBasement, SinkLevel, SinkBasement,
RootPath, NewSQN, MaxSQN, RootPath, NewSQN, MaxSQN,

View file

@ -179,7 +179,7 @@
pcl_fetchnextkey/5, pcl_fetchnextkey/5,
pcl_checksequencenumber/3, pcl_checksequencenumber/3,
pcl_workforclerk/1, pcl_workforclerk/1,
pcl_confirmmanifestchange/2, pcl_manifestchange/2,
pcl_confirml0complete/4, pcl_confirml0complete/4,
pcl_confirmdelete/2, pcl_confirmdelete/2,
pcl_close/1, pcl_close/1,
@ -291,7 +291,7 @@ pcl_checksequencenumber(Pid, Key, SQN) ->
pcl_workforclerk(Pid) -> pcl_workforclerk(Pid) ->
gen_server:call(Pid, work_for_clerk, infinity). gen_server:call(Pid, work_for_clerk, infinity).
pcl_confirmmanifestchange(Pid, Manifest) -> pcl_manifestchange(Pid, Manifest) ->
gen_server:cast(Pid, {manifest_change, Manifest}). gen_server:cast(Pid, {manifest_change, Manifest}).
pcl_confirml0complete(Pid, FN, StartKey, EndKey) -> pcl_confirml0complete(Pid, FN, StartKey, EndKey) ->
@ -548,7 +548,7 @@ terminate(Reason, State) ->
L0 = leveled_manifest:key_lookup(State#state.manifest, 0, all), L0 = leveled_manifest:key_lookup(State#state.manifest, 0, all),
case {State#state.levelzero_pending, L0} of case {State#state.levelzero_pending, L0} of
{false, false} -> {false, false} ->
L0Pid = roll_memory(State, true), L0Pid = roll_memory(State, true),
ok = leveled_sst:sst_close(L0Pid); ok = leveled_sst:sst_close(L0Pid);
StatusTuple -> StatusTuple ->
leveled_log:log("P0010", [StatusTuple]) leveled_log:log("P0010", [StatusTuple])
@ -842,10 +842,9 @@ find_nextkey(QueryArray, LCnt, {BestKeyLevel, BestKV},
LCnt + 1, LCnt + 1,
{BKL, BKV}, {BKL, BKV},
StartKey, EndKey, Width); StartKey, EndKey, Width);
{{next, ManifestEntry, _SK}, BKL, BKV} -> {{next, Owner, _SK}, BKL, BKV} ->
% The first key at this level is pointer to a file - need to query % The first key at this level is pointer to a file - need to query
% the file to expand this level out before proceeding % the file to expand this level out before proceeding
Owner = ManifestEntry#manifest_entry.owner,
Pointer = {next, Owner, StartKey, EndKey}, Pointer = {next, Owner, StartKey, EndKey},
UpdList = leveled_sst:expand_list_by_pointer(Pointer, UpdList = leveled_sst:expand_list_by_pointer(Pointer,
RestOfKeys, RestOfKeys,
@ -1036,7 +1035,7 @@ generate_randomkeys(Count, SQN, Acc) ->
clean_testdir(RootPath) -> clean_testdir(RootPath) ->
clean_subdir(filepath(RootPath, manifest)), clean_subdir(leveled_manifest:filepath(RootPath, manifest)),
clean_subdir(filepath(RootPath, files)). clean_subdir(filepath(RootPath, files)).
clean_subdir(DirPath) -> clean_subdir(DirPath) ->
@ -1186,10 +1185,6 @@ simple_server_test() ->
1)), 1)),
ok = pcl_close(PclSnap), ok = pcl_close(PclSnap),
% Ignore a fake pending mnaifest on startup
ok = file:write_file(RootPath ++ "/" ++ ?MANIFEST_FP ++ "nonzero_99.pnd",
term_to_binary("Hello")),
{ok, PclSnap2} = pcl_start(SnapOpts), {ok, PclSnap2} = pcl_start(SnapOpts),
leveled_bookie:load_snapshot(PclSnap2, leveled_bookie:empty_ledgercache()), leveled_bookie:load_snapshot(PclSnap2, leveled_bookie:empty_ledgercache()),
?assertMatch(false, pcl_checksequencenumber(PclSnap2, ?assertMatch(false, pcl_checksequencenumber(PclSnap2,
@ -1204,6 +1199,12 @@ simple_server_test() ->
"Key0001", "Key0001",
null}, null},
4005)), 4005)),
io:format("Snap2 B2 K2 ~w~n",
[pcl_fetch(PclSnap2, {o, "Bucket0002", "Key0002", null})]),
io:format("r B2 K2 ~w~n",
[pcl_fetch(PCLr, {o, "Bucket0002", "Key0002", null})]),
?assertMatch(true, pcl_checksequencenumber(PclSnap2, ?assertMatch(true, pcl_checksequencenumber(PclSnap2,
{o, {o,
"Bucket0002", "Bucket0002",