Manifest refactor - STILL BROKEN
Some working tests now, but sitll broken
This commit is contained in:
parent
0204a23a58
commit
76bdd83346
4 changed files with 186 additions and 84 deletions
|
@ -282,6 +282,8 @@ turn_to_string(Item) ->
|
||||||
|
|
||||||
% Compare a key against a query key, only comparing elements that are non-null
|
% Compare a key against a query key, only comparing elements that are non-null
|
||||||
% in the Query key. This is used for comparing against end keys in queries.
|
% in the Query key. This is used for comparing against end keys in queries.
|
||||||
|
endkey_passed(all, _) ->
|
||||||
|
false;
|
||||||
endkey_passed({EK1, null, null, null}, {CK1, _, _, _}) ->
|
endkey_passed({EK1, null, null, null}, {CK1, _, _, _}) ->
|
||||||
EK1 < CK1;
|
EK1 < CK1;
|
||||||
endkey_passed({EK1, EK2, null, null}, {CK1, CK2, _, _}) ->
|
endkey_passed({EK1, EK2, null, null}, {CK1, CK2, _, _}) ->
|
||||||
|
|
|
@ -52,12 +52,15 @@
|
||||||
pointer_convert/2
|
pointer_convert/2
|
||||||
]).
|
]).
|
||||||
|
|
||||||
|
-export([
|
||||||
|
filepath/2
|
||||||
|
]).
|
||||||
|
|
||||||
-include_lib("eunit/include/eunit.hrl").
|
-include_lib("eunit/include/eunit.hrl").
|
||||||
|
|
||||||
-define(MANIFEST_FILEX, "man").
|
-define(MANIFEST_FILEX, "man").
|
||||||
-define(MANIFEST_FP, "ledger_manifest").
|
-define(MANIFEST_FP, "ledger_manifest").
|
||||||
-define(MAX_LEVELS, 8).
|
-define(MAX_LEVELS, 8).
|
||||||
-define(END_KEY, {null, null, null, null}).
|
|
||||||
|
|
||||||
-record(manifest, {table,
|
-record(manifest, {table,
|
||||||
% A Multi-Version ETS table for lookup
|
% A Multi-Version ETS table for lookup
|
||||||
|
@ -71,7 +74,7 @@
|
||||||
% An array of level counts to speed up compation work assessment
|
% An array of level counts to speed up compation work assessment
|
||||||
snapshots :: list(),
|
snapshots :: list(),
|
||||||
% A list of snaphots (i.e. clones)
|
% A list of snaphots (i.e. clones)
|
||||||
delete_sqn :: integer()|infinity
|
delete_sqn :: integer()
|
||||||
% The lowest SQN of any clone
|
% The lowest SQN of any clone
|
||||||
}).
|
}).
|
||||||
|
|
||||||
|
@ -80,7 +83,7 @@
|
||||||
%%%============================================================================
|
%%%============================================================================
|
||||||
|
|
||||||
new_manifest() ->
|
new_manifest() ->
|
||||||
Table = ets:new(manifest, [ordered_set]),
|
Table = ets:new(manifest, [ordered_set, public]),
|
||||||
new_manifest(Table).
|
new_manifest(Table).
|
||||||
|
|
||||||
open_manifest(RootPath) ->
|
open_manifest(RootPath) ->
|
||||||
|
@ -101,9 +104,8 @@ open_manifest(RootPath) ->
|
||||||
ValidManSQNs = lists:reverse(lists:sort(lists:foldl(ExtractSQNFun,
|
ValidManSQNs = lists:reverse(lists:sort(lists:foldl(ExtractSQNFun,
|
||||||
[],
|
[],
|
||||||
Filenames))),
|
Filenames))),
|
||||||
{ManSQN, Table} = open_manifestfile(RootPath, ValidManSQNs),
|
{ManSQN, Manifest} = open_manifestfile(RootPath, ValidManSQNs),
|
||||||
Manifest = new_manifest(Table),
|
Manifest#manifest{manifest_sqn = ManSQN, delete_sqn = ManSQN}.
|
||||||
Manifest#manifest{manifest_sqn = ManSQN}.
|
|
||||||
|
|
||||||
copy_manifest(Manifest) ->
|
copy_manifest(Manifest) ->
|
||||||
% Copy the manifest ensuring anything only the master process should care
|
% Copy the manifest ensuring anything only the master process should care
|
||||||
|
@ -162,7 +164,7 @@ insert_manifest_entry(Manifest, ManSQN, Level, Entry) ->
|
||||||
|
|
||||||
remove_manifest_entry(Manifest, ManSQN, Level, Entry) ->
|
remove_manifest_entry(Manifest, ManSQN, Level, Entry) ->
|
||||||
Key = {Level, Entry#manifest_entry.end_key, Entry#manifest_entry.filename},
|
Key = {Level, Entry#manifest_entry.end_key, Entry#manifest_entry.filename},
|
||||||
[{Key, Value0}] = ets:lookup(Manifest, Key),
|
[{Key, Value0}] = ets:lookup(Manifest#manifest.table, Key),
|
||||||
{StartKey, {active, ActiveSQN}, {tomb, infinity}} = Value0,
|
{StartKey, {active, ActiveSQN}, {tomb, infinity}} = Value0,
|
||||||
Value1 = {StartKey, {active, ActiveSQN}, {tomb, ManSQN}},
|
Value1 = {StartKey, {active, ActiveSQN}, {tomb, ManSQN}},
|
||||||
true = ets:insert(Manifest#manifest.table, {Key, Value1}),
|
true = ets:insert(Manifest#manifest.table, {Key, Value1}),
|
||||||
|
@ -205,7 +207,13 @@ key_lookup(Manifest, Level, Key) ->
|
||||||
range_lookup(Manifest, Level, StartKey, EndKey) ->
|
range_lookup(Manifest, Level, StartKey, EndKey) ->
|
||||||
MapFun =
|
MapFun =
|
||||||
fun({{_Level, _LastKey, FN}, FirstKey}) ->
|
fun({{_Level, _LastKey, FN}, FirstKey}) ->
|
||||||
{next, dict:fetch(FN, Manifest#manifest.pidmap), FirstKey}
|
{Pid, _SQN} = dict:fetch(FN, Manifest#manifest.pidmap),
|
||||||
|
case FirstKey < StartKey of
|
||||||
|
true ->
|
||||||
|
{next, Pid, StartKey};
|
||||||
|
false ->
|
||||||
|
{next, Pid, FirstKey}
|
||||||
|
end
|
||||||
end,
|
end,
|
||||||
range_lookup(Manifest, Level, StartKey, EndKey, MapFun).
|
range_lookup(Manifest, Level, StartKey, EndKey, MapFun).
|
||||||
|
|
||||||
|
@ -245,7 +253,7 @@ mergefile_selector(Manifest, Level) ->
|
||||||
Level,
|
Level,
|
||||||
{all, 0},
|
{all, 0},
|
||||||
all,
|
all,
|
||||||
?END_KEY,
|
all,
|
||||||
[],
|
[],
|
||||||
Manifest#manifest.manifest_sqn),
|
Manifest#manifest.manifest_sqn),
|
||||||
{{Level, LastKey, FN},
|
{{Level, LastKey, FN},
|
||||||
|
@ -267,14 +275,15 @@ release_snapshot(Manifest, Pid) ->
|
||||||
fun({P, SQN, TS}, {Acc, MinSQN}) ->
|
fun({P, SQN, TS}, {Acc, MinSQN}) ->
|
||||||
case P of
|
case P of
|
||||||
Pid ->
|
Pid ->
|
||||||
Acc;
|
{Acc, MinSQN};
|
||||||
_ ->
|
_ ->
|
||||||
{[{P, SQN, TS}|Acc], min(SQN, MinSQN)}
|
{[{P, SQN, TS}|Acc], min(SQN, MinSQN)}
|
||||||
end
|
end
|
||||||
end,
|
end,
|
||||||
{SnapList0, DeleteSQN} = lists:foldl(FilterFun,
|
{SnapList0,
|
||||||
{[], infinity},
|
DeleteSQN} = lists:foldl(FilterFun,
|
||||||
Manifest#manifest.snapshots),
|
{[], Manifest#manifest.manifest_sqn},
|
||||||
|
Manifest#manifest.snapshots),
|
||||||
leveled_log:log("P0004", [SnapList0]),
|
leveled_log:log("P0004", [SnapList0]),
|
||||||
Manifest#manifest{snapshots = SnapList0, delete_sqn = DeleteSQN}.
|
Manifest#manifest{snapshots = SnapList0, delete_sqn = DeleteSQN}.
|
||||||
|
|
||||||
|
@ -292,7 +301,7 @@ ready_to_delete(Manifest, Filename) ->
|
||||||
check_for_work(Manifest, Thresholds) ->
|
check_for_work(Manifest, Thresholds) ->
|
||||||
CheckLevelFun =
|
CheckLevelFun =
|
||||||
fun({Level, MaxCount}, {AccL, AccC}) ->
|
fun({Level, MaxCount}, {AccL, AccC}) ->
|
||||||
case dict:fetch(Level, Manifest#manifest.level_counts) of
|
case array:get(Level, Manifest#manifest.level_counts) of
|
||||||
LC when LC > MaxCount ->
|
LC when LC > MaxCount ->
|
||||||
{[Level|AccL], AccC + LC - MaxCount};
|
{[Level|AccL], AccC + LC - MaxCount};
|
||||||
_ ->
|
_ ->
|
||||||
|
@ -335,7 +344,7 @@ new_manifest(Table) ->
|
||||||
pidmap = dict:new(),
|
pidmap = dict:new(),
|
||||||
level_counts = array:new([{size, ?MAX_LEVELS + 1}, {default, 0}]),
|
level_counts = array:new([{size, ?MAX_LEVELS + 1}, {default, 0}]),
|
||||||
snapshots = [],
|
snapshots = [],
|
||||||
delete_sqn = infinity
|
delete_sqn = 0
|
||||||
}.
|
}.
|
||||||
|
|
||||||
range_lookup(Manifest, Level, StartKey, EndKey, MapFun) ->
|
range_lookup(Manifest, Level, StartKey, EndKey, MapFun) ->
|
||||||
|
@ -430,7 +439,7 @@ open_manifestfile(RootPath, [TopManSQN|Rest]) ->
|
||||||
open_manifestfile(RootPath, Rest);
|
open_manifestfile(RootPath, Rest);
|
||||||
{ok, Table} ->
|
{ok, Table} ->
|
||||||
leveled_log:log("P0012", [TopManSQN]),
|
leveled_log:log("P0012", [TopManSQN]),
|
||||||
{TopManSQN, Table}
|
{TopManSQN, new_manifest(Table)}
|
||||||
end.
|
end.
|
||||||
|
|
||||||
key_lookup(Manifest, Level, KeyToFind, ManSQN, GC) ->
|
key_lookup(Manifest, Level, KeyToFind, ManSQN, GC) ->
|
||||||
|
@ -483,88 +492,178 @@ key_lookup(Manifest, Level, {LastKey, LastFN}, KeyToFind, ManSQN, GC) ->
|
||||||
|
|
||||||
-ifdef(TEST).
|
-ifdef(TEST).
|
||||||
|
|
||||||
rangequery_manifest_test() ->
|
initial_setup() ->
|
||||||
E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"},
|
E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"},
|
||||||
end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"},
|
end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"},
|
||||||
filename="Z1"},
|
filename="Z1",
|
||||||
|
owner="pid_z1"},
|
||||||
E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"},
|
E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"},
|
||||||
end_key={o, "Bucket1", "K71", null},
|
end_key={o, "Bucket1", "K71", null},
|
||||||
filename="Z2"},
|
filename="Z2",
|
||||||
|
owner="pid_z2"},
|
||||||
E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null},
|
E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null},
|
||||||
end_key={o, "Bucket1", "K993", null},
|
end_key={o, "Bucket1", "K993", null},
|
||||||
filename="Z3"},
|
filename="Z3",
|
||||||
|
owner="pid_z3"},
|
||||||
E4 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"},
|
E4 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"},
|
||||||
end_key={i, "Bucket1", {"Idx1", "Fld7"}, "K93"},
|
end_key={i, "Bucket1", {"Idx1", "Fld7"}, "K93"},
|
||||||
filename="Z4"},
|
filename="Z4",
|
||||||
|
owner="pid_z4"},
|
||||||
E5 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld7"}, "K97"},
|
E5 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld7"}, "K97"},
|
||||||
end_key={o, "Bucket1", "K78", null},
|
end_key={o, "Bucket1", "K78", null},
|
||||||
filename="Z5"},
|
filename="Z5",
|
||||||
|
owner="pid_z5"},
|
||||||
E6 = #manifest_entry{start_key={o, "Bucket1", "K81", null},
|
E6 = #manifest_entry{start_key={o, "Bucket1", "K81", null},
|
||||||
end_key={o, "Bucket1", "K996", null},
|
end_key={o, "Bucket1", "K996", null},
|
||||||
filename="Z6"},
|
filename="Z6",
|
||||||
|
owner="pid_z6"},
|
||||||
Manifest = new_manifest(),
|
|
||||||
insert_manifest_entry(Manifest, 1, 1, E1),
|
|
||||||
insert_manifest_entry(Manifest, 1, 1, E2),
|
|
||||||
insert_manifest_entry(Manifest, 1, 1, E3),
|
|
||||||
insert_manifest_entry(Manifest, 1, 2, E4),
|
|
||||||
insert_manifest_entry(Manifest, 1, 2, E5),
|
|
||||||
insert_manifest_entry(Manifest, 1, 2, E6),
|
|
||||||
|
|
||||||
SK1 = {o, "Bucket1", "K711", null},
|
|
||||||
EK1 = {o, "Bucket1", "K999", null},
|
|
||||||
RL1_1 = range_lookup(Manifest, 1, SK1, EK1),
|
|
||||||
?assertMatch(["Z3"], RL1_1),
|
|
||||||
RL1_2 = range_lookup(Manifest, 2, SK1, EK1),
|
|
||||||
?assertMatch(["Z5", "Z6"], RL1_2),
|
|
||||||
SK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null},
|
|
||||||
EK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null},
|
|
||||||
RL2_1 = range_lookup(Manifest, 1, SK2, EK2),
|
|
||||||
?assertMatch(["Z1"], RL2_1),
|
|
||||||
RL2_2 = range_lookup(Manifest, 2, SK2, EK2),
|
|
||||||
?assertMatch(["Z5"], RL2_2),
|
|
||||||
|
|
||||||
SK3 = {o, "Bucket1", "K994", null},
|
|
||||||
EK3 = {o, "Bucket1", "K995", null},
|
|
||||||
RL3_1 = range_lookup(Manifest, 1, SK3, EK3),
|
|
||||||
?assertMatch([], RL3_1),
|
|
||||||
RL3_2 = range_lookup(Manifest, 2, SK3, EK3),
|
|
||||||
?assertMatch(["Z6"], RL3_2),
|
|
||||||
|
|
||||||
|
Man0 = new_manifest(),
|
||||||
|
% insert_manifest_entry(Manifest, ManSQN, Level, Entry)
|
||||||
|
Man1 = insert_manifest_entry(Man0, 1, 1, E1),
|
||||||
|
Man2 = insert_manifest_entry(Man1, 1, 1, E2),
|
||||||
|
Man3 = insert_manifest_entry(Man2, 1, 1, E3),
|
||||||
|
Man4 = insert_manifest_entry(Man3, 1, 2, E4),
|
||||||
|
Man5 = insert_manifest_entry(Man4, 1, 2, E5),
|
||||||
|
Man6 = insert_manifest_entry(Man5, 1, 2, E6),
|
||||||
|
{Man0, Man1, Man2, Man3, Man4, Man5, Man6}.
|
||||||
|
|
||||||
|
changeup_setup(Man6) ->
|
||||||
|
E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"},
|
||||||
|
end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"},
|
||||||
|
filename="Z1",
|
||||||
|
owner="pid_z1"},
|
||||||
|
E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"},
|
||||||
|
end_key={o, "Bucket1", "K71", null},
|
||||||
|
filename="Z2",
|
||||||
|
owner="pid_z2"},
|
||||||
|
E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null},
|
||||||
|
end_key={o, "Bucket1", "K993", null},
|
||||||
|
filename="Z3",
|
||||||
|
owner="pid_z3"},
|
||||||
|
|
||||||
E1_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld4"}, "K8"},
|
E1_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld4"}, "K8"},
|
||||||
end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K62"},
|
end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K62"},
|
||||||
|
owner="pid_y1",
|
||||||
filename="Y1"},
|
filename="Y1"},
|
||||||
E2_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K67"},
|
E2_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K67"},
|
||||||
end_key={o, "Bucket1", "K45", null},
|
end_key={o, "Bucket1", "K45", null},
|
||||||
filename="Y2"},
|
owner="pid_y2",
|
||||||
|
filename="Y2"},
|
||||||
E3_2 = #manifest_entry{start_key={o, "Bucket1", "K47", null},
|
E3_2 = #manifest_entry{start_key={o, "Bucket1", "K47", null},
|
||||||
end_key={o, "Bucket1", "K812", null},
|
end_key={o, "Bucket1", "K812", null},
|
||||||
|
owner="pid_y3",
|
||||||
filename="Y3"},
|
filename="Y3"},
|
||||||
E4_2 = #manifest_entry{start_key={o, "Bucket1", "K815", null},
|
E4_2 = #manifest_entry{start_key={o, "Bucket1", "K815", null},
|
||||||
end_key={o, "Bucket1", "K998", null},
|
end_key={o, "Bucket1", "K998", null},
|
||||||
|
owner="pid_y4",
|
||||||
filename="Y4"},
|
filename="Y4"},
|
||||||
|
|
||||||
insert_manifest_entry(Manifest, 2, 1, E1_2),
|
Man7 = insert_manifest_entry(Man6, 2, 1, E1_2),
|
||||||
insert_manifest_entry(Manifest, 2, 1, E2_2),
|
Man8 = insert_manifest_entry(Man7, 2, 1, E2_2),
|
||||||
insert_manifest_entry(Manifest, 2, 1, E3_2),
|
Man9 = insert_manifest_entry(Man8, 2, 1, E3_2),
|
||||||
insert_manifest_entry(Manifest, 2, 1, E4_2),
|
Man10 = insert_manifest_entry(Man9, 2, 1, E4_2),
|
||||||
remove_manifest_entry(Manifest, 2, 1, E1),
|
% remove_manifest_entry(Manifest, ManSQN, Level, Entry)
|
||||||
remove_manifest_entry(Manifest, 2, 1, E2),
|
Man11 = remove_manifest_entry(Man10, 2, 1, E1),
|
||||||
remove_manifest_entry(Manifest, 2, 1, E3),
|
Man12 = remove_manifest_entry(Man11, 2, 1, E2),
|
||||||
|
Man13 = remove_manifest_entry(Man12, 2, 1, E3),
|
||||||
|
{Man7, Man8, Man9, Man10, Man11, Man12, Man13}.
|
||||||
|
|
||||||
|
keylookup_manifest_test() ->
|
||||||
|
{Man0, Man1, Man2, Man3, _Man4, _Man5, Man6} = initial_setup(),
|
||||||
|
LK1_1 = {o, "Bucket1", "K711", null},
|
||||||
|
LK1_2 = {o, "Bucket1", "K70", null},
|
||||||
|
LK1_3 = {o, "Bucket1", "K71", null},
|
||||||
|
LK1_4 = {o, "Bucket1", "K75", null},
|
||||||
|
LK1_5 = {o, "Bucket1", "K76", null},
|
||||||
|
|
||||||
RL1_1A = range_lookup(Manifest, 1, SK1, EK1),
|
?assertMatch(false, key_lookup(Man0, 1, LK1_1)),
|
||||||
?assertMatch(["Z3"], RL1_1A),
|
?assertMatch(false, key_lookup(Man1, 1, LK1_1)),
|
||||||
RL2_1A = range_lookup(Manifest, 1, SK2, EK2),
|
?assertMatch(false, key_lookup(Man2, 1, LK1_1)),
|
||||||
?assertMatch(["Z1"], RL2_1A),
|
?assertMatch(false, key_lookup(Man3, 1, LK1_1)),
|
||||||
RL3_1A = range_lookup(Manifest, 1, SK3, EK3),
|
?assertMatch(false, key_lookup(Man6, 1, LK1_1)),
|
||||||
|
|
||||||
|
?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_2)),
|
||||||
|
?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_3)),
|
||||||
|
?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_4)),
|
||||||
|
?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_5)),
|
||||||
|
|
||||||
|
?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_2)),
|
||||||
|
?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_3)),
|
||||||
|
?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_4)),
|
||||||
|
?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_5)),
|
||||||
|
|
||||||
|
{_Man7, _Man8, _Man9, _Man10, _Man11, _Man12,
|
||||||
|
Man13} = changeup_setup(Man6),
|
||||||
|
|
||||||
|
?assertMatch(false, key_lookup(Man0, 1, LK1_1)),
|
||||||
|
?assertMatch(false, key_lookup(Man1, 1, LK1_1)),
|
||||||
|
?assertMatch(false, key_lookup(Man2, 1, LK1_1)),
|
||||||
|
?assertMatch(false, key_lookup(Man3, 1, LK1_1)),
|
||||||
|
?assertMatch(false, key_lookup(Man6, 1, LK1_1)),
|
||||||
|
|
||||||
|
?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_2)),
|
||||||
|
?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_3)),
|
||||||
|
io:format("Commencing failing test:~n"),
|
||||||
|
?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_4)),
|
||||||
|
?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_5)),
|
||||||
|
|
||||||
|
?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_2)),
|
||||||
|
?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_3)),
|
||||||
|
?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_4)),
|
||||||
|
?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_5)),
|
||||||
|
|
||||||
|
?assertMatch("pid_y3", key_lookup(Man13, 1, LK1_4)),
|
||||||
|
?assertMatch("pid_z5", key_lookup(Man13, 2, LK1_4)).
|
||||||
|
|
||||||
|
|
||||||
|
rangequery_manifest_test() ->
|
||||||
|
{_Man0, _Man1, _Man2, _Man3, _Man4, _Man5, Man6} = initial_setup(),
|
||||||
|
|
||||||
|
SK1 = {o, "Bucket1", "K711", null},
|
||||||
|
EK1 = {o, "Bucket1", "K999", null},
|
||||||
|
RL1_1 = range_lookup(Man6, 1, SK1, EK1),
|
||||||
|
?assertMatch([{next, "pid_z3", {o, "Bucket1", "K75", null}}], RL1_1),
|
||||||
|
RL1_2 = range_lookup(Man6, 2, SK1, EK1),
|
||||||
|
?assertMatch([{next, "pid_z5", {o, "Bucket1", "K711", null}},
|
||||||
|
{next, "pid_z6", {o, "Bucket1", "K81", null}}],
|
||||||
|
RL1_2),
|
||||||
|
SK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null},
|
||||||
|
EK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null},
|
||||||
|
RL2_1 = range_lookup(Man6, 1, SK2, EK2),
|
||||||
|
?assertMatch([{next, "pid_z1", {i, "Bucket1", {"Idx1", "Fld8"}, null}}],
|
||||||
|
RL2_1),
|
||||||
|
RL2_2 = range_lookup(Man6, 2, SK2, EK2),
|
||||||
|
?assertMatch([{next, "pid_z5", {i, "Bucket1", {"Idx1", "Fld8"}, null}}],
|
||||||
|
RL2_2),
|
||||||
|
|
||||||
|
SK3 = {o, "Bucket1", "K994", null},
|
||||||
|
EK3 = {o, "Bucket1", "K995", null},
|
||||||
|
RL3_1 = range_lookup(Man6, 1, SK3, EK3),
|
||||||
|
?assertMatch([], RL3_1),
|
||||||
|
RL3_2 = range_lookup(Man6, 2, SK3, EK3),
|
||||||
|
?assertMatch([{next, "pid_z6", {o, "Bucket1", "K994", null}}], RL3_2),
|
||||||
|
|
||||||
|
{_Man7, _Man8, _Man9, _Man10, _Man11, _Man12,
|
||||||
|
Man13} = changeup_setup(Man6),
|
||||||
|
|
||||||
|
% Results unchanged despiter ES table change if using old manifest
|
||||||
|
RL1_1A = range_lookup(Man6, 1, SK1, EK1),
|
||||||
|
?assertMatch([{next, "pid_z3", {o, "Bucket1", "K75", null}}], RL1_1A),
|
||||||
|
RL2_1A = range_lookup(Man6, 1, SK2, EK2),
|
||||||
|
?assertMatch([{next, "pid_z1", {i, "Bucket1", {"Idx1", "Fld8"}, null}}],
|
||||||
|
RL2_1A),
|
||||||
|
RL3_1A = range_lookup(Man6, 1, SK3, EK3),
|
||||||
?assertMatch([], RL3_1A),
|
?assertMatch([], RL3_1A),
|
||||||
|
|
||||||
RL1_1B = range_lookup(Manifest, 1, SK1, EK1),
|
RL1_1B = range_lookup(Man13, 1, SK1, EK1),
|
||||||
?assertMatch(["Y3", "Y4"], RL1_1B),
|
?assertMatch([{next, "pid_y3", {o, "Bucket1", "K711", null}},
|
||||||
RL2_1B = range_lookup(Manifest, 1, SK2, EK2),
|
{next, "pid_y4", {o, "Bucket1", "K815", null}}], RL1_1B),
|
||||||
?assertMatch(["Y1"], RL2_1B),
|
RL2_1B = range_lookup(Man13, 1, SK2, EK2),
|
||||||
RL3_1B = range_lookup(Manifest, 1, SK3, EK3),
|
?assertMatch([{next, "pid_y1", {i, "Bucket1", {"Idx1", "Fld8"}, null}}],
|
||||||
?assertMatch(["Y4"], RL3_1B).
|
RL2_1B),
|
||||||
|
RL3_1B = range_lookup(Man13, 1, SK3, EK3),
|
||||||
|
?assertMatch([{next, "pid_y4", {o, "Bucket1", "K994", null}}], RL3_1B).
|
||||||
|
|
||||||
|
|
||||||
-endif.
|
-endif.
|
|
@ -111,7 +111,7 @@ requestandhandle_work(State) ->
|
||||||
Manifest,
|
Manifest,
|
||||||
State#state.root_path),
|
State#state.root_path),
|
||||||
leveled_log:log("PC007", []),
|
leveled_log:log("PC007", []),
|
||||||
ok = leveled_penciller:pcl_commitmanifestchange(State#state.owner,
|
ok = leveled_penciller:pcl_manifestchange(State#state.owner,
|
||||||
UpdManifest),
|
UpdManifest),
|
||||||
ok = leveled_manifest:save_manifest(UpdManifest,
|
ok = leveled_manifest:save_manifest(UpdManifest,
|
||||||
State#state.root_path),
|
State#state.root_path),
|
||||||
|
@ -171,7 +171,7 @@ perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN) ->
|
||||||
SinkPointerList = leveled_manifest:pointer_convert(Manifest, SinkList),
|
SinkPointerList = leveled_manifest:pointer_convert(Manifest, SinkList),
|
||||||
MaxSQN = leveled_sst:sst_getmaxsequencenumber(Src#manifest_entry.owner),
|
MaxSQN = leveled_sst:sst_getmaxsequencenumber(Src#manifest_entry.owner),
|
||||||
SinkLevel = SrcLevel + 1,
|
SinkLevel = SrcLevel + 1,
|
||||||
SinkBasement = leveled_basement:is_basement(Manifest, SinkLevel),
|
SinkBasement = leveled_manifest:is_basement(Manifest, SinkLevel),
|
||||||
Man0 = do_merge(SrcList, SinkPointerList,
|
Man0 = do_merge(SrcList, SinkPointerList,
|
||||||
SinkLevel, SinkBasement,
|
SinkLevel, SinkBasement,
|
||||||
RootPath, NewSQN, MaxSQN,
|
RootPath, NewSQN, MaxSQN,
|
||||||
|
|
|
@ -179,7 +179,7 @@
|
||||||
pcl_fetchnextkey/5,
|
pcl_fetchnextkey/5,
|
||||||
pcl_checksequencenumber/3,
|
pcl_checksequencenumber/3,
|
||||||
pcl_workforclerk/1,
|
pcl_workforclerk/1,
|
||||||
pcl_confirmmanifestchange/2,
|
pcl_manifestchange/2,
|
||||||
pcl_confirml0complete/4,
|
pcl_confirml0complete/4,
|
||||||
pcl_confirmdelete/2,
|
pcl_confirmdelete/2,
|
||||||
pcl_close/1,
|
pcl_close/1,
|
||||||
|
@ -291,7 +291,7 @@ pcl_checksequencenumber(Pid, Key, SQN) ->
|
||||||
pcl_workforclerk(Pid) ->
|
pcl_workforclerk(Pid) ->
|
||||||
gen_server:call(Pid, work_for_clerk, infinity).
|
gen_server:call(Pid, work_for_clerk, infinity).
|
||||||
|
|
||||||
pcl_confirmmanifestchange(Pid, Manifest) ->
|
pcl_manifestchange(Pid, Manifest) ->
|
||||||
gen_server:cast(Pid, {manifest_change, Manifest}).
|
gen_server:cast(Pid, {manifest_change, Manifest}).
|
||||||
|
|
||||||
pcl_confirml0complete(Pid, FN, StartKey, EndKey) ->
|
pcl_confirml0complete(Pid, FN, StartKey, EndKey) ->
|
||||||
|
@ -548,7 +548,7 @@ terminate(Reason, State) ->
|
||||||
L0 = leveled_manifest:key_lookup(State#state.manifest, 0, all),
|
L0 = leveled_manifest:key_lookup(State#state.manifest, 0, all),
|
||||||
case {State#state.levelzero_pending, L0} of
|
case {State#state.levelzero_pending, L0} of
|
||||||
{false, false} ->
|
{false, false} ->
|
||||||
L0Pid = roll_memory(State, true),
|
L0Pid = roll_memory(State, true),
|
||||||
ok = leveled_sst:sst_close(L0Pid);
|
ok = leveled_sst:sst_close(L0Pid);
|
||||||
StatusTuple ->
|
StatusTuple ->
|
||||||
leveled_log:log("P0010", [StatusTuple])
|
leveled_log:log("P0010", [StatusTuple])
|
||||||
|
@ -842,10 +842,9 @@ find_nextkey(QueryArray, LCnt, {BestKeyLevel, BestKV},
|
||||||
LCnt + 1,
|
LCnt + 1,
|
||||||
{BKL, BKV},
|
{BKL, BKV},
|
||||||
StartKey, EndKey, Width);
|
StartKey, EndKey, Width);
|
||||||
{{next, ManifestEntry, _SK}, BKL, BKV} ->
|
{{next, Owner, _SK}, BKL, BKV} ->
|
||||||
% The first key at this level is pointer to a file - need to query
|
% The first key at this level is pointer to a file - need to query
|
||||||
% the file to expand this level out before proceeding
|
% the file to expand this level out before proceeding
|
||||||
Owner = ManifestEntry#manifest_entry.owner,
|
|
||||||
Pointer = {next, Owner, StartKey, EndKey},
|
Pointer = {next, Owner, StartKey, EndKey},
|
||||||
UpdList = leveled_sst:expand_list_by_pointer(Pointer,
|
UpdList = leveled_sst:expand_list_by_pointer(Pointer,
|
||||||
RestOfKeys,
|
RestOfKeys,
|
||||||
|
@ -1036,7 +1035,7 @@ generate_randomkeys(Count, SQN, Acc) ->
|
||||||
|
|
||||||
|
|
||||||
clean_testdir(RootPath) ->
|
clean_testdir(RootPath) ->
|
||||||
clean_subdir(filepath(RootPath, manifest)),
|
clean_subdir(leveled_manifest:filepath(RootPath, manifest)),
|
||||||
clean_subdir(filepath(RootPath, files)).
|
clean_subdir(filepath(RootPath, files)).
|
||||||
|
|
||||||
clean_subdir(DirPath) ->
|
clean_subdir(DirPath) ->
|
||||||
|
@ -1186,10 +1185,6 @@ simple_server_test() ->
|
||||||
1)),
|
1)),
|
||||||
ok = pcl_close(PclSnap),
|
ok = pcl_close(PclSnap),
|
||||||
|
|
||||||
% Ignore a fake pending mnaifest on startup
|
|
||||||
ok = file:write_file(RootPath ++ "/" ++ ?MANIFEST_FP ++ "nonzero_99.pnd",
|
|
||||||
term_to_binary("Hello")),
|
|
||||||
|
|
||||||
{ok, PclSnap2} = pcl_start(SnapOpts),
|
{ok, PclSnap2} = pcl_start(SnapOpts),
|
||||||
leveled_bookie:load_snapshot(PclSnap2, leveled_bookie:empty_ledgercache()),
|
leveled_bookie:load_snapshot(PclSnap2, leveled_bookie:empty_ledgercache()),
|
||||||
?assertMatch(false, pcl_checksequencenumber(PclSnap2,
|
?assertMatch(false, pcl_checksequencenumber(PclSnap2,
|
||||||
|
@ -1204,6 +1199,12 @@ simple_server_test() ->
|
||||||
"Key0001",
|
"Key0001",
|
||||||
null},
|
null},
|
||||||
4005)),
|
4005)),
|
||||||
|
|
||||||
|
io:format("Snap2 B2 K2 ~w~n",
|
||||||
|
[pcl_fetch(PclSnap2, {o, "Bucket0002", "Key0002", null})]),
|
||||||
|
io:format("r B2 K2 ~w~n",
|
||||||
|
[pcl_fetch(PCLr, {o, "Bucket0002", "Key0002", null})]),
|
||||||
|
|
||||||
?assertMatch(true, pcl_checksequencenumber(PclSnap2,
|
?assertMatch(true, pcl_checksequencenumber(PclSnap2,
|
||||||
{o,
|
{o,
|
||||||
"Bucket0002",
|
"Bucket0002",
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue