Add bucket whitelist test
This commit is contained in:
martinsumner
parent
1af9ac56dc
commit
439bf8c3b8
1 changed files with 181 additions and 4 deletions
|
|
@ -6,14 +6,16 @@
|
||||||
many_put_compare/1,
|
many_put_compare/1,
|
||||||
index_compare/1,
|
index_compare/1,
|
||||||
recent_aae_noaae/1,
|
recent_aae_noaae/1,
|
||||||
recent_aae_allaae/1
|
recent_aae_allaae/1,
|
||||||
|
recent_aae_bucketaae/1
|
||||||
]).
|
]).
|
||||||
|
|
||||||
all() -> [
|
all() -> [
|
||||||
many_put_compare,
|
many_put_compare,
|
||||||
index_compare,
|
index_compare,
|
||||||
recent_aae_noaae,
|
recent_aae_noaae,
|
||||||
recent_aae_allaae
|
recent_aae_allaae,
|
||||||
|
recent_aae_bucketaae
|
||||||
].
|
].
|
||||||
|
|
||||||
-define(LMD_FORMAT, "~4..0w~2..0w~2..0w~2..0w~2..0w").
|
-define(LMD_FORMAT, "~4..0w~2..0w~2..0w~2..0w~2..0w").
|
||||||
|
|
@ -627,10 +629,185 @@ recent_aae_allaae(_Config) ->
|
||||||
ok = leveled_bookie:book_close(Book2D).
|
ok = leveled_bookie:book_close(Book2D).
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
recent_aae_bucketaae(_Config) ->
|
||||||
|
% Configure AAE to work only on a single whitelisted bucket
|
||||||
|
% Confirm that we can spot a delta in this bucket, but not
|
||||||
|
% in another bucket
|
||||||
|
|
||||||
|
TreeSize = small,
|
||||||
|
% SegmentCount = 256 * 256,
|
||||||
|
UnitMins = 2,
|
||||||
|
AAE = {[<<"Bucket">>], 60, UnitMins},
|
||||||
|
|
||||||
|
% Test requires multiple different databases, so want to mount them all
|
||||||
|
% on individual file paths
|
||||||
|
RootPathA = testutil:reset_filestructure("testA"),
|
||||||
|
RootPathB = testutil:reset_filestructure("testB"),
|
||||||
|
RootPathC = testutil:reset_filestructure("testC"),
|
||||||
|
RootPathD = testutil:reset_filestructure("testD"),
|
||||||
|
StartOptsA = aae_startopts(RootPathA, AAE),
|
||||||
|
StartOptsB = aae_startopts(RootPathB, AAE),
|
||||||
|
StartOptsC = aae_startopts(RootPathC, AAE),
|
||||||
|
StartOptsD = aae_startopts(RootPathD, AAE),
|
||||||
|
|
||||||
|
% Book1A to get all objects
|
||||||
|
{ok, Book1A} = leveled_bookie:book_start(StartOptsA),
|
||||||
|
% Book1B/C/D will have objects partitioned across it
|
||||||
|
{ok, Book1B} = leveled_bookie:book_start(StartOptsB),
|
||||||
|
{ok, Book1C} = leveled_bookie:book_start(StartOptsC),
|
||||||
|
{ok, Book1D} = leveled_bookie:book_start(StartOptsD),
|
||||||
|
|
||||||
|
{B1, K1, V1, S1, MD} = {<<"Bucket">>,
|
||||||
|
"Key1.1.4567.4321",
|
||||||
|
"Value1",
|
||||||
|
[],
|
||||||
|
[{"MDK1", "MDV1"}]},
|
||||||
|
{TestObject, TestSpec} = testutil:generate_testobject(B1, K1, V1, S1, MD),
|
||||||
|
|
||||||
|
SW_StartLoad = os:timestamp(),
|
||||||
|
|
||||||
|
ok = testutil:book_riakput(Book1A, TestObject, TestSpec),
|
||||||
|
ok = testutil:book_riakput(Book1B, TestObject, TestSpec),
|
||||||
|
testutil:check_forobject(Book1A, TestObject),
|
||||||
|
testutil:check_forobject(Book1B, TestObject),
|
||||||
|
|
||||||
|
{TicTacTreeJoined, TicTacTreeFull, EmptyTree, LMDIndexes} =
|
||||||
|
load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D,
|
||||||
|
SW_StartLoad, TreeSize, UnitMins,
|
||||||
|
false, <<"Bucket">>),
|
||||||
|
% Go compare! Also confirm we're not comparing empty trees
|
||||||
|
DL1_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull,
|
||||||
|
TicTacTreeJoined),
|
||||||
|
|
||||||
|
DL1_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree),
|
||||||
|
true = DL1_0 == [],
|
||||||
|
true = length(DL1_1) > 100,
|
||||||
|
|
||||||
|
ok = leveled_bookie:book_close(Book1A),
|
||||||
|
ok = leveled_bookie:book_close(Book1B),
|
||||||
|
ok = leveled_bookie:book_close(Book1C),
|
||||||
|
ok = leveled_bookie:book_close(Book1D),
|
||||||
|
|
||||||
|
% Book2A to get all objects
|
||||||
|
{ok, Book2A} = leveled_bookie:book_start(StartOptsA),
|
||||||
|
% Book2B/C/D will have objects partitioned across it
|
||||||
|
{ok, Book2B} = leveled_bookie:book_start(StartOptsB),
|
||||||
|
{ok, Book2C} = leveled_bookie:book_start(StartOptsC),
|
||||||
|
{ok, Book2D} = leveled_bookie:book_start(StartOptsD),
|
||||||
|
|
||||||
|
% Change the value for a key in another bucket
|
||||||
|
% If we get trees for this period, no difference should be found
|
||||||
|
|
||||||
|
V2 = "Value2",
|
||||||
|
{TestObject2, TestSpec2} =
|
||||||
|
testutil:generate_testobject(<<"NotBucket">>, K1, V2, S1, MD),
|
||||||
|
|
||||||
|
New_startTS2 = os:timestamp(),
|
||||||
|
|
||||||
|
ok = testutil:book_riakput(Book2B, TestObject2, TestSpec2),
|
||||||
|
testutil:check_forobject(Book2B, TestObject2),
|
||||||
|
testutil:check_forobject(Book2A, TestObject),
|
||||||
|
|
||||||
|
New_endTS2 = os:timestamp(),
|
||||||
|
NewLMDIndexes2 = determine_lmd_indexes(New_startTS2, New_endTS2, UnitMins),
|
||||||
|
{TicTacTreeJoined2, TicTacTreeFull2, _EmptyTree, NewLMDIndexes2} =
|
||||||
|
load_and_check_recentaae(Book2A, Book2B, Book2C, Book2D,
|
||||||
|
New_startTS2, TreeSize, UnitMins,
|
||||||
|
NewLMDIndexes2, <<"Bucket">>),
|
||||||
|
DL2_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull2,
|
||||||
|
TicTacTreeJoined2),
|
||||||
|
true = length(DL2_0) == 0,
|
||||||
|
|
||||||
|
% Now create an object that is a change to an existing key in the
|
||||||
|
% monitored bucket. A differrence should be found
|
||||||
|
|
||||||
|
{TestObject3, TestSpec3} =
|
||||||
|
testutil:generate_testobject(B1, K1, V2, S1, MD),
|
||||||
|
|
||||||
|
New_startTS3 = os:timestamp(),
|
||||||
|
|
||||||
|
ok = testutil:book_riakput(Book2B, TestObject3, TestSpec3),
|
||||||
|
testutil:check_forobject(Book2B, TestObject3),
|
||||||
|
testutil:check_forobject(Book2A, TestObject),
|
||||||
|
|
||||||
|
New_endTS3 = os:timestamp(),
|
||||||
|
NewLMDIndexes3 = determine_lmd_indexes(New_startTS3, New_endTS3, UnitMins),
|
||||||
|
{TicTacTreeJoined3, TicTacTreeFull3, _EmptyTree, NewLMDIndexes3} =
|
||||||
|
load_and_check_recentaae(Book2A, Book2B, Book2C, Book2D,
|
||||||
|
New_startTS3, TreeSize, UnitMins,
|
||||||
|
NewLMDIndexes3, <<"Bucket">>),
|
||||||
|
DL3_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull3,
|
||||||
|
TicTacTreeJoined3),
|
||||||
|
|
||||||
|
% DL2_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree),
|
||||||
|
true = length(DL3_0) == 1,
|
||||||
|
|
||||||
|
% Find the dirty segment, and use that to find the dirty key
|
||||||
|
%
|
||||||
|
% Note that unlike when monitoring $all, fold_keys can be used as there
|
||||||
|
% is no need to return the Bucket (as hte bucket is known)
|
||||||
|
|
||||||
|
[DirtySeg] = DL3_0,
|
||||||
|
TermPrefix = string:right(integer_to_list(DirtySeg), 8, $0),
|
||||||
|
|
||||||
|
LMDSegFolder =
|
||||||
|
fun(LMD, {Acc, Bookie}) ->
|
||||||
|
IdxLMD = list_to_binary("$aae." ++ LMD ++ "_bin"),
|
||||||
|
IdxQ1 =
|
||||||
|
{index_query,
|
||||||
|
<<"Bucket">>,
|
||||||
|
{fun testutil:foldkeysfun/3, []},
|
||||||
|
{IdxLMD,
|
||||||
|
list_to_binary(TermPrefix ++ "."),
|
||||||
|
list_to_binary(TermPrefix ++ "|")},
|
||||||
|
{true, undefined}},
|
||||||
|
{async, IdxFolder} =
|
||||||
|
leveled_bookie:book_returnfolder(Bookie, IdxQ1),
|
||||||
|
{Acc ++ IdxFolder(), Bookie}
|
||||||
|
end,
|
||||||
|
{KeysTerms2A, _} = lists:foldl(LMDSegFolder,
|
||||||
|
{[], Book2A},
|
||||||
|
lists:usort(LMDIndexes ++ NewLMDIndexes3)),
|
||||||
|
true = length(KeysTerms2A) >= 1,
|
||||||
|
|
||||||
|
{KeysTerms2B, _} = lists:foldl(LMDSegFolder,
|
||||||
|
{[], Book2B},
|
||||||
|
lists:usort(LMDIndexes ++ NewLMDIndexes3)),
|
||||||
|
{KeysTerms2C, _} = lists:foldl(LMDSegFolder,
|
||||||
|
{[], Book2C},
|
||||||
|
lists:usort(LMDIndexes ++ NewLMDIndexes3)),
|
||||||
|
{KeysTerms2D, _} = lists:foldl(LMDSegFolder,
|
||||||
|
{[], Book2D},
|
||||||
|
lists:usort(LMDIndexes ++ NewLMDIndexes3)),
|
||||||
|
|
||||||
|
KeysTerms2Joined = KeysTerms2B ++ KeysTerms2C ++ KeysTerms2D,
|
||||||
|
DeltaX = lists:subtract(KeysTerms2A, KeysTerms2Joined),
|
||||||
|
DeltaY = lists:subtract(KeysTerms2Joined, KeysTerms2A),
|
||||||
|
|
||||||
|
io:format("DeltaX ~w~n", [DeltaX]),
|
||||||
|
io:format("DeltaY ~w~n", [DeltaY]),
|
||||||
|
|
||||||
|
true = length(DeltaX) == 0, % This hasn't seen any extra changes
|
||||||
|
true = length(DeltaY) == 1, % This has seen an extra change
|
||||||
|
[{_, K1}] = DeltaY,
|
||||||
|
|
||||||
|
ok = leveled_bookie:book_close(Book2A),
|
||||||
|
ok = leveled_bookie:book_close(Book2B),
|
||||||
|
ok = leveled_bookie:book_close(Book2C),
|
||||||
|
ok = leveled_bookie:book_close(Book2D).
|
||||||
|
|
||||||
|
|
||||||
load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D,
|
load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D,
|
||||||
SW_StartLoad, TreeSize, UnitMins,
|
SW_StartLoad, TreeSize, UnitMins,
|
||||||
LMDIndexes_Loaded) ->
|
LMDIndexes_Loaded) ->
|
||||||
|
load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D,
|
||||||
|
SW_StartLoad, TreeSize, UnitMins,
|
||||||
|
LMDIndexes_Loaded, <<"$all">>).
|
||||||
|
|
||||||
|
load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D,
|
||||||
|
SW_StartLoad, TreeSize, UnitMins,
|
||||||
|
LMDIndexes_Loaded, Bucket) ->
|
||||||
LMDIndexes =
|
LMDIndexes =
|
||||||
case LMDIndexes_Loaded of
|
case LMDIndexes_Loaded of
|
||||||
false ->
|
false ->
|
||||||
|
|
@ -677,7 +854,7 @@ load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D,
|
||||||
ST = <<"0">>,
|
ST = <<"0">>,
|
||||||
ET = <<"A">>,
|
ET = <<"A">>,
|
||||||
Q = {tictactree_idx,
|
Q = {tictactree_idx,
|
||||||
{<<"$all">>,
|
{Bucket,
|
||||||
list_to_binary("$aae." ++ LMD ++ "_bin"),
|
list_to_binary("$aae." ++ LMD ++ "_bin"),
|
||||||
ST,
|
ST,
|
||||||
ET},
|
ET},
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue