umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Add test with fold_heads

Browse source 
Build the AAE tree equally using fold_heads.  This is a pre-cursor to running this within Riak.

In part this leans on some of the work done to improve standard Riak AAE with leveled.  When rebuilding the standard AAE store only the head is required, and so this process was switched in riak_kv_sweeper to make a fold_heads request if supported by the backend.

The head response is a proxy object, which when loaded into a riak_object will allow for access to object metadata, but will use the passed function if access to object contents is requested.
This commit is contained in:
Martin Sumner 2017-08-05 16:43:03 +01:00
parent 22709abcd6
commit dd20132892
1 changed files with 175 additions and 143 deletions
Download patch file Download diff file Expand all files Collapse all files

318
test/end_to_end/tictac_SUITE.erl
View file

@ -21,6 +21,8 @@ all() -> [
].
-define(LMD_FORMAT, "~4..0w~2..0w~2..0w~2..0w~2..0w").
-define(V1_VERS, 1).
-define(MAGIC, 53). % riak_kv -> riak_object
many_put_compare(_Config) ->
TreeSize = small,
@ -31,7 +33,7 @@ many_put_compare(_Config) ->
RootPathB = testutil:reset_filestructure("testB"),
RootPathC = testutil:reset_filestructure("testC"),
RootPathD = testutil:reset_filestructure("testD"),
% Start the first database, load a test object, close it, start it again
StartOpts1 = [{root_path, RootPathA},
{max_pencillercachesize, 16000},
@ -52,11 +54,11 @@ many_put_compare(_Config) ->
{sync_strategy, testutil:sync_strategy()}],
{ok, Bookie2} = leveled_bookie:book_start(StartOpts2),
testutil:check_forobject(Bookie2, TestObject),
% Generate 200K objects to be sued within the test, and load them into
% the first store (outputting the generated objects as a list of lists)
% to be used elsewhere
GenList = [2, 20002, 40002, 60002, 80002,
100002, 120002, 140002, 160002, 180002],
CLs = testutil:load_objects(20000,
@ -65,20 +67,20 @@ many_put_compare(_Config) ->
TestObject,
fun testutil:generate_smallobjects/2,
20000),
% Start a new store, and load the same objects (except fot the original
% test object) into this store
StartOpts3 = [{root_path, RootPathB},
{max_journalsize, 200000000},
{max_pencillercachesize, 16000},
{sync_strategy, testutil:sync_strategy()}],
{ok, Bookie3} = leveled_bookie:book_start(StartOpts3),
lists:foreach(fun(ObjL) -> testutil:riakload(Bookie3, ObjL) end, CLs),
% Now run a tictac query against both stores to see th extent to which
% Now run a tictac query against both stores to see the extent to which
% state between stores is consistent
TicTacQ = {tictactree_obj,
{o_rkv, "Bucket", null, null, false},
TreeSize,
@ -99,15 +101,45 @@ many_put_compare(_Config) ->
[timer:now_diff(os:timestamp(), SWC0)]),
io:format("Tree comparison shows ~w different leaves~n",
[length(SegList0)]),
AltList = leveled_tictac:find_dirtyleaves(TreeA,
leveled_tictac:new_tree(0)),
AltList =
leveled_tictac:find_dirtyleaves(TreeA,
leveled_tictac:new_tree(0, TreeSize)),
io:format("Tree comparison shows ~w altered leaves~n",
[length(AltList)]),
true = length(SegList0) == 1,
true = length(SegList0) == 1,
% only the test object should be different
true = length(AltList) > 10000,
true = length(AltList) > 10000,
% check there are a significant number of differences from empty
% Now run the same query by putting the tree-building responsibility onto
% the fold_objects_fun
HashFun =
fun(_Key, Value) ->
{proxy_object, HeadBin, _Size, _FetchFun} = binary_to_term(Value),
<<?MAGIC:8/integer, ?V1_VERS:8/integer, VclockLen:32/integer,
Rest/binary>> = HeadBin,
<<VclockBin:VclockLen/binary, _NotNeeded/binary>> = Rest,
erlang:phash2(lists:sort(binary_to_term(VclockBin)))
end,
FoldObjectsFun =
fun(_Bucket, Key, Value, Acc) ->
leveled_tictac:add_kv(Acc, Key, Value, HashFun)
end,
FoldQ = {foldheads_bybucket,
o_rkv,
"Bucket",
{FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)}},
{async, TreeAObjFolder} = leveled_bookie:book_returnfolder(Bookie2, FoldQ),
SWB0Obj = os:timestamp(),
TreeAObj = TreeAObjFolder(),
io:format("Build tictac tree via object foldwith 200K objects in ~w~n",
[timer:now_diff(os:timestamp(), SWB0Obj)]),
SegList0Obj = leveled_tictac:find_dirtyleaves(TreeA, TreeAObj),
io:format("Fold object compared with tictac fold has ~w diffs~n",
[length(SegList0Obj)]),
true = length(SegList0Obj) == 0,
%% Finding differing keys
FoldKeysFun =
fun(SegListToFind) ->
fun(_B, K, Acc) ->
@ -129,14 +161,14 @@ many_put_compare(_Config) ->
[length(SegKeyList),
length(SegList0),
timer:now_diff(os:timestamp(), SWSKL0)]),
true = length(SegKeyList) >= 1,
true = length(SegKeyList) < 10,
true = lists:member("Key1.1.4567.4321", SegKeyList),
% Now remove the object which represents the difference between these
% stores and confirm that the tictac trees will now match
testutil:book_riakdelete(Bookie2, B1, K1, []),
{async, TreeAFolder0} = leveled_bookie:book_returnfolder(Bookie2, TicTacQ),
SWA1 = os:timestamp(),
@ -149,7 +181,7 @@ many_put_compare(_Config) ->
[length(SegList1)]),
true = length(SegList1) == 0,
% Removed test object so tictac trees should match
ok = testutil:book_riakput(Bookie3, TestObject, TestSpec),
{async, TreeBFolder0} = leveled_bookie:book_returnfolder(Bookie3, TicTacQ),
SWB1 = os:timestamp(),
@ -162,10 +194,10 @@ many_put_compare(_Config) ->
% Bookie 2 (compared to it being in Bookie2 not Bookie3)
ok = leveled_bookie:book_close(Bookie3),
% Replace Bookie 3 with two stores Bookie 4 and Bookie 5 where the ojects
% have been randomly split between the stores
StartOpts4 = [{root_path, RootPathC},
{max_journalsize, 200000000},
{max_pencillercachesize, 24000},
@ -176,7 +208,7 @@ many_put_compare(_Config) ->
{max_pencillercachesize, 24000},
{sync_strategy, testutil:sync_strategy()}],
{ok, Bookie5} = leveled_bookie:book_start(StartOpts5),
SplitFun =
fun(Obj) ->
case erlang:phash2(Obj) rem 2 of
@ -192,11 +224,11 @@ many_put_compare(_Config) ->
testutil:riakload(Bookie5, ObjLB)
end,
CLs),
% query both the stores, then merge the trees - the result should be the
% same as the result from the tree created aginst the store with both
% partitions
{async, TreeC0Folder} = leveled_bookie:book_returnfolder(Bookie4, TicTacQ),
{async, TreeC1Folder} = leveled_bookie:book_returnfolder(Bookie5, TicTacQ),
SWD0 = os:timestamp(),
@ -207,18 +239,18 @@ many_put_compare(_Config) ->
TreeC1 = TreeC1Folder(),
io:format("Build tictac tree with 100K objects in ~w~n",
[timer:now_diff(os:timestamp(), SWD1)]),
TreeC2 = leveled_tictac:merge_trees(TreeC0, TreeC1),
SegList3 = leveled_tictac:find_dirtyleaves(TreeC2, TreeB),
io:format("Tree comparison following delete shows ~w different leaves~n",
[length(SegList3)]),
true = length(SegList3) == 0,
ok = leveled_bookie:book_close(Bookie2),
ok = leveled_bookie:book_close(Bookie4),
ok = leveled_bookie:book_close(Bookie5).
index_compare(_Config) ->
TreeSize = small,
@ -226,7 +258,7 @@ index_compare(_Config) ->
JS = 50000000,
SS = testutil:sync_strategy(),
SegmentCount = 256 * 256,
% Test requires multiple different databases, so want to mount them all
% on individual file paths
RootPathA = testutil:reset_filestructure("testA"),
@ -239,7 +271,7 @@ index_compare(_Config) ->
{ok, Book1B} = leveled_bookie:book_start(RootPathB, LS, JS, SS),
{ok, Book1C} = leveled_bookie:book_start(RootPathC, LS, JS, SS),
{ok, Book1D} = leveled_bookie:book_start(RootPathD, LS, JS, SS),
% Generate nine lists of objects
BucketBin = list_to_binary("Bucket"),
GenMapFun =
@ -248,13 +280,13 @@ index_compare(_Config) ->
Indexes = testutil:get_randomindexes_generator(8),
testutil:generate_objects(10000, binary_uuid, [], V, Indexes)
end,
ObjLists = lists:map(GenMapFun, lists:seq(1, 9)),
% Load all nine lists into Book1A
lists:foreach(fun(ObjL) -> testutil:riakload(Book1A, ObjL) end,
ObjLists),
ObjLists),
% Split nine lists across Book1B to Book1D, three object lists in each
lists:foreach(fun(ObjL) -> testutil:riakload(Book1B, ObjL) end,
lists:sublist(ObjLists, 1, 3)),
@ -262,7 +294,7 @@ index_compare(_Config) ->
lists:sublist(ObjLists, 4, 3)),
lists:foreach(fun(ObjL) -> testutil:riakload(Book1D, ObjL) end,
lists:sublist(ObjLists, 7, 3)),
GetTicTacTreeFun =
fun(X, Bookie) ->
SW = os:timestamp(),
@ -279,18 +311,18 @@ index_compare(_Config) ->
[X, timer:now_diff(os:timestamp(), SW)]),
R
end,
% Get a TicTac tree representing one of the indexes in Bucket A
TicTacTree1_Full = GetTicTacTreeFun(1, Book1A),
TicTacTree1_P1 = GetTicTacTreeFun(1, Book1B),
TicTacTree1_P2 = GetTicTacTreeFun(1, Book1C),
TicTacTree1_P3 = GetTicTacTreeFun(1, Book1D),
% Merge the tree across the partitions
TicTacTree1_Joined = lists:foldl(fun leveled_tictac:merge_trees/2,
TicTacTree1_P1,
[TicTacTree1_P2, TicTacTree1_P3]),
% Go compare! Also check we're not comparing empty trees
DL1_0 = leveled_tictac:find_dirtyleaves(TicTacTree1_Full,
TicTacTree1_Joined),
@ -303,7 +335,7 @@ index_compare(_Config) ->
ok = leveled_bookie:book_close(Book1B),
ok = leveled_bookie:book_close(Book1C),
ok = leveled_bookie:book_close(Book1D),
% Double chekc all is well still after a restart
% Book1A to get all objects
{ok, Book2A} = leveled_bookie:book_start(RootPathA, LS, JS, SS),
@ -316,12 +348,12 @@ index_compare(_Config) ->
TicTacTree2_P1 = GetTicTacTreeFun(2, Book2B),
TicTacTree2_P2 = GetTicTacTreeFun(2, Book2C),
TicTacTree2_P3 = GetTicTacTreeFun(2, Book2D),
% Merge the tree across the partitions
TicTacTree2_Joined = lists:foldl(fun leveled_tictac:merge_trees/2,
TicTacTree2_P1,
[TicTacTree2_P2, TicTacTree2_P3]),
% Go compare! Also check we're not comparing empty trees
DL2_0 = leveled_tictac:find_dirtyleaves(TicTacTree2_Full,
TicTacTree2_Joined),
@ -329,7 +361,7 @@ index_compare(_Config) ->
DL2_1 = leveled_tictac:find_dirtyleaves(TicTacTree2_Full, EmptyTree),
true = DL2_0 == [],
true = length(DL2_1) > 100,
IdxSpc = {add, "idx2_bin", "zz999"},
{TestObj, TestSpc} = testutil:generate_testobject(BucketBin,
term_to_binary("K9.Z"),
@ -338,17 +370,17 @@ index_compare(_Config) ->
[{"MDK1", "MDV1"}]),
ok = testutil:book_riakput(Book2C, TestObj, TestSpc),
testutil:check_forobject(Book2C, TestObj),
TicTacTree3_Full = GetTicTacTreeFun(2, Book2A),
TicTacTree3_P1 = GetTicTacTreeFun(2, Book2B),
TicTacTree3_P2 = GetTicTacTreeFun(2, Book2C),
TicTacTree3_P3 = GetTicTacTreeFun(2, Book2D),
% Merge the tree across the partitions
TicTacTree3_Joined = lists:foldl(fun leveled_tictac:merge_trees/2,
TicTacTree3_P1,
[TicTacTree3_P2, TicTacTree3_P3]),
% Find all keys index, and then just the last key
IdxQ1 = {index_query,
BucketBin,
@ -357,7 +389,7 @@ index_compare(_Config) ->
{true, undefined}},
{async, IdxFolder1} = leveled_bookie:book_returnfolder(Book2C, IdxQ1),
true = IdxFolder1() >= 1,
DL_3to2B = leveled_tictac:find_dirtyleaves(TicTacTree2_P1,
TicTacTree3_P1),
DL_3to2C = leveled_tictac:find_dirtyleaves(TicTacTree2_P2,
@ -366,23 +398,23 @@ index_compare(_Config) ->
TicTacTree3_P3),
io:format("Individual tree comparison found dirty leaves of ~w ~w ~w~n",
[DL_3to2B, DL_3to2C, DL_3to2D]),
true = length(DL_3to2B) == 0,
true = length(DL_3to2C) == 1,
true = length(DL_3to2D) == 0,
% Go compare! Should find a difference in one leaf
DL3_0 = leveled_tictac:find_dirtyleaves(TicTacTree3_Full,
TicTacTree3_Joined),
io:format("Different leaves count ~w~n", [length(DL3_0)]),
true = length(DL3_0) == 1,
% Now we want to find for the {Term, Key} pairs that make up the segment
% diferrence (there should only be one)
%
% We want the database to filter on segment - so this doesn't have the
% overheads of key listing
FoldKeysIndexQFun =
fun(_Bucket, {Term, Key}, Acc) ->
Seg = leveled_tictac:get_segment(Key, SegmentCount),
@ -393,7 +425,7 @@ index_compare(_Config) ->
Acc
end
end,
MismatchQ = {index_query,
BucketBin,
{FoldKeysIndexQFun, []},
@ -403,21 +435,21 @@ index_compare(_Config) ->
{async, MMFldr_2B} = leveled_bookie:book_returnfolder(Book2B, MismatchQ),
{async, MMFldr_2C} = leveled_bookie:book_returnfolder(Book2C, MismatchQ),
{async, MMFldr_2D} = leveled_bookie:book_returnfolder(Book2D, MismatchQ),
SWSS = os:timestamp(),
SL_Joined = MMFldr_2B() ++ MMFldr_2C() ++ MMFldr_2D(),
SL_Full = MMFldr_2A(),
io:format("Segment search across both clusters took ~w~n",
[timer:now_diff(os:timestamp(), SWSS)]),
io:format("Joined SegList ~w~n", [SL_Joined]),
io:format("Full SegList ~w~n", [SL_Full]),
Diffs = lists:subtract(SL_Full, SL_Joined)
++ lists:subtract(SL_Joined, SL_Full),
io:format("Differences between lists ~w~n", [Diffs]),
% The actual difference is discovered
true = lists:member({"zz999", term_to_binary("K9.Z")}, Diffs),
% Without discovering too many others
@ -433,11 +465,11 @@ index_compare(_Config) ->
recent_aae_noaae(_Config) ->
% Starts databases with recent_aae tables, and attempt to query to fetch
% recent aae trees returns empty trees as no index entries are found.
TreeSize = small,
% SegmentCount = 256 * 256,
UnitMins = 2,
% Test requires multiple different databases, so want to mount them all
% on individual file paths
RootPathA = testutil:reset_filestructure("testA"),
@ -448,28 +480,28 @@ recent_aae_noaae(_Config) ->
StartOptsB = aae_startopts(RootPathB, false),
StartOptsC = aae_startopts(RootPathC, false),
StartOptsD = aae_startopts(RootPathD, false),
% Book1A to get all objects
{ok, Book1A} = leveled_bookie:book_start(StartOptsA),
% Book1B/C/D will have objects partitioned across it
{ok, Book1B} = leveled_bookie:book_start(StartOptsB),
{ok, Book1C} = leveled_bookie:book_start(StartOptsC),
{ok, Book1D} = leveled_bookie:book_start(StartOptsD),
{B1, K1, V1, S1, MD} = {"Bucket",
"Key1.1.4567.4321",
"Value1",
[],
[{"MDK1", "MDV1"}]},
{TestObject, TestSpec} = testutil:generate_testobject(B1, K1, V1, S1, MD),
SW_StartLoad = os:timestamp(),
ok = testutil:book_riakput(Book1A, TestObject, TestSpec),
ok = testutil:book_riakput(Book1B, TestObject, TestSpec),
testutil:check_forobject(Book1A, TestObject),
testutil:check_forobject(Book1B, TestObject),
{TicTacTreeJoined, TicTacTreeFull, EmptyTree, _LMDIndexes} =
load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D,
SW_StartLoad, TreeSize, UnitMins,
@ -477,7 +509,7 @@ recent_aae_noaae(_Config) ->
% Go compare! Also confirm we're not comparing empty trees
DL1_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull,
TicTacTreeJoined),
DL1_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree),
true = DL1_0 == [],
true = length(DL1_1) == 0,
@ -489,7 +521,7 @@ recent_aae_noaae(_Config) ->
recent_aae_allaae(_Config) ->
% Leveled is started in blacklisted mode with no buckets blacklisted.
% Leveled is started in blacklisted mode with no buckets blacklisted.
%
% A number of changes are then loaded into a store, and also partitioned
% across a separate set of three stores. A merge tree is returned from
@ -503,12 +535,12 @@ recent_aae_allaae(_Config) ->
% The segment Id found is then used in a query to find the Keys that make
% up that segment, and the delta discovered should be just that one key
% which was known to have been changed
TreeSize = small,
% SegmentCount = 256 * 256,
UnitMins = 2,
AAE = {blacklist, [], 60, UnitMins},
% Test requires multiple different databases, so want to mount them all
% on individual file paths
RootPathA = testutil:reset_filestructure("testA"),
@ -519,28 +551,28 @@ recent_aae_allaae(_Config) ->
StartOptsB = aae_startopts(RootPathB, AAE),
StartOptsC = aae_startopts(RootPathC, AAE),
StartOptsD = aae_startopts(RootPathD, AAE),
% Book1A to get all objects
{ok, Book1A} = leveled_bookie:book_start(StartOptsA),
% Book1B/C/D will have objects partitioned across it
{ok, Book1B} = leveled_bookie:book_start(StartOptsB),
{ok, Book1C} = leveled_bookie:book_start(StartOptsC),
{ok, Book1D} = leveled_bookie:book_start(StartOptsD),
{B1, K1, V1, S1, MD} = {"Bucket",
"Key1.1.4567.4321",
"Value1",
[],
[{"MDK1", "MDV1"}]},
{TestObject, TestSpec} = testutil:generate_testobject(B1, K1, V1, S1, MD),
SW_StartLoad = os:timestamp(),
ok = testutil:book_riakput(Book1A, TestObject, TestSpec),
ok = testutil:book_riakput(Book1B, TestObject, TestSpec),
testutil:check_forobject(Book1A, TestObject),
testutil:check_forobject(Book1B, TestObject),
{TicTacTreeJoined, TicTacTreeFull, EmptyTree, LMDIndexes} =
load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D,
SW_StartLoad, TreeSize, UnitMins,
@ -548,7 +580,7 @@ recent_aae_allaae(_Config) ->
% Go compare! Also confirm we're not comparing empty trees
DL1_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull,
TicTacTreeJoined),
DL1_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree),
true = DL1_0 == [],
true = length(DL1_1) > 100,
@ -557,14 +589,14 @@ recent_aae_allaae(_Config) ->
ok = leveled_bookie:book_close(Book1B),
ok = leveled_bookie:book_close(Book1C),
ok = leveled_bookie:book_close(Book1D),
% Book2A to get all objects
{ok, Book2A} = leveled_bookie:book_start(StartOptsA),
% Book2B/C/D will have objects partitioned across it
{ok, Book2B} = leveled_bookie:book_start(StartOptsB),
{ok, Book2C} = leveled_bookie:book_start(StartOptsC),
{ok, Book2D} = leveled_bookie:book_start(StartOptsD),
{TicTacTreeJoined, TicTacTreeFull, EmptyTree, LMDIndexes} =
load_and_check_recentaae(Book2A, Book2B, Book2C, Book2D,
SW_StartLoad, TreeSize, UnitMins,
@ -572,21 +604,21 @@ recent_aae_allaae(_Config) ->
% Go compare! Also confirm we're not comparing empty trees
DL1_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull,
TicTacTreeJoined),
DL1_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree),
true = DL1_0 == [],
true = length(DL1_1) > 100,
V2 = "Value2",
{TestObject2, TestSpec2} =
testutil:generate_testobject(B1, K1, V2, S1, MD),
New_startTS = os:timestamp(),
ok = testutil:book_riakput(Book2B, TestObject2, TestSpec2),
testutil:check_forobject(Book2B, TestObject2),
testutil:check_forobject(Book2A, TestObject),
New_endTS = os:timestamp(),
NewLMDIndexes = determine_lmd_indexes(New_startTS, New_endTS, UnitMins),
{TicTacTreeJoined2, TicTacTreeFull2, _EmptyTree, NewLMDIndexes} =
@ -595,10 +627,10 @@ recent_aae_allaae(_Config) ->
NewLMDIndexes),
DL2_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull2,
TicTacTreeJoined2),
% DL2_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree),
true = length(DL2_0) == 1,
[DirtySeg] = DL2_0,
TermPrefix = string:right(integer_to_list(DirtySeg), 8, $0),
@ -631,18 +663,18 @@ recent_aae_allaae(_Config) ->
{KeysTerms2D, _} = lists:foldl(LMDSegFolder,
{[], Book2D},
lists:usort(LMDIndexes ++ NewLMDIndexes)),
KeysTerms2Joined = KeysTerms2B ++ KeysTerms2C ++ KeysTerms2D,
DeltaX = lists:subtract(KeysTerms2A, KeysTerms2Joined),
DeltaY = lists:subtract(KeysTerms2Joined, KeysTerms2A),
io:format("DeltaX ~w~n", [DeltaX]),
io:format("DeltaY ~w~n", [DeltaY]),
true = length(DeltaX) == 0, % This hasn't seen any extra changes
true = length(DeltaY) == 1, % This has seen an extra change
[{_, {B1, K1}}] = DeltaY,
ok = leveled_bookie:book_close(Book2A),
ok = leveled_bookie:book_close(Book2B),
ok = leveled_bookie:book_close(Book2C),
@ -654,12 +686,12 @@ recent_aae_bucketaae(_Config) ->
% Configure AAE to work only on a single whitelisted bucket
% Confirm that we can spot a delta in this bucket, but not
% in another bucket
TreeSize = small,
% SegmentCount = 256 * 256,
UnitMins = 2,
AAE = {whitelist, [<<"Bucket">>], 60, UnitMins},
% Test requires multiple different databases, so want to mount them all
% on individual file paths
RootPathA = testutil:reset_filestructure("testA"),
@ -670,28 +702,28 @@ recent_aae_bucketaae(_Config) ->
StartOptsB = aae_startopts(RootPathB, AAE),
StartOptsC = aae_startopts(RootPathC, AAE),
StartOptsD = aae_startopts(RootPathD, AAE),
% Book1A to get all objects
{ok, Book1A} = leveled_bookie:book_start(StartOptsA),
% Book1B/C/D will have objects partitioned across it
{ok, Book1B} = leveled_bookie:book_start(StartOptsB),
{ok, Book1C} = leveled_bookie:book_start(StartOptsC),
{ok, Book1D} = leveled_bookie:book_start(StartOptsD),
{B1, K1, V1, S1, MD} = {<<"Bucket">>,
"Key1.1.4567.4321",
"Value1",
[],
[{"MDK1", "MDV1"}]},
{TestObject, TestSpec} = testutil:generate_testobject(B1, K1, V1, S1, MD),
SW_StartLoad = os:timestamp(),
ok = testutil:book_riakput(Book1A, TestObject, TestSpec),
ok = testutil:book_riakput(Book1B, TestObject, TestSpec),
testutil:check_forobject(Book1A, TestObject),
testutil:check_forobject(Book1B, TestObject),
{TicTacTreeJoined, TicTacTreeFull, EmptyTree, LMDIndexes} =
load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D,
SW_StartLoad, TreeSize, UnitMins,
@ -699,7 +731,7 @@ recent_aae_bucketaae(_Config) ->
% Go compare! Also confirm we're not comparing empty trees
DL1_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull,
TicTacTreeJoined),
DL1_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree),
true = DL1_0 == [],
true = length(DL1_1) > 100,
@ -708,27 +740,27 @@ recent_aae_bucketaae(_Config) ->
ok = leveled_bookie:book_close(Book1B),
ok = leveled_bookie:book_close(Book1C),
ok = leveled_bookie:book_close(Book1D),
% Book2A to get all objects
{ok, Book2A} = leveled_bookie:book_start(StartOptsA),
% Book2B/C/D will have objects partitioned across it
{ok, Book2B} = leveled_bookie:book_start(StartOptsB),
{ok, Book2C} = leveled_bookie:book_start(StartOptsC),
{ok, Book2D} = leveled_bookie:book_start(StartOptsD),
% Change the value for a key in another bucket
% If we get trees for this period, no difference should be found
V2 = "Value2",
{TestObject2, TestSpec2} =
testutil:generate_testobject(<<"NotBucket">>, K1, V2, S1, MD),
New_startTS2 = os:timestamp(),
ok = testutil:book_riakput(Book2B, TestObject2, TestSpec2),
testutil:check_forobject(Book2B, TestObject2),
testutil:check_forobject(Book2A, TestObject),
New_endTS2 = os:timestamp(),
NewLMDIndexes2 = determine_lmd_indexes(New_startTS2, New_endTS2, UnitMins),
{TicTacTreeJoined2, TicTacTreeFull2, _EmptyTree, NewLMDIndexes2} =
@ -738,19 +770,19 @@ recent_aae_bucketaae(_Config) ->
DL2_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull2,
TicTacTreeJoined2),
true = length(DL2_0) == 0,
% Now create an object that is a change to an existing key in the
% monitored bucket. A differrence should be found
{TestObject3, TestSpec3} =
testutil:generate_testobject(B1, K1, V2, S1, MD),
New_startTS3 = os:timestamp(),
ok = testutil:book_riakput(Book2B, TestObject3, TestSpec3),
testutil:check_forobject(Book2B, TestObject3),
testutil:check_forobject(Book2A, TestObject),
New_endTS3 = os:timestamp(),
NewLMDIndexes3 = determine_lmd_indexes(New_startTS3, New_endTS3, UnitMins),
{TicTacTreeJoined3, TicTacTreeFull3, _EmptyTree, NewLMDIndexes3} =
@ -759,15 +791,15 @@ recent_aae_bucketaae(_Config) ->
NewLMDIndexes3, <<"Bucket">>),
DL3_0 = leveled_tictac:find_dirtyleaves(TicTacTreeFull3,
TicTacTreeJoined3),
% DL2_1 = leveled_tictac:find_dirtyleaves(TicTacTreeFull, EmptyTree),
true = length(DL3_0) == 1,
% Find the dirty segment, and use that to find the dirty key
%
% Note that unlike when monitoring $all, fold_keys can be used as there
% is no need to return the Bucket (as hte bucket is known)
[DirtySeg] = DL3_0,
TermPrefix = string:right(integer_to_list(DirtySeg), 8, $0),
@ -800,18 +832,18 @@ recent_aae_bucketaae(_Config) ->
{KeysTerms2D, _} = lists:foldl(LMDSegFolder,
{[], Book2D},
lists:usort(LMDIndexes ++ NewLMDIndexes3)),
KeysTerms2Joined = KeysTerms2B ++ KeysTerms2C ++ KeysTerms2D,
DeltaX = lists:subtract(KeysTerms2A, KeysTerms2Joined),
DeltaY = lists:subtract(KeysTerms2Joined, KeysTerms2A),
io:format("DeltaX ~w~n", [DeltaX]),
io:format("DeltaY ~w~n", [DeltaY]),
true = length(DeltaX) == 0, % This hasn't seen any extra changes
true = length(DeltaY) == 1, % This has seen an extra change
[{_, K1}] = DeltaY,
ok = leveled_bookie:book_close(Book2A),
ok = leveled_bookie:book_close(Book2B),
ok = leveled_bookie:book_close(Book2C),
@ -820,21 +852,21 @@ recent_aae_bucketaae(_Config) ->
recent_aae_expiry(_Config) ->
% Proof that the index entries are indeed expired
TreeSize = small,
% SegmentCount = 256 * 256,
UnitMins = 1,
TotalMins = 2,
AAE = {blacklist, [], TotalMins, UnitMins},
% Test requires multiple different databases, so want to mount them all
% on individual file paths
RootPathA = testutil:reset_filestructure("testA"),
StartOptsA = aae_startopts(RootPathA, AAE),
% Book1A to get all objects
{ok, Book1A} = leveled_bookie:book_start(StartOptsA),
GenMapFun =
fun(_X) ->
V = testutil:get_compressiblevalue(),
@ -845,13 +877,13 @@ recent_aae_expiry(_Config) ->
V,
Indexes)
end,
ObjLists = lists:map(GenMapFun, lists:seq(1, 3)),
SW0 = os:timestamp(),
% Load all nine lists into Book1A
lists:foreach(fun(ObjL) -> testutil:riakload(Book1A, ObjL) end,
ObjLists),
ObjLists),
SW1 = os:timestamp(),
% sleep for two minutes, so all index entries will have expired
GetTicTacTreeFun =
@ -860,19 +892,19 @@ recent_aae_expiry(_Config) ->
end,
EmptyTree = leveled_tictac:new_tree(empty, TreeSize),
LMDIndexes = determine_lmd_indexes(SW0, SW1, UnitMins),
% Should get a non-empty answer to the query
TicTacTree1_Full =
lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes),
lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes),
DL3_0 = leveled_tictac:find_dirtyleaves(TicTacTree1_Full, EmptyTree),
io:format("Dirty leaves found before expiry ~w~n", [length(DL3_0)]),
true = length(DL3_0) > 0,
SecondsSinceLMD = timer:now_diff(os:timestamp(), SW0) div 1000000,
SecondsToExpiry = (TotalMins + UnitMins) * 60,
io:format("SecondsToExpiry ~w SecondsSinceLMD ~w~n",
io:format("SecondsToExpiry ~w SecondsSinceLMD ~w~n",
[SecondsToExpiry, SecondsSinceLMD]),
io:format("LMDIndexes ~w~n", [LMDIndexes]),
@ -882,17 +914,17 @@ recent_aae_expiry(_Config) ->
false ->
timer:sleep(1000)
end,
% Should now get an empty answer - all entries have expired
TicTacTree2_Full =
lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes),
lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes),
DL4_0 = leveled_tictac:find_dirtyleaves(TicTacTree2_Full, EmptyTree),
io:format("Dirty leaves found after expiry ~w~n", [length(DL4_0)]),
timer:sleep(10000),
TicTacTree3_Full =
lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes),
lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes),
DL5_0 = leveled_tictac:find_dirtyleaves(TicTacTree3_Full, EmptyTree),
io:format("Dirty leaves found after expiry plus 10s ~w~n", [length(DL5_0)]),
@ -913,7 +945,7 @@ load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D,
load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D,
SW_StartLoad, TreeSize, UnitMins,
LMDIndexes_Loaded, Bucket) ->
LMDIndexes =
LMDIndexes =
case LMDIndexes_Loaded of
false ->
% Generate nine lists of objects
@ -928,13 +960,13 @@ load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D,
V,
Indexes)
end,
ObjLists = lists:map(GenMapFun, lists:seq(1, 9)),
% Load all nine lists into Book1A
lists:foreach(fun(ObjL) -> testutil:riakload(Book1A, ObjL) end,
ObjLists),
ObjLists),
% Split nine lists across Book1B to Book1D, three object lists
% in each
lists:foreach(fun(ObjL) -> testutil:riakload(Book1B, ObjL) end,
@ -943,38 +975,38 @@ load_and_check_recentaae(Book1A, Book1B, Book1C, Book1D,
lists:sublist(ObjLists, 4, 3)),
lists:foreach(fun(ObjL) -> testutil:riakload(Book1D, ObjL) end,
lists:sublist(ObjLists, 7, 3)),
SW_EndLoad = os:timestamp(),
determine_lmd_indexes(SW_StartLoad, SW_EndLoad, UnitMins);
_ ->
LMDIndexes_Loaded
end,
EmptyTree = leveled_tictac:new_tree(empty, TreeSize),
GetTicTacTreeFun =
fun(Bookie) ->
get_tictactree_fun(Bookie, Bucket, TreeSize)
end,
% Get a TicTac tree representing one of the indexes in Bucket A
TicTacTree1_Full =
lists:foldl(GetTicTacTreeFun(Book1A), EmptyTree, LMDIndexes),
TicTacTree1_P1 =
lists:foldl(GetTicTacTreeFun(Book1B), EmptyTree, LMDIndexes),
TicTacTree1_P2 =
lists:foldl(GetTicTacTreeFun(Book1C), EmptyTree, LMDIndexes),
TicTacTree1_P3 =
lists:foldl(GetTicTacTreeFun(Book1D), EmptyTree, LMDIndexes),
% Merge the tree across the partitions
TicTacTree1_Joined = lists:foldl(fun leveled_tictac:merge_trees/2,
TicTacTree1_P1,
[TicTacTree1_P2, TicTacTree1_P3]),
{TicTacTree1_Full, TicTacTree1_Joined, EmptyTree, LMDIndexes}.
aae_startopts(RootPath, AAE) ->
LS = 2000,
@ -992,7 +1024,7 @@ determine_lmd_indexes(StartTS, EndTS, UnitMins) ->
EndDT = calendar:now_to_datetime(EndTS),
StartTimeStr = get_strtime(StartDT, UnitMins),
EndTimeStr = get_strtime(EndDT, UnitMins),
AddTimeFun =
fun(X, Acc) ->
case lists:member(EndTimeStr, Acc) of
@ -1007,10 +1039,10 @@ determine_lmd_indexes(StartTS, EndTS, UnitMins) ->
Acc ++ [get_strtime(NextDT, UnitMins)]
end
end,
lists:foldl(AddTimeFun, [StartTimeStr], lists:seq(1, 10)).
lists:foldl(AddTimeFun, [StartTimeStr], lists:seq(1, 10)).
get_strtime(DateTime, UnitMins) ->
{{Y, M, D}, {Hour, Minute, _Second}} = DateTime,
RoundMins =