245 lines
No EOL
8.5 KiB
Erlang
245 lines
No EOL
8.5 KiB
Erlang
-module(appdefined_SUITE).
|
|
-include_lib("common_test/include/ct.hrl").
|
|
-include("include/leveled.hrl").
|
|
-export([all/0]).
|
|
-export([
|
|
application_defined_tag/1,
|
|
bespoketag_recalc/1
|
|
]).
|
|
|
|
all() -> [
|
|
% application_defined_tag,
|
|
bespoketag_recalc
|
|
].
|
|
|
|
|
|
|
|
application_defined_tag(_Config) ->
|
|
T1 = os:timestamp(),
|
|
application_defined_tag_tester(40000, ?STD_TAG, [], false),
|
|
io:format("Completed with std tag in ~w ms~n",
|
|
[timer:now_diff(os:timestamp(), T1)/1000]),
|
|
|
|
T2 = os:timestamp(),
|
|
application_defined_tag_tester(40000, bespoke_tag1, [], false),
|
|
io:format("Completed with app tag but not function in ~w ms~n",
|
|
[timer:now_diff(os:timestamp(), T2)/1000]),
|
|
|
|
ExtractMDFun =
|
|
fun(Tag, Size, Obj) ->
|
|
[{hash, Hash}, {shard, Shard}, {random, Random}, {value, _V}]
|
|
= Obj,
|
|
case Tag of
|
|
bespoke_tag1 ->
|
|
{{Hash, Size, [{shard, Shard}, {random, Random}]}, []};
|
|
bespoke_tag2 ->
|
|
{{Hash, Size, [{shard, Shard}]}, [os:timestamp()]}
|
|
end
|
|
end,
|
|
|
|
T3 = os:timestamp(),
|
|
application_defined_tag_tester(40000, ?STD_TAG,
|
|
[{extract_metadata, ExtractMDFun}],
|
|
false),
|
|
io:format("Completed with std tag and override function in ~w ms~n",
|
|
[timer:now_diff(os:timestamp(), T3)/1000]),
|
|
|
|
T4 = os:timestamp(),
|
|
application_defined_tag_tester(40000, bespoke_tag1,
|
|
[{extract_metadata, ExtractMDFun}],
|
|
true),
|
|
io:format("Completed with app tag and override function in ~w ms~n",
|
|
[timer:now_diff(os:timestamp(), T4)/1000]),
|
|
|
|
|
|
T5 = os:timestamp(),
|
|
application_defined_tag_tester(40000, bespoke_tag2,
|
|
[{extract_metadata, ExtractMDFun}],
|
|
true),
|
|
io:format("Completed with app tag and override function in ~w ms~n",
|
|
[timer:now_diff(os:timestamp(), T5)/1000]).
|
|
|
|
|
|
application_defined_tag_tester(KeyCount, Tag, Functions, ExpectMD) ->
|
|
RootPath = testutil:reset_filestructure(),
|
|
StartOpts1 = [{root_path, RootPath},
|
|
{sync_strategy, testutil:sync_strategy()},
|
|
{log_level, warn},
|
|
{reload_strategy,
|
|
[{bespoke_tag1, retain}, {bespoke_tag2, retain}]},
|
|
{override_functions, Functions}],
|
|
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
|
|
Value = leveled_rand:rand_bytes(512),
|
|
MapFun =
|
|
fun(C) ->
|
|
{C, object_generator(C, Value)}
|
|
end,
|
|
CBKVL = lists:map(MapFun, lists:seq(1, KeyCount)),
|
|
|
|
PutFun =
|
|
fun({_C, {B, K, O}}) ->
|
|
R = leveled_bookie:book_put(Bookie1, B, K, O, [], Tag),
|
|
case R of
|
|
ok -> ok;
|
|
pause -> timer:sleep(100)
|
|
end
|
|
end,
|
|
lists:foreach(PutFun, CBKVL),
|
|
|
|
CheckFun =
|
|
fun(Book) ->
|
|
fun({C, {B, K, O}}) ->
|
|
{ok, O} = leveled_bookie:book_get(Book, B, K, Tag),
|
|
{ok, H} = leveled_bookie:book_head(Book, B, K, Tag),
|
|
MD = element(3, H),
|
|
case ExpectMD of
|
|
true ->
|
|
true =
|
|
{shard, C rem 10} == lists:keyfind(shard, 1, MD);
|
|
false ->
|
|
true =
|
|
undefined == MD
|
|
end
|
|
end
|
|
end,
|
|
|
|
lists:foreach(CheckFun(Bookie1), CBKVL),
|
|
|
|
ok = leveled_bookie:book_close(Bookie1),
|
|
{ok, Bookie2} = leveled_bookie:book_start(StartOpts1),
|
|
|
|
lists:foreach(CheckFun(Bookie2), CBKVL),
|
|
|
|
ok = leveled_bookie:book_close(Bookie2).
|
|
|
|
|
|
object_generator(Count, V) ->
|
|
Hash = erlang:phash2({count, V}),
|
|
Random = leveled_rand:uniform(1000),
|
|
Key = list_to_binary(leveled_util:generate_uuid()),
|
|
Bucket = <<"B">>,
|
|
{Bucket,
|
|
Key,
|
|
[{hash, Hash}, {shard, Count rem 10},
|
|
{random, Random}, {value, V}]}.
|
|
|
|
|
|
|
|
bespoketag_recalc(_Config) ->
|
|
%% Get a sensible behaviour using the recalc compaction strategy with a
|
|
%% bespoke tag
|
|
|
|
RootPath = testutil:reset_filestructure(),
|
|
B0 = <<"B0">>,
|
|
KeyCount = 7000,
|
|
|
|
ExtractMDFun =
|
|
fun(bespoke_tag, Size, Obj) ->
|
|
[{index, IL}, {value, _V}] = Obj,
|
|
{{erlang:phash2(term_to_binary(Obj)),
|
|
Size,
|
|
{index, IL}},
|
|
[os:timestamp()]}
|
|
end,
|
|
CalcIndexFun =
|
|
fun(bespoke_tag, UpdMeta, PrvMeta) ->
|
|
% io:format("UpdMeta ~w PrvMeta ~w~n", [UpdMeta, PrvMeta]),
|
|
{index, UpdIndexes} = element(3, UpdMeta),
|
|
IndexDeltas =
|
|
case PrvMeta of
|
|
not_present ->
|
|
UpdIndexes;
|
|
PrvMeta when is_tuple(PrvMeta) ->
|
|
{index, PrvIndexes} = element(3, PrvMeta),
|
|
lists:subtract(UpdIndexes, PrvIndexes)
|
|
end,
|
|
lists:map(fun(I) -> {add, <<"temp_int">>, I} end, IndexDeltas)
|
|
end,
|
|
|
|
BookOpts = [{root_path, RootPath},
|
|
{cache_size, 1000},
|
|
{max_journalobjectcount, 6000},
|
|
{max_pencillercachesize, 8000},
|
|
{sync_strategy, testutil:sync_strategy()},
|
|
{reload_strategy, [{bespoke_tag, recalc}]},
|
|
{override_functions,
|
|
[{extract_metadata, ExtractMDFun},
|
|
{diff_indexspecs, CalcIndexFun}]}],
|
|
|
|
{ok, Book1} = leveled_bookie:book_start(BookOpts),
|
|
LoadFun =
|
|
fun(Book, MustFind) ->
|
|
fun(I) ->
|
|
testutil:stdload_object(Book,
|
|
B0, list_to_binary(["A"|integer_to_list(I rem KeyCount)]),
|
|
I, erlang:phash2({value, I}),
|
|
infinity, bespoke_tag, false, MustFind)
|
|
end
|
|
end,
|
|
lists:foreach(LoadFun(Book1, false), lists:seq(1, KeyCount)),
|
|
lists:foreach(LoadFun(Book1, true), lists:seq(KeyCount + 1, KeyCount * 2)),
|
|
|
|
FoldFun =
|
|
fun(_B0, {IV0, _K0}, Acc) ->
|
|
case IV0 - 1 of
|
|
Acc ->
|
|
Acc + 1;
|
|
_Unexpected ->
|
|
% io:format("Eh? - ~w ~w~n", [Unexpected, Acc]),
|
|
Acc + 1
|
|
end
|
|
end,
|
|
|
|
CountFold =
|
|
fun(Book, CurrentCount) ->
|
|
leveled_bookie:book_indexfold(Book,
|
|
B0,
|
|
{FoldFun, 0},
|
|
{<<"temp_int">>, 0, CurrentCount},
|
|
{true, undefined})
|
|
end,
|
|
|
|
{async, FolderA} = CountFold(Book1, 2 * KeyCount),
|
|
CountA = FolderA(),
|
|
io:format("Counted double index entries ~w - everything loaded OK~n",
|
|
[CountA]),
|
|
true = 2 * KeyCount == CountA,
|
|
|
|
io:format("Before close looking for Key 999 ~w~n",
|
|
[leveled_bookie:book_head(Book1, B0, <<"A999">>, bespoke_tag)]),
|
|
|
|
ok = leveled_bookie:book_close(Book1),
|
|
|
|
{ok, Book2} = leveled_bookie:book_start(BookOpts),
|
|
io:format("After opening looking for Key 999 ~w~n",
|
|
[leveled_bookie:book_head(Book2, B0, <<"A999">>, bespoke_tag)]),
|
|
|
|
lists:foreach(LoadFun(Book2, true), lists:seq(KeyCount * 2 + 1, KeyCount * 3)),
|
|
|
|
io:format("After fresh load looking for Key 999 ~w~n",
|
|
[leveled_bookie:book_head(Book2, B0, <<"A999">>, bespoke_tag)]),
|
|
|
|
{async, FolderB} = CountFold(Book2, 3 * KeyCount),
|
|
CountB = FolderB(),
|
|
io:format("Counted triple index entries ~w - everything re-loaded~n",
|
|
[CountB]),
|
|
true = 3 * KeyCount == CountB,
|
|
|
|
testutil:compact_and_wait(Book2),
|
|
ok = leveled_bookie:book_close(Book2),
|
|
|
|
io:format("Restart from blank ledger~n"),
|
|
|
|
leveled_penciller:clean_testdir(proplists:get_value(root_path, BookOpts) ++
|
|
"/ledger"),
|
|
{ok, Book3} = leveled_bookie:book_start(BookOpts),
|
|
|
|
{async, FolderC} = CountFold(Book3, 3 * KeyCount),
|
|
CountC = FolderC(),
|
|
io:format("All index entries ~w present - recalc ok~n",
|
|
[CountC]),
|
|
true = 3 * KeyCount == CountC,
|
|
|
|
ok = leveled_bookie:book_close(Book3),
|
|
|
|
testutil:reset_filestructure(). |