Mas p401 coverage (#404)

* refactor leveled_sst from gen_fsm to gen_statem

* format_status/2 takes State and State Data
but this function is deprecated... put in for backward compatibility

* refactor leveled_cdb from gen_fsm to gen_statem

* disable irrelevant warning ignorer

* Remove unnecessary code paths

Only support messages, especially info messages, where they are possible.

* Mas i1820 offlinedeserialisation cbo (#403)

* Log report GC Info by manifest level

* Hibernate on range query

If Block Index Cache is not full, and we're not yielding

* Spawn to deserialise blocks offline

Hypothesis is that the growth in the heap necessary due to continual term_to_binary calls to deserialise blocks is wasting memory - so do this memory-intensive task in a short-lived process.

* Start with hibernate_after option

* Always build BIC

Testing indicates that the BIC itself is not a primary memory issue - the primary issue is due to a lack of garbage collection and a growing heap.

This change enhances the patch to offline serialisation so that:
- get_sqn & get_kv are standardised to build the BIC, and hibernate when it is built.
- the offline PId is linked to crash this process on failure (as would happen now).

* Standardise spawning for check_block/3

Now deserialise in both parts of the code.

* Only spawn for check_block if cache not full

* Update following review

* Standardise formatting

Make test more reliable.  Show no new compaction after third compaction.

* Update comments

---------

Co-authored-by: Thomas Arts <thomas.arts@quviq.com>
This commit is contained in:
Martin Sumner 2023-03-13 11:46:08 +00:00 committed by GitHub
parent e06d2a538f
commit 3d3d284805
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 1255 additions and 1253 deletions

View file

@ -2,7 +2,6 @@
{erl_opts, {erl_opts,
[warnings_as_errors, [warnings_as_errors,
{platform_define, "^2[0-5]{1}", fsm_deprecated},
{platform_define, "^2[2-5]{1}", if_check}]}. {platform_define, "^2[2-5]{1}", if_check}]}.
{xref_checks, [undefined_function_calls,undefined_functions]}. {xref_checks, [undefined_function_calls,undefined_functions]}.

File diff suppressed because it is too large Load diff

View file

@ -168,6 +168,8 @@
{info, <<"At level=~w file_count=~w avg_mem=~w file with most memory fn=~s p=~w mem=~w">>}, {info, <<"At level=~w file_count=~w avg_mem=~w file with most memory fn=~s p=~w mem=~w">>},
pc024 => pc024 =>
{info, <<"Grooming compaction picked file with tomb_count=~w">>}, {info, <<"Grooming compaction picked file with tomb_count=~w">>},
pc025 =>
{info, <<"At level=~w file_count=~w average words for heap_block_size=~w heap_size=~w recent_size=~w bin_vheap_size=~w">>},
pm002 => pm002 =>
{info, <<"Completed dump of L0 cache to list of l0cache_size=~w">>}, {info, <<"Completed dump of L0 cache to list of l0cache_size=~w">>},
sst03 => sst03 =>

View file

@ -200,11 +200,15 @@ handle_work(
list(leveled_pmanifest:manifest_entry())}. list(leveled_pmanifest:manifest_entry())}.
merge(SrcLevel, Manifest, RootPath, OptsSST) -> merge(SrcLevel, Manifest, RootPath, OptsSST) ->
case leveled_pmanifest:report_manifest_level(Manifest, SrcLevel + 1) of case leveled_pmanifest:report_manifest_level(Manifest, SrcLevel + 1) of
{0, 0, undefined} -> {0, 0, undefined, 0, 0, 0, 0} ->
ok; ok;
{FCnt, AvgMem, {MaxFN, MaxP, MaxMem}} -> {FCnt, MnMem, {MaxFN, MaxP, MaxMem}, MnHBS, MnHS, MnLHS, MnBVHS} ->
leveled_log:log(pc023, leveled_log:log(
[SrcLevel + 1, FCnt, AvgMem, MaxFN, MaxP, MaxMem]) pc023,
[SrcLevel + 1, FCnt, MnMem, MaxFN, MaxP, MaxMem]),
leveled_log:log(
pc025,
[SrcLevel + 1, FCnt, MnHBS, MnHS, MnLHS, MnBVHS])
end, end,
SelectMethod = SelectMethod =
case leveled_rand:uniform(100) of case leveled_rand:uniform(100) of

View file

@ -498,7 +498,7 @@ pcl_workforclerk(Pid) ->
-spec pcl_manifestchange(pid(), leveled_pmanifest:manifest()) -> ok. -spec pcl_manifestchange(pid(), leveled_pmanifest:manifest()) -> ok.
%% @doc %% @doc
%% Provide a manifest record (i.e. the output of the leveled_pmanifest module) %% Provide a manifest record (i.e. the output of the leveled_pmanifest module)
%% that is required to beocme the new manifest. %% that is required to become the new manifest.
pcl_manifestchange(Pid, Manifest) -> pcl_manifestchange(Pid, Manifest) ->
gen_server:cast(Pid, {manifest_change, Manifest}). gen_server:cast(Pid, {manifest_change, Manifest}).

View file

@ -266,11 +266,16 @@ remove_manifest(RootPath, GC_SQN) ->
end. end.
-spec report_manifest_level(manifest(), non_neg_integer()) -> -spec report_manifest_level(
{non_neg_integer(), manifest(), non_neg_integer()) ->
non_neg_integer(), {non_neg_integer(),
{string(), pid(), non_neg_integer()} | non_neg_integer(),
undefined}. {string(), pid(), non_neg_integer()} |
undefined,
non_neg_integer(),
non_neg_integer(),
non_neg_integer(),
non_neg_integer()}.
%% @doc %% @doc
%% Report on a level in the manifest %% Report on a level in the manifest
%% - How many files in the level %% - How many files in the level
@ -287,7 +292,7 @@ report_manifest_level(Manifest, LevelIdx) ->
{leveled_tree:tsize(Level), leveled_tree:to_list(Level)} {leveled_tree:tsize(Level), leveled_tree:to_list(Level)}
end, end,
AccMemFun = AccMemFun =
fun(MaybeME, {MemAcc, Max}) -> fun(MaybeME, {MemAcc, Max, HBSAcc, HSAcc, LHSAcc, BVHSAcc}) ->
ME = get_manifest_entry(MaybeME), ME = get_manifest_entry(MaybeME),
P = ME#manifest_entry.owner, P = ME#manifest_entry.owner,
{memory, PM} = process_info(P, memory), {memory, PM} = process_info(P, memory),
@ -298,15 +303,26 @@ report_manifest_level(Manifest, LevelIdx) ->
_ -> _ ->
{ME#manifest_entry.filename, P, PM} {ME#manifest_entry.filename, P, PM}
end, end,
{MemAcc + PM, UpdMax} {garbage_collection_info, GCI} =
process_info(P, garbage_collection_info),
HBS = proplists:get_value(heap_block_size, GCI),
HS = proplists:get_value(heap_size, GCI),
LHS = proplists:get_value(recent_size, GCI),
BVHS = proplists:get_value(bin_vheap_size, GCI),
{MemAcc + PM, UpdMax,
HBSAcc + HBS, HSAcc + HS, LHSAcc + LHS, BVHSAcc + BVHS}
end, end,
case LevelSize of case LevelSize of
0 -> 0 ->
{0, 0, undefined}; {0, 0, undefined, 0, 0, 0, 0};
_ -> _ ->
{TotalMem, BiggestMem} = {TotalMem, BiggestMem, TotalHBS, TotalHS, TotalLHS, TotalBVBS} =
lists:foldl(AccMemFun, {0, undefined}, LevelList), lists:foldl(AccMemFun, {0, undefined, 0, 0, 0, 0}, LevelList),
{LevelSize, TotalMem div LevelSize, BiggestMem} {LevelSize, TotalMem div LevelSize, BiggestMem,
TotalHBS div LevelSize,
TotalHS div LevelSize,
TotalLHS div LevelSize,
TotalBVBS div LevelSize}
end. end.

File diff suppressed because it is too large Load diff

View file

@ -68,6 +68,7 @@ replace_everything(_Config) ->
testutil:put_altered_indexed_objects(Book1, BKT, KSpcL1), testutil:put_altered_indexed_objects(Book1, BKT, KSpcL1),
ok = testutil:check_indexed_objects(Book1, BKT, KSpcL2, V2), ok = testutil:check_indexed_objects(Book1, BKT, KSpcL2, V2),
compact_and_wait(Book1, 1000), compact_and_wait(Book1, 1000),
compact_and_wait(Book1, 1000),
{ok, FileList1} = file:list_dir(CompPath), {ok, FileList1} = file:list_dir(CompPath),
io:format("Number of files after compaction ~w~n", [length(FileList1)]), io:format("Number of files after compaction ~w~n", [length(FileList1)]),
compact_and_wait(Book1, 1000), compact_and_wait(Book1, 1000),