Two memory management helpers

Two helpers for memory management:

1 - a scan over the cdb file may lead to a lot of binary references being made.  So force a GC fater the scan.

2 - the penciller files contain slots that will be frequently read - so advice the page cache to pre-load them on startup.

This is in response to unexpected memory mangement issues in a potentially non-conventional setup - where the erlang VM held a lot of memory (that could be GC'd , in preference to the page cache - and consequently disk I/O and request latency were higher than expected.
This commit is contained in:
Martin Sumner 2019-07-15 13:44:39 +01:00
parent b2d4d766cd
commit f8b3101a3a
3 changed files with 57 additions and 4 deletions

View file

@ -1301,6 +1301,7 @@ scan_over_file(Handle, Position, FilterFun, Output, LastKey) ->
end,
% Bring file back to that position
{ok, Position} = file:position(Handle, {bof, Position}),
garbage_collect(),
{eof, Output};
{Key, ValueAsBin, KeyLength, ValueLength} ->
NewPosition = case Key of
@ -1316,10 +1317,12 @@ scan_over_file(Handle, Position, FilterFun, Output, LastKey) ->
Output,
fun extract_valueandsize/1) of
{stop, UpdOutput} ->
garbage_collect(),
{Position, UpdOutput};
{loop, UpdOutput} ->
case NewPosition of
eof ->
garbage_collect(),
{eof, UpdOutput};
_ ->
scan_over_file(Handle,

View file

@ -1277,6 +1277,7 @@ open_reader(Filename) ->
<<FileVersion:8/integer,
SlotsLength:32/integer,
SummaryLength:32/integer>> = Lengths,
ok = file:advise(Handle, 9, SlotsLength, will_need),
{ok, SummaryBin} = file:pread(Handle, SlotsLength + 9, SummaryLength),
{Handle, FileVersion, SummaryBin}.