Experiment

Perhaps the results ar epaging related - change settings to hold less in
memory and see

src/leveled_penciller.erl

@@ -197,8 +197,8 @@
 -define(CURRENT_FILEX, "crr").
 -define(PENDING_FILEX, "pnd").
 -define(MEMTABLE, mem).
--define(MAX_TABLESIZE, 28000). % This is less than max - but COIN_SIDECOUNT
--define(SUPER_MAX_TABLE_SIZE, 40000).
+-define(MAX_TABLESIZE, 16000). % This is less than max - but COIN_SIDECOUNT
+-define(SUPER_MAX_TABLE_SIZE, 22000).
 -define(PROMPT_WAIT_ONL0, 5).
 -define(WORKQUEUE_BACKLOG_TOLERANCE, 4).
 -define(COIN_SIDECOUNT, 5).

src/leveled_sft.erl

@@ -178,7 +178,7 @@
 -define(DWORD_SIZE, 8).
 -define(CURRENT_VERSION, {0,1}).
 -define(SLOT_COUNT, 256).
--define(SLOT_GROUPWRITE_COUNT, 64).
+-define(SLOT_GROUPWRITE_COUNT, 16).
 -define(BLOCK_SIZE, 32).
 -define(BLOCK_COUNT, 4).
 -define(FOOTERPOS_HEADERPOS, 2).
@@ -188,7 +188,7 @@
 -define(COMPRESSION_LEVEL, 1).
 -define(HEADER_LEN, 56).
 -define(ITERATOR_SCANWIDTH, 1).
--define(MERGE_SCANWIDTH, 32).
+-define(MERGE_SCANWIDTH, 8).
 -define(BLOOM_WIDTH, 48).
 -define(DELETE_TIMEOUT, 10000).
 -define(MAX_KEYS, ?SLOT_COUNT * ?BLOCK_COUNT * ?BLOCK_SIZE).