umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Add ZSTD compression (#430)

Browse source 
* Add support for zstd and split compression

Add support for using zstd as an alternative to native, lz4.

Upgrade lz4 to v1.9.4 (with ARM enhancements).

Allow for split compression algorithms - i.e. use native on journal, but lz4 on ledger.

* Switch to AdRoll zstd

Development appears to be active and ongoing.  No issues running on different linux flavours.

* Use realistic bucket name

* Update README.md

* Switch branch

* Add comment following review
This commit is contained in:
Martin Sumner 2024-01-23 16:25:03 +00:00 • committed by GitHub
parent c294570bce
commit 999ce8ba5b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 156 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

20
test/end_to_end/basic_SUITE.erl
View file

@ -1036,25 +1036,36 @@ remove_journal_test(_Config) ->
ok = leveled_bookie:book_destroy(Bookie3).
many_put_fetch_switchcompression(_Config) ->
{T0, ok} =
timer:tc(fun many_put_fetch_switchcompression_tester/1, [native]),
{T1, ok} =
timer:tc(fun many_put_fetch_switchcompression_tester/1, [lz4]),
{T2, ok} =
timer:tc(fun many_put_fetch_switchcompression_tester/1, [zstd]),
io:format("Test timings native=~w lz4=~w, zstd=~w", [T0, T1, T2]).
many_put_fetch_switchcompression_tester(CompressionMethod) ->
RootPath = testutil:reset_filestructure(),
StartOpts1 = [{root_path, RootPath},
{max_pencillercachesize, 16000},
{max_journalobjectcount, 30000},
{compression_level, 3},
{sync_strategy, testutil:sync_strategy()},
{compression_method, native}],
{compression_method, native},
{ledger_compression, none}],
StartOpts2 = [{root_path, RootPath},
{max_pencillercachesize, 24000},
{max_journalobjectcount, 30000},
{sync_strategy, testutil:sync_strategy()},
{compression_method, lz4}],
{compression_method, CompressionMethod},
{ledger_compression, as_store}],
StartOpts3 = [{root_path, RootPath},
{max_pencillercachesize, 16000},
{max_journalobjectcount, 30000},
{sync_strategy, testutil:sync_strategy()},
{compression_method, none}],
{compression_method, none},
{ledger_compression, as_store}],
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
@ -1173,7 +1184,6 @@ many_put_fetch_switchcompression(_Config) ->
ok = leveled_bookie:book_destroy(Bookie6).
safereaderror_startup(_Config) ->
RootPath = testutil:reset_filestructure(),
StartOpts1 = [{root_path, RootPath},