Mas d31 i416 (#418)
* Add compression controls (#417) * Add compression controls Add configuration options to allow for a compression algorithm of `none` to disable compression altogether. Also an option to change the point in the LSM tree when compression is applied. * Handle configurable defaults consistently Move them into leveled.hrl. This forces double-definitions to be resolved. There are some other constants in leveled_bookie that are relevant outside of leveled_bookie. These are all now in the non-configurable startup defaults section. * Clarify referred-to default is OTP not leveled * Update leveled_bookie.erl Handle xref issue with eunit include
This commit is contained in:
Martin Sumner
GitHub
parent
b96518c32a
commit
9e804924a8
11 changed files with 338 additions and 163 deletions
|
|
@ -202,12 +202,17 @@ bigsst_littlesst(_Config) ->
|
|||
ObjL1 =
|
||||
lists:keysort(
|
||||
1,
|
||||
testutil:generate_objects(80000, 1, [],
|
||||
leveled_rand:rand_bytes(100),
|
||||
fun() -> [] end, <<"B">>)
|
||||
testutil:generate_objects(
|
||||
100000,
|
||||
1,
|
||||
[],
|
||||
leveled_rand:rand_bytes(100),
|
||||
fun() -> [] end,
|
||||
<<"B">>)
|
||||
),
|
||||
testutil:riakload(Bookie1, ObjL1),
|
||||
testutil:check_forlist(Bookie1, ObjL1),
|
||||
timer:sleep(10000), % Wait for delete timeout
|
||||
JFP = RootPath ++ "/ledger/ledger_files/",
|
||||
{ok, FNS1} = file:list_dir(JFP),
|
||||
ok = leveled_bookie:book_destroy(Bookie1),
|
||||
|
|
@ -216,6 +221,7 @@ bigsst_littlesst(_Config) ->
|
|||
{ok, Bookie2} = leveled_bookie:book_start(StartOpts2),
|
||||
testutil:riakload(Bookie2, ObjL1),
|
||||
testutil:check_forlist(Bookie2, ObjL1),
|
||||
timer:sleep(10000), % Wait for delete timeout
|
||||
{ok, FNS2} = file:list_dir(JFP),
|
||||
ok = leveled_bookie:book_destroy(Bookie2),
|
||||
io:format("Big SST ~w files Little SST ~w files~n",
|
||||
|
|
@ -1007,46 +1013,137 @@ many_put_fetch_switchcompression(_Config) ->
|
|||
RootPath = testutil:reset_filestructure(),
|
||||
StartOpts1 = [{root_path, RootPath},
|
||||
{max_pencillercachesize, 16000},
|
||||
{sync_strategy, riak_sync},
|
||||
{max_journalobjectcount, 30000},
|
||||
{compression_level, 3},
|
||||
{sync_strategy, testutil:sync_strategy()},
|
||||
{compression_method, native}],
|
||||
StartOpts2 = [{root_path, RootPath},
|
||||
{max_pencillercachesize, 24000},
|
||||
{max_journalobjectcount, 30000},
|
||||
{sync_strategy, testutil:sync_strategy()},
|
||||
{compression_method, lz4}],
|
||||
StartOpts3 = [{root_path, RootPath},
|
||||
{max_pencillercachesize, 16000},
|
||||
{max_journalobjectcount, 30000},
|
||||
{sync_strategy, testutil:sync_strategy()},
|
||||
{compression_method, none}],
|
||||
|
||||
|
||||
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
|
||||
{TestObject, TestSpec} = testutil:generate_testobject(),
|
||||
ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
|
||||
testutil:check_forobject(Bookie1, TestObject),
|
||||
ok = leveled_bookie:book_close(Bookie1),
|
||||
StartOpts2 = [{root_path, RootPath},
|
||||
{max_journalsize, 500000000},
|
||||
{max_pencillercachesize, 32000},
|
||||
{sync_strategy, testutil:sync_strategy()},
|
||||
{compression_method, lz4}],
|
||||
CL1s =
|
||||
testutil:load_objects(
|
||||
40000,
|
||||
[2, 40002],
|
||||
Bookie1,
|
||||
TestObject,
|
||||
fun testutil:generate_smallobjects/2),
|
||||
|
||||
%% Change compression method
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie1, CL) end, CL1s),
|
||||
ok = leveled_bookie:book_close(Bookie1),
|
||||
|
||||
%% Change compression method -> lz4
|
||||
{ok, Bookie2} = leveled_bookie:book_start(StartOpts2),
|
||||
testutil:check_forobject(Bookie2, TestObject),
|
||||
GenList = [2, 40002, 80002, 120002],
|
||||
CLs = testutil:load_objects(40000, GenList, Bookie2, TestObject,
|
||||
fun testutil:generate_smallobjects/2),
|
||||
CL1A = lists:nth(1, CLs),
|
||||
ChkListFixed = lists:nth(length(CLs), CLs),
|
||||
testutil:check_forlist(Bookie2, CL1A),
|
||||
ObjList2A = testutil:generate_objects(5000, 2),
|
||||
testutil:riakload(Bookie2, ObjList2A),
|
||||
ChkList2A = lists:sublist(lists:sort(ObjList2A), 1000),
|
||||
testutil:check_forlist(Bookie2, ChkList2A),
|
||||
testutil:check_forlist(Bookie2, ChkListFixed),
|
||||
testutil:check_forobject(Bookie2, TestObject),
|
||||
testutil:check_forlist(Bookie2, ChkList2A),
|
||||
testutil:check_forlist(Bookie2, ChkListFixed),
|
||||
testutil:check_forobject(Bookie2, TestObject),
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie2, CL) end, CL1s),
|
||||
|
||||
CL2s =
|
||||
testutil:load_objects(
|
||||
40000,
|
||||
[80002, 120002],
|
||||
Bookie2,
|
||||
TestObject,
|
||||
fun testutil:generate_smallobjects/2),
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie2, CL) end, CL2s),
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie2, CL) end, CL1s),
|
||||
ok = leveled_bookie:book_close(Bookie2),
|
||||
|
||||
%% Change method back again
|
||||
{ok, Bookie3} = leveled_bookie:book_start(StartOpts1),
|
||||
testutil:check_forlist(Bookie3, ChkList2A),
|
||||
testutil:check_forlist(Bookie3, ChkListFixed),
|
||||
testutil:check_forobject(Bookie3, TestObject),
|
||||
testutil:check_formissingobject(Bookie3, "Bookie1", "MissingKey0123"),
|
||||
ok = leveled_bookie:book_destroy(Bookie3).
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie3, CL) end, CL2s),
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie3, CL) end, CL1s),
|
||||
|
||||
CL3s =
|
||||
testutil:load_objects(
|
||||
40000,
|
||||
[160002, 200002],
|
||||
Bookie3,
|
||||
TestObject,
|
||||
fun testutil:generate_smallobjects/2,
|
||||
30000
|
||||
),
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie3, CL) end, CL3s),
|
||||
ok = leveled_bookie:book_close(Bookie3),
|
||||
|
||||
% Change method to no compression
|
||||
{ok, Bookie4} = leveled_bookie:book_start(StartOpts3),
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie4, CL) end, CL2s),
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie4, CL) end, CL1s),
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie4, CL) end, CL3s),
|
||||
|
||||
CL4s =
|
||||
testutil:load_objects(
|
||||
40000,
|
||||
[240002, 280002],
|
||||
Bookie4,
|
||||
TestObject,
|
||||
fun testutil:generate_smallobjects/2
|
||||
),
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie4, CL) end, CL3s),
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie4, CL) end, CL4s),
|
||||
testutil:delete_some_objects(Bookie4, lists:flatten(CL3s), 60000),
|
||||
CL5s =
|
||||
testutil:load_objects(
|
||||
40000,
|
||||
[320002, 360002],
|
||||
Bookie4,
|
||||
TestObject,
|
||||
fun testutil:generate_smallobjects/2
|
||||
),
|
||||
ok = leveled_bookie:book_compactjournal(Bookie4, 30000),
|
||||
testutil:wait_for_compaction(Bookie4),
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie4, CL) end, CL4s),
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie4, CL) end, CL5s),
|
||||
|
||||
ok = leveled_bookie:book_close(Bookie4),
|
||||
|
||||
%% Change compression method -> lz4
|
||||
{ok, Bookie5} = leveled_bookie:book_start(StartOpts2),
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie5, CL) end, CL1s),
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie5, CL) end, CL4s),
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie5, CL) end, CL5s),
|
||||
ok = leveled_bookie:book_close(Bookie5),
|
||||
|
||||
%% Change compression method -> native
|
||||
{ok, Bookie6} = leveled_bookie:book_start(StartOpts1),
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie6, CL) end, CL1s),
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie6, CL) end, CL4s),
|
||||
lists:foreach(
|
||||
fun(CL) -> ok = testutil:check_forlist(Bookie6, CL) end, CL5s),
|
||||
|
||||
ok = leveled_bookie:book_destroy(Bookie6).
|
||||
|
||||
|
||||
safereaderror_startup(_Config) ->
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue