umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Mas d31 i416 (#418)

Browse source 
* Add compression controls (#417)

* Add compression controls

Add configuration options to allow for a compression algorithm of `none` to disable compression altogether.  Also an option to change the point in the LSM tree when compression is applied.

* Handle configurable defaults consistently

Move them into leveled.hrl.  This forces double-definitions to be resolved.

There are some other constants in leveled_bookie that are relevant outside of leveled_bookie.  These are all now in the non-configurable startup defaults section.

* Clarify referred-to default is OTP not leveled

* Update leveled_bookie.erl

Handle xref issue with eunit include
This commit is contained in:
Martin Sumner 2023-11-07 14:58:43 +00:00 • committed by GitHub
parent b96518c32a
commit 9e804924a8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 338 additions and 163 deletions
Download patch file Download diff file Expand all files Collapse all files

157
test/end_to_end/basic_SUITE.erl
View file

@ -202,12 +202,17 @@ bigsst_littlesst(_Config) ->
ObjL1 =
lists:keysort(
1,
testutil:generate_objects(80000, 1, [],
leveled_rand:rand_bytes(100),
fun() -> [] end, <<"B">>)
testutil:generate_objects(
100000,
1,
[],
leveled_rand:rand_bytes(100),
fun() -> [] end,
<<"B">>)
),
testutil:riakload(Bookie1, ObjL1),
testutil:check_forlist(Bookie1, ObjL1),
timer:sleep(10000), % Wait for delete timeout
JFP = RootPath ++ "/ledger/ledger_files/",
{ok, FNS1} = file:list_dir(JFP),
ok = leveled_bookie:book_destroy(Bookie1),
@ -216,6 +221,7 @@ bigsst_littlesst(_Config) ->
{ok, Bookie2} = leveled_bookie:book_start(StartOpts2),
testutil:riakload(Bookie2, ObjL1),
testutil:check_forlist(Bookie2, ObjL1),
timer:sleep(10000), % Wait for delete timeout
{ok, FNS2} = file:list_dir(JFP),
ok = leveled_bookie:book_destroy(Bookie2),
io:format("Big SST ~w files Little SST ~w files~n",
@ -1007,46 +1013,137 @@ many_put_fetch_switchcompression(_Config) ->
RootPath = testutil:reset_filestructure(),
StartOpts1 = [{root_path, RootPath},
{max_pencillercachesize, 16000},
{sync_strategy, riak_sync},
{max_journalobjectcount, 30000},
{compression_level, 3},
{sync_strategy, testutil:sync_strategy()},
{compression_method, native}],
StartOpts2 = [{root_path, RootPath},
{max_pencillercachesize, 24000},
{max_journalobjectcount, 30000},
{sync_strategy, testutil:sync_strategy()},
{compression_method, lz4}],
StartOpts3 = [{root_path, RootPath},
{max_pencillercachesize, 16000},
{max_journalobjectcount, 30000},
{sync_strategy, testutil:sync_strategy()},
{compression_method, none}],
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
{TestObject, TestSpec} = testutil:generate_testobject(),
ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
testutil:check_forobject(Bookie1, TestObject),
ok = leveled_bookie:book_close(Bookie1),
StartOpts2 = [{root_path, RootPath},
{max_journalsize, 500000000},
{max_pencillercachesize, 32000},
{sync_strategy, testutil:sync_strategy()},
{compression_method, lz4}],
CL1s =
testutil:load_objects(
40000,
[2, 40002],
Bookie1,
TestObject,
fun testutil:generate_smallobjects/2),
%% Change compression method
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie1, CL) end, CL1s),
ok = leveled_bookie:book_close(Bookie1),
%% Change compression method -> lz4
{ok, Bookie2} = leveled_bookie:book_start(StartOpts2),
testutil:check_forobject(Bookie2, TestObject),
GenList = [2, 40002, 80002, 120002],
CLs = testutil:load_objects(40000, GenList, Bookie2, TestObject,
fun testutil:generate_smallobjects/2),
CL1A = lists:nth(1, CLs),
ChkListFixed = lists:nth(length(CLs), CLs),
testutil:check_forlist(Bookie2, CL1A),
ObjList2A = testutil:generate_objects(5000, 2),
testutil:riakload(Bookie2, ObjList2A),
ChkList2A = lists:sublist(lists:sort(ObjList2A), 1000),
testutil:check_forlist(Bookie2, ChkList2A),
testutil:check_forlist(Bookie2, ChkListFixed),
testutil:check_forobject(Bookie2, TestObject),
testutil:check_forlist(Bookie2, ChkList2A),
testutil:check_forlist(Bookie2, ChkListFixed),
testutil:check_forobject(Bookie2, TestObject),
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie2, CL) end, CL1s),
CL2s =
testutil:load_objects(
40000,
[80002, 120002],
Bookie2,
TestObject,
fun testutil:generate_smallobjects/2),
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie2, CL) end, CL2s),
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie2, CL) end, CL1s),
ok = leveled_bookie:book_close(Bookie2),
%% Change method back again
{ok, Bookie3} = leveled_bookie:book_start(StartOpts1),
testutil:check_forlist(Bookie3, ChkList2A),
testutil:check_forlist(Bookie3, ChkListFixed),
testutil:check_forobject(Bookie3, TestObject),
testutil:check_formissingobject(Bookie3, "Bookie1", "MissingKey0123"),
ok = leveled_bookie:book_destroy(Bookie3).
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie3, CL) end, CL2s),
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie3, CL) end, CL1s),
CL3s =
testutil:load_objects(
40000,
[160002, 200002],
Bookie3,
TestObject,
fun testutil:generate_smallobjects/2,
30000
),
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie3, CL) end, CL3s),
ok = leveled_bookie:book_close(Bookie3),
% Change method to no compression
{ok, Bookie4} = leveled_bookie:book_start(StartOpts3),
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie4, CL) end, CL2s),
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie4, CL) end, CL1s),
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie4, CL) end, CL3s),
CL4s =
testutil:load_objects(
40000,
[240002, 280002],
Bookie4,
TestObject,
fun testutil:generate_smallobjects/2
),
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie4, CL) end, CL3s),
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie4, CL) end, CL4s),
testutil:delete_some_objects(Bookie4, lists:flatten(CL3s), 60000),
CL5s =
testutil:load_objects(
40000,
[320002, 360002],
Bookie4,
TestObject,
fun testutil:generate_smallobjects/2
),
ok = leveled_bookie:book_compactjournal(Bookie4, 30000),
testutil:wait_for_compaction(Bookie4),
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie4, CL) end, CL4s),
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie4, CL) end, CL5s),
ok = leveled_bookie:book_close(Bookie4),
%% Change compression method -> lz4
{ok, Bookie5} = leveled_bookie:book_start(StartOpts2),
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie5, CL) end, CL1s),
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie5, CL) end, CL4s),
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie5, CL) end, CL5s),
ok = leveled_bookie:book_close(Bookie5),
%% Change compression method -> native
{ok, Bookie6} = leveled_bookie:book_start(StartOpts1),
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie6, CL) end, CL1s),
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie6, CL) end, CL4s),
lists:foreach(
fun(CL) -> ok = testutil:check_forlist(Bookie6, CL) end, CL5s),
ok = leveled_bookie:book_destroy(Bookie6).
safereaderror_startup(_Config) ->

69
test/end_to_end/iterator_SUITE.erl
View file

@ -28,19 +28,21 @@ all() -> [
expiring_indexes(_Config) ->
% Add objects to ths tore with index entries, where the objects (and hence
% Add objects to the store with index entries, where the objects (and hence
% the indexes have an expiry time. Confirm that the indexes and the
% objects are no longer present after the expiry time (and are present
% before). Confirm that replacing an object has the expected outcome, if
% the IndexSpecs are updated as part of the request.
KeyCount = 50000,
Future = 120,
% 2 minutes - if running tests on a slow machine, may need to increase
Future = 60,
% 1 minute - if running tests on a slow machine, may need to increase
% this value
RootPath = testutil:reset_filestructure(),
StartOpts1 = [{root_path, RootPath},
{max_journalsize, 100000000},
{sync_strategy, testutil:sync_strategy()}],
StartOpts1 =
[{root_path, RootPath},
{max_pencillercachesize, 16000},
{max_journalobjectcount, 30000},
{sync_strategy, testutil:sync_strategy()}],
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
SW1 = os:timestamp(),
@ -48,12 +50,11 @@ expiring_indexes(_Config) ->
LoadTime = timer:now_diff(os:timestamp(), SW1)/1000000,
io:format("Load of ~w std objects in ~w seconds~n", [KeyCount, LoadTime]),
FilterFun = fun({I, _B, _K}) -> lists:member(I, [5, 6, 7, 8]) end,
LoadedEntriesInRange = lists:sort(lists:filter(FilterFun, IBKL1)),
true = LoadTime < (Future - 30),
% need 30 seconds spare to run query
true = LoadTime < (Future - 20),
% need 20 seconds spare to run query
% and add replacements
{I0, B0, K0} = hd(IBKL1),
@ -62,12 +63,12 @@ expiring_indexes(_Config) ->
CountI0Fold =
fun() ->
leveled_bookie:book_indexfold(Bookie1,
B0,
{fun(_BF, _KT, Acc) -> Acc + 1 end,
0},
{<<"temp_int">>, I0, I0},
{true, undefined})
leveled_bookie:book_indexfold(
Bookie1,
B0,
{fun(_BF, _KT, Acc) -> Acc + 1 end, 0},
{<<"temp_int">>, I0, I0},
{true, undefined})
end,
{async, I0Counter1} = CountI0Fold(),
I0Count1 = I0Counter1(),
@ -76,29 +77,30 @@ expiring_indexes(_Config) ->
InitAcc = [],
IndexFold =
fun() ->
leveled_bookie:book_indexfold(Bookie1,
B0,
{FoldFun, InitAcc},
{<<"temp_int">>, 5, 8},
{true, undefined})
leveled_bookie:book_indexfold(
Bookie1,
B0,
{FoldFun, InitAcc},
{<<"temp_int">>, 5, 8},
{true, undefined})
end,
{async, Folder1} = IndexFold(),
QR1 = Folder1(),
true = lists:sort(QR1) == LoadedEntriesInRange,
% Replace object with one with an index value of 6
testutil:stdload_object(Bookie1, B0, K0, 6, <<"value">>,
leveled_util:integer_now() + 600),
testutil:stdload_object(
Bookie1, B0, K0, 6, <<"value">>, leveled_util:integer_now() + 600),
% Confirm that this has reduced the index entries in I0 by 1
{async, I0Counter2} = CountI0Fold(),
I0Count2 = I0Counter2(),
io:format("Count with index value ~w changed from ~w to ~w~n",
[I0, I0Count1, I0Count2]),
true = I0Count2 == (I0Count1 - 1),
% Now replace again, shortening the timeout to 15s,
% Now replace again, shortening the timeout to 10s,
% this time index value of 6
testutil:stdload_object(Bookie1, B0, K0, 5, <<"value">>,
leveled_util:integer_now() + 15),
testutil:stdload_object(
Bookie1, B0, K0, 5, <<"value">>, leveled_util:integer_now() + 10),
{async, Folder2} = IndexFold(),
leveled_bookie:book_indexfold(Bookie1,
B0,
@ -108,8 +110,8 @@ expiring_indexes(_Config) ->
QR2 = Folder2(),
io:format("Query with additional entry length ~w~n", [length(QR2)]),
true = lists:sort(QR2) == lists:sort([{5, B0, K0}|LoadedEntriesInRange]),
% Wait for a 15s timeout + lus a second to be sure
timer:sleep(15000 + 1000),
% Wait for a 10s timeout plus a second to be sure
timer:sleep(10000 + 1000),
{async, Folder3} = IndexFold(),
QR3 = Folder3(),
% Now the entry should be missing (and the 600s TTL entry should not have
@ -118,16 +120,23 @@ expiring_indexes(_Config) ->
true = lists:sort(QR3) == LoadedEntriesInRange,
FoldTime = timer:now_diff(os:timestamp(), SW1)/1000000 - LoadTime,
io:format("Query returned ~w entries in ~w seconds - 3 queries + 15s wait~n",
io:format("Query returned ~w entries in ~w seconds - 3 queries + 10s wait~n",
[length(QR1), FoldTime]),
true = (LoadTime + FoldTime) < Future,
SleepTime = round((Future - (LoadTime + FoldTime)) * 1000 + 1000), % add a second
SleepTime = round((Future - (LoadTime + FoldTime)) * 1000),
io:format("Sleeping ~w s for all to expire~n", [SleepTime/1000]),
timer:sleep(SleepTime),
timer:sleep(SleepTime + 1000), % add a second
% Index entries should now have expired
{async, Folder4} = IndexFold(),
QR4 = Folder4(),
io:format("Unexpired indexes of length ~w~n", [length(QR4)]),
lists:foreach(
fun(I) ->
io:format("Unexpired index ~p~n", [I])
end,
QR4
),
true = QR4 == [],
ok = leveled_bookie:book_close(Bookie1),