Mas i335 otp24 (#336)

* Address OTP24 warnings, ct and eunit paths

* Reorg to add OTP 24 support

* Update VOLUME.md

* Correct broken refs

* Update README.md

* CI on all main branches

Co-authored-by: Ulf Wiger <ulf@wiger.net>
This commit is contained in:
Martin Sumner 2021-05-25 13:41:20 +01:00 committed by GitHub
parent 2c53c0a85a
commit ed0301e2cf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
43 changed files with 99 additions and 553 deletions

View file

@ -169,18 +169,7 @@ encode_maybe_binary(Bin) ->
%% =================================================
sync_strategy() ->
case erlang:system_info(otp_release) of
"17" ->
sync;
"18" ->
sync;
"19" ->
sync;
_ ->
% running the sync strategy with OTP16 on macbook is
% super slow. So revert to no sync
none
end.
none.
book_riakput(Pid, RiakObject, IndexSpecs) ->
leveled_bookie:book_put(Pid,

View file

@ -1,323 +0,0 @@
-module(lookup_test).
-export([go_dict/1,
go_ets/1,
go_gbtree/1,
go_arrayofdict/1,
go_arrayofgbtree/1,
go_arrayofdict_withcache/1,
create_blocks/3,
size_testblocks/1,
test_testblocks/2]).
-define(CACHE_SIZE, 512).
hash(Key) ->
H = 5381,
hash1(H,Key) band 16#FFFFFFFF.
hash1(H,[]) ->H;
hash1(H,[B|Rest]) ->
H1 = H * 33,
H2 = H1 bxor B,
hash1(H2,Rest).
% Get the least significant 8 bits from the hash.
hash_to_index(Hash) ->
Hash band 255.
%%
%% Timings (microseconds):
%%
%% go_dict(200000) : 1569894
%% go_dict(1000000) : 17191365
%% go_dict(5000000) : forever
go_dict(N) ->
go_dict(dict:new(), N, N).
go_dict(_, 0, _) ->
{erlang:memory(), statistics(garbage_collection)};
go_dict(D, N, M) ->
% Lookup a random key - which may not be present
LookupKey = lists:concat(["key-", leveled_rand:uniform(M)]),
LookupHash = hash(LookupKey),
dict:find(LookupHash, D),
% Add a new key - which may be present so value to be appended
Key = lists:concat(["key-", N]),
Hash = hash(Key),
case dict:find(Hash, D) of
error ->
go_dict(dict:store(Hash, [N], D), N-1, M);
{ok, List} ->
go_dict(dict:store(Hash, [N|List], D), N-1, M)
end.
%%
%% Timings (microseconds):
%%
%% go_ets(200000) : 609119
%% go_ets(1000000) : 3520757
%% go_ets(5000000) : 19974562
go_ets(N) ->
go_ets(ets:new(ets_test, [private, bag]), N, N).
go_ets(_, 0, _) ->
{erlang:memory(), statistics(garbage_collection)};
go_ets(Ets, N, M) ->
% Lookup a random key - which may not be present
LookupKey = lists:concat(["key-", leveled_rand:uniform(M)]),
LookupHash = hash(LookupKey),
ets:lookup(Ets, LookupHash),
% Add a new key - which may be present so value to be appended
Key = lists:concat(["key-", N]),
Hash = hash(Key),
ets:insert(Ets, {Hash, N}),
go_ets(Ets, N - 1, M).
%%
%% Timings (microseconds):
%%
%% go_gbtree(200000) : 1393936
%% go_gbtree(1000000) : 8430997
%% go_gbtree(5000000) : 45630810
go_gbtree(N) ->
go_gbtree(gb_trees:empty(), N, N).
go_gbtree(_, 0, _) ->
{erlang:memory(), statistics(garbage_collection)};
go_gbtree(Tree, N, M) ->
% Lookup a random key - which may not be present
LookupKey = lists:concat(["key-", leveled_rand:uniform(M)]),
LookupHash = hash(LookupKey),
gb_trees:lookup(LookupHash, Tree),
% Add a new key - which may be present so value to be appended
Key = lists:concat(["key-", N]),
Hash = hash(Key),
case gb_trees:lookup(Hash, Tree) of
none ->
go_gbtree(gb_trees:insert(Hash, [N], Tree), N - 1, M);
{value, List} ->
go_gbtree(gb_trees:update(Hash, [N|List], Tree), N - 1, M)
end.
%%
%% Timings (microseconds):
%%
%% go_arrayofidict(200000) : 1266931
%% go_arrayofidict(1000000) : 7387219
%% go_arrayofidict(5000000) : 49511484
go_arrayofdict(N) ->
go_arrayofdict(array:new(256, {default, dict:new()}), N, N).
go_arrayofdict(_, 0, _) ->
% dict:to_list(array:get(0, Array)),
% dict:to_list(array:get(1, Array)),
% dict:to_list(array:get(2, Array)),
% dict:to_list(array:get(3, Array)),
% dict:to_list(array:get(4, Array)),
% dict:to_list(array:get(5, Array)),
% dict:to_list(array:get(6, Array)),
% dict:to_list(array:get(7, Array)),
% dict:to_list(array:get(8, Array)),
% dict:to_list(array:get(9, Array)),
{erlang:memory(), statistics(garbage_collection)};
go_arrayofdict(Array, N, M) ->
% Lookup a random key - which may not be present
LookupKey = lists:concat(["key-", leveled_rand:uniform(M)]),
LookupHash = hash(LookupKey),
LookupIndex = hash_to_index(LookupHash),
dict:find(LookupHash, array:get(LookupIndex, Array)),
% Add a new key - which may be present so value to be appended
Key = lists:concat(["key-", N]),
Hash = hash(Key),
Index = hash_to_index(Hash),
D = array:get(Index, Array),
case dict:find(Hash, D) of
error ->
go_arrayofdict(array:set(Index,
dict:store(Hash, [N], D), Array), N-1, M);
{ok, List} ->
go_arrayofdict(array:set(Index,
dict:store(Hash, [N|List], D), Array), N-1, M)
end.
%%
%% Timings (microseconds):
%%
%% go_arrayofgbtree(200000) : 1176224
%% go_arrayofgbtree(1000000) : 7480653
%% go_arrayofgbtree(5000000) : 41266701
go_arrayofgbtree(N) ->
go_arrayofgbtree(array:new(256, {default, gb_trees:empty()}), N, N).
go_arrayofgbtree(_, 0, _) ->
% gb_trees:to_list(array:get(0, Array)),
% gb_trees:to_list(array:get(1, Array)),
% gb_trees:to_list(array:get(2, Array)),
% gb_trees:to_list(array:get(3, Array)),
% gb_trees:to_list(array:get(4, Array)),
% gb_trees:to_list(array:get(5, Array)),
% gb_trees:to_list(array:get(6, Array)),
% gb_trees:to_list(array:get(7, Array)),
% gb_trees:to_list(array:get(8, Array)),
% gb_trees:to_list(array:get(9, Array)),
{erlang:memory(), statistics(garbage_collection)};
go_arrayofgbtree(Array, N, M) ->
% Lookup a random key - which may not be present
LookupKey = lists:concat(["key-", leveled_rand:uniform(M)]),
LookupHash = hash(LookupKey),
LookupIndex = hash_to_index(LookupHash),
gb_trees:lookup(LookupHash, array:get(LookupIndex, Array)),
% Add a new key - which may be present so value to be appended
Key = lists:concat(["key-", N]),
Hash = hash(Key),
Index = hash_to_index(Hash),
Tree = array:get(Index, Array),
case gb_trees:lookup(Hash, Tree) of
none ->
go_arrayofgbtree(array:set(Index,
gb_trees:insert(Hash, [N], Tree), Array), N - 1, M);
{value, List} ->
go_arrayofgbtree(array:set(Index,
gb_trees:update(Hash, [N|List], Tree), Array), N - 1, M)
end.
%%
%% Timings (microseconds):
%%
%% go_arrayofdict_withcache(200000) : 1432951
%% go_arrayofdict_withcache(1000000) : 9140169
%% go_arrayofdict_withcache(5000000) : 59435511
go_arrayofdict_withcache(N) ->
go_arrayofdict_withcache({array:new(256, {default, dict:new()}),
array:new(256, {default, dict:new()})}, N, N).
go_arrayofdict_withcache(_, 0, _) ->
{erlang:memory(), statistics(garbage_collection)};
go_arrayofdict_withcache({MArray, CArray}, N, M) ->
% Lookup a random key - which may not be present
LookupKey = lists:concat(["key-", leveled_rand:uniform(M)]),
LookupHash = hash(LookupKey),
LookupIndex = hash_to_index(LookupHash),
dict:find(LookupHash, array:get(LookupIndex, CArray)),
dict:find(LookupHash, array:get(LookupIndex, MArray)),
% Add a new key - which may be present so value to be appended
Key = lists:concat(["key-", N]),
Hash = hash(Key),
Index = hash_to_index(Hash),
Cache = array:get(Index, CArray),
case dict:find(Hash, Cache) of
error ->
UpdCache = dict:store(Hash, [N], Cache);
{ok, _} ->
UpdCache = dict:append(Hash, N, Cache)
end,
case dict:size(UpdCache) of
?CACHE_SIZE ->
UpdCArray = array:set(Index, dict:new(), CArray),
UpdMArray = array:set(Index, dict:merge(fun merge_values/3, UpdCache, array:get(Index, MArray)), MArray),
go_arrayofdict_withcache({UpdMArray, UpdCArray}, N - 1, M);
_ ->
UpdCArray = array:set(Index, UpdCache, CArray),
go_arrayofdict_withcache({MArray, UpdCArray}, N - 1, M)
end.
merge_values(_, Value1, Value2) ->
lists:append(Value1, Value2).
%% Some functions for testing options compressing term_to_binary
create_block(N, BlockType) ->
case BlockType of
keylist ->
create_block(N, BlockType, []);
keygbtree ->
create_block(N, BlockType, gb_trees:empty())
end.
create_block(0, _, KeyStruct) ->
KeyStruct;
create_block(N, BlockType, KeyStruct) ->
Bucket = <<"pdsRecord">>,
case N of
20 ->
Key = lists:concat(["key-20-special"]);
_ ->
Key = lists:concat(["key-", N, "-", leveled_rand:uniform(1000)])
end,
SequenceNumber = leveled_rand:uniform(1000000000),
Indexes = [{<<"DateOfBirth_int">>, leveled_rand:uniform(10000)}, {<<"index1_bin">>, lists:concat([leveled_rand:uniform(1000), "SomeCommonText"])}, {<<"index2_bin">>, <<"RepetitionRepetitionRepetition">>}],
case BlockType of
keylist ->
Term = {o, Bucket, Key, {Indexes, SequenceNumber}},
create_block(N-1, BlockType, [Term|KeyStruct]);
keygbtree ->
create_block(N-1, BlockType, gb_trees:insert({o, Bucket, Key}, {Indexes, SequenceNumber}, KeyStruct))
end.
create_blocks(N, Compression, BlockType) ->
create_blocks(N, Compression, BlockType, 10000, []).
create_blocks(_, _, _, 0, BlockList) ->
BlockList;
create_blocks(N, Compression, BlockType, TestLoops, BlockList) ->
NewBlock = term_to_binary(create_block(N, BlockType), [{compressed, Compression}]),
create_blocks(N, Compression, BlockType, TestLoops - 1, [NewBlock|BlockList]).
size_testblocks(BlockList) ->
size_testblocks(BlockList,0).
size_testblocks([], Acc) ->
Acc;
size_testblocks([H|T], Acc) ->
size_testblocks(T, Acc + byte_size(H)).
test_testblocks([], _) ->
true;
test_testblocks([H|T], BlockType) ->
Block = binary_to_term(H),
case findkey("key-20-special", Block, BlockType) of
true ->
test_testblocks(T, BlockType);
not_found ->
false
end.
findkey(_, [], keylist) ->
not_found;
findkey(Key, [H|T], keylist) ->
case H of
{o, <<"pdsRecord">>, Key, _} ->
true;
_ ->
findkey(Key,T, keylist)
end;
findkey(Key, Tree, keygbtree) ->
case gb_trees:lookup({o, <<"pdsRecord">>, Key}, Tree) of
none ->
not_found;
_ ->
true
end.

View file

@ -1,51 +0,0 @@
-module(member_test).
-export([test_membership/0]).
-define(SEGMENTS_TO_CHECK, 32768). % a whole SST file
-define(MEMBERSHIP_LENGTHS, [8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096]).
segments(Length) ->
AllSegs = lists:seq(1, ?SEGMENTS_TO_CHECK),
AllSegsBin =
lists:foldl(fun(I, Acc) -> <<Acc/binary, (I - 1):16/integer>> end,
<<>>,
AllSegs),
StartPos = leveled_rand:uniform(length(AllSegs) - Length),
{<<AllSegsBin/binary, AllSegsBin/binary,
AllSegsBin/binary, AllSegsBin/binary>>,
lists:sublist(AllSegs, StartPos, Length)}.
test_membership(Length) ->
{AllSegsBin, TestList} = segments(Length),
ExpectedOutput =
lists:reverse(TestList ++ TestList ++ TestList ++ TestList),
SW0 = os:timestamp(),
TestListFun = fun(I) -> lists:member(I, TestList) end,
true = test_binary(AllSegsBin, [], TestListFun) == ExpectedOutput,
ListT = timer:now_diff(os:timestamp(), SW0) / 131072,
SW1 = os:timestamp(),
TestSet = sets:from_list(TestList),
TestSetsFun = fun(I) -> sets:is_element(I, TestSet) end,
true = test_binary(AllSegsBin, [], TestSetsFun) == ExpectedOutput,
SetsT = timer:now_diff(os:timestamp(), SW1) / 131072,
io:format("Test with segment count ~w ..."
++ " took ~w ms per 1000 checks with list ..."
++ " took ~w ms per 1000 checks with set~n", [Length, ListT, SetsT]).
test_binary(<<>>, Acc, _TestFun) ->
Acc;
test_binary(<<0:1/integer, TestSeg:15/integer, Rest/binary>>, Acc, TestFun) ->
case TestFun(TestSeg) of
true ->
test_binary(Rest, [TestSeg|Acc], TestFun);
false ->
test_binary(Rest, Acc, TestFun)
end.
test_membership() ->
lists:foreach(fun(I) -> test_membership(I) end, ?MEMBERSHIP_LENGTHS).

View file

@ -1,59 +0,0 @@
%% Test performance and accuracy of rice-encoded bloom filters
%%
%% Calling check_negative(2048, 1000000) should return about 122 false
%% positives in around 11 seconds, with a size below 4KB
%%
%% The equivalent positive check is check_positive(2048, 488) and this
%% should take around 6 seconds.
%%
%% So a blooom with 2048 members should support o(100K) checks per second
%% on a modern CPU, whilst requiring 2 bytes per member.
-module(rice_test).
-export([check_positive/2, check_negative/2, calc_hash/2]).
check_positive(KeyCount, LoopCount) ->
KeyList = produce_keylist(KeyCount),
Bloom = leveled_rice:create_bloom(KeyList),
check_positive(KeyList, Bloom, LoopCount).
check_positive(_, Bloom, 0) ->
{ok, byte_size(Bloom)};
check_positive(KeyList, Bloom, LoopCount) ->
true = leveled_rice:check_keys(KeyList, Bloom),
check_positive(KeyList, Bloom, LoopCount - 1).
produce_keylist(KeyCount) ->
KeyPrefix = lists:concat(["PositiveKey-", leveled_rand:uniform(KeyCount)]),
produce_keylist(KeyCount, [], KeyPrefix).
produce_keylist(0, KeyList, _) ->
KeyList;
produce_keylist(KeyCount, KeyList, KeyPrefix) ->
Key = lists:concat([KeyPrefix, KeyCount]),
produce_keylist(KeyCount - 1, [Key|KeyList], KeyPrefix).
check_negative(KeyCount, CheckCount) ->
KeyList = produce_keylist(KeyCount),
Bloom = leveled_rice:create_bloom(KeyList),
check_negative(Bloom, CheckCount, 0).
check_negative(Bloom, 0, FalsePos) ->
{byte_size(Bloom), FalsePos};
check_negative(Bloom, CheckCount, FalsePos) ->
Key = lists:concat(["NegativeKey-", CheckCount, leveled_rand:uniform(CheckCount)]),
case leveled_rice:check_key(Key, Bloom) of
true -> check_negative(Bloom, CheckCount - 1, FalsePos + 1);
false -> check_negative(Bloom, CheckCount - 1, FalsePos)
end.
calc_hash(_, 0) ->
ok;
calc_hash(Key, Count) ->
erlang:phash2(lists:concat([Key, Count, "sometxt"])),
calc_hash(Key, Count -1).

Binary file not shown.

Before

Width:  |  Height:  |  Size: 129 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 102 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 115 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 78 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 93 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 111 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 83 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 97 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 79 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 98 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 81 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 109 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 73 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 99 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 76 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 41 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 99 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 78 KiB

View file

@ -1,21 +0,0 @@
{mode, max}.
{duration, 30}.
{concurrent, 24}.
{driver, basho_bench_driver_eleveldb}.
{key_generator, {int_to_bin_bigendian,{uniform_int, 1000000}}}.
{value_generator, {fixed_bin, 8000}}.
{operations, [{get, 5}, {put, 1}]}.
%% the second element in the list below (e.g., "../../public/eleveldb") must
%% point to the relevant directory of a eleveldb installation
{code_paths, ["../eleveldb/ebin"]}.
{eleveldb_dir, "/tmp/eleveldb.bench"}.
{eleveldb_num_instances, 12}.

View file

@ -1,21 +0,0 @@
{mode, max}.
{duration, 30}.
{concurrent, 24}.
{driver, basho_bench_driver_eleveldb}.
{key_generator, {int_to_bin_bigendian,{partitioned_sequential_int, 10000000}}}.
{value_generator, {fixed_bin, 8000}}.
{operations, [{put, 1}]}.
%% the second element in the list below (e.g., "../../public/eleveldb") must
%% point to the relevant directory of a eleveldb installation
{code_paths, ["../eleveldb/ebin"]}.
{eleveldb_dir, "/tmp/eleveldb.bench"}.
{eleveldb_num_instances, 12}.

View file

@ -1,21 +0,0 @@
{mode, max}.
{duration, 30}.
{concurrent, 24}.
{driver, basho_bench_driver_eleveleddb}.
{key_generator, {int_to_bin_bigendian,{uniform_int, 1000000}}}.
{value_generator, {fixed_bin, 8000}}.
{operations, [{get, 5}, {put, 1}]}.
%% the second element in the list below (e.g., "../../public/eleveldb") must
%% point to the relevant directory of a eleveldb installation
{code_paths, ["../eleveleddb/_build/default/lib/eleveleddb/ebin"]}.
{eleveleddb_dir, "/tmp/eleveleddb.bench"}.
{eleveleddb_num_instances, 12}.

View file

@ -1,21 +0,0 @@
{mode, max}.
{duration, 30}.
{concurrent, 24}.
{driver, basho_bench_driver_eleveleddb}.
{key_generator, {int_to_bin_bigendian,{partitioned_sequential_int, 10000000}}}.
{value_generator, {fixed_bin, 8000}}.
{operations, [{put, 1}]}.
%% the second element in the list below (e.g., "../../public/eleveldb") must
%% point to the relevant directory of a eleveleddb installation
{code_paths, ["../eleveleddb/_build/default/lib/eleveleddb/ebin"]}.
{eleveleddb_dir, "/tmp/eleveleddb.bench"}.
{eleveleddb_num_instances, 12}.

Binary file not shown.

Before

Width:  |  Height:  |  Size: 316 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 315 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 333 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 274 KiB

View file

@ -1,93 +0,0 @@
%% -------------------------------------------------------------------
%%
%% Copyright (c) 2015 Basho Techonologies
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License. You may obtain
%% a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied. See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%
%% -------------------------------------------------------------------
%% Raw eleveldb driver. It opens a number of eleveldb instances and assigns
%% one to each created worker in round robin fashion. So, for example, creating
%% 32 instances and 64 concurrent workers would bind a pair of workers to
%% each instance for all operations.
-module(basho_bench_driver_eleveleddb).
-export([new/1,
run/4]).
% -include("basho_bench.hrl").
-record(state, {
instance
}).
get_instances() ->
case basho_bench_config:get(eleveleddb_instances, undefined) of
undefined ->
Instances = start_instances(),
% ?INFO("Instances started ~w~n", [Instances]),
basho_bench_config:set(eleveleddb_instances, Instances),
Instances;
Instances ->
Instances
end.
start_instances() ->
BaseDir = basho_bench_config:get(eleveleddb_dir, "."),
Num = basho_bench_config:get(eleveleddb_num_instances, 1),
% ?INFO("Starting up ~p eleveleddb instances under ~s .\n",
% [Num, BaseDir]),
Refs = [begin
Dir = filename:join(BaseDir, "instance." ++ integer_to_list(N)),
% ?INFO("Opening eleveleddb instance in ~s\n", [Dir]),
{ok, Ref} = leveled_bookie:book_start(Dir, 2000, 500000000),
Ref
end || N <- lists:seq(1, Num)],
list_to_tuple(Refs).
new(Id) ->
Instances = get_instances(),
Count = size(Instances),
Idx = ((Id - 1) rem Count) + 1,
% ?INFO("Worker ~p using instance ~p.\n", [Id, Idx]),
State = #state{instance = element(Idx, Instances)},
{ok, State}.
run(get, KeyGen, _ValueGen, State = #state{instance = Ref}) ->
Key = KeyGen(),
case leveled_bookie:book_get(Ref, "PerfBucket", Key, o) of
{ok, _Value} ->
{ok, State};
not_found ->
{ok, State};
{error, Reason} ->
{error, Reason}
end;
run(put, KeyGen, ValGen, State = #state{instance = Ref}) ->
Key = KeyGen(),
Value = ValGen(),
case leveled_bookie:book_put(Ref, "PerfBucket", Key, Value, []) of
ok ->
{ok, State};
pause ->
timer:sleep(1000),
{ok, State};
{error, Reason} ->
{error, Reason}
end.