umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Additional unit test work

Browse source 
Reverts a previous ct test fix
This commit is contained in:
martinsumner 2017-01-23 15:15:40 +00:00
parent 5105df1cd6
commit 90c920fe86
4 changed files with 139 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

35
src/leveled_codec.erl
View file

@ -34,7 +34,6 @@
-export([
inker_reload_strategy/1,
strip_to_keyonly/1,
strip_to_seqonly/1,
strip_to_statusonly/1,
strip_to_keyseqonly/1,
@ -44,7 +43,6 @@
endkey_passed/2,
key_dominates/2,
maybe_reap_expiredkey/2,
print_key/1,
to_ledgerkey/3,
to_ledgerkey/5,
from_ledgerkey/1,
@ -108,8 +106,6 @@ inker_reload_strategy(AltList) ->
ReloadStrategy0,
AltList).
strip_to_keyonly({K, _V}) -> K.
strip_to_statusonly({_, {_, St, _, _}}) -> St.
strip_to_seqonly({_, {SeqN, _, _, _}}) -> SeqN.
@ -252,33 +248,6 @@ create_value_for_journal(Value) ->
hash(Obj) ->
erlang:phash2(term_to_binary(Obj)).
% Return a tuple of strings to ease the printing of keys to logs
print_key(Key) ->
{A_STR, B_TERM, C_TERM} = case Key of
{?STD_TAG, B, K, _SK} ->
{"Object", B, K};
{?RIAK_TAG, B, K, _SK} ->
{"RiakObject", B, K};
{?IDX_TAG, B, {F, _V}, _K} ->
{"Index", B, F}
end,
B_STR = turn_to_string(B_TERM),
C_STR = turn_to_string(C_TERM),
{A_STR, B_STR, C_STR}.
turn_to_string(Item) ->
if
is_binary(Item) == true ->
binary_to_list(Item);
is_integer(Item) == true ->
integer_to_list(Item);
is_list(Item) == true ->
Item;
true ->
[Output] = io_lib:format("~w", [Item]),
Output
end.
% Compare a key against a query key, only comparing elements that are non-null
% in the Query key. This is used for comparing against end keys in queries.
@ -461,10 +430,6 @@ endkey_passed_test() ->
?assertMatch(false, endkey_passed(TestKey, K1)),
?assertMatch(true, endkey_passed(TestKey, K2)).
stringcheck_test() ->
?assertMatch("Bucket", turn_to_string("Bucket")),
?assertMatch("Bucket", turn_to_string(<<"Bucket">>)),
?assertMatch("bucket", turn_to_string(bucket)).
%% Test below proved that the overhead of performing hashes was trivial
%% Maybe 5 microseconds per hash

4
src/leveled_log.erl
View file

@ -15,8 +15,8 @@
sst_timing/3]).
-define(PUT_LOGPOINT, 20000).
-define(HEAD_LOGPOINT, 160000).
-define(GET_LOGPOINT, 160000).
-define(HEAD_LOGPOINT, 50000).
-define(GET_LOGPOINT, 50000).
-define(SST_LOGPOINT, 20000).
-define(LOG_LEVEL, [info, warn, error, critical]).
-define(SAMPLE_RATE, 16).

70
src/leveled_pmanifest.erl
View file

@ -151,8 +151,6 @@ save_manifest(Manifest, RootPath) ->
ok = file:write_file(FP, <<CRC:32/integer, ManBin/binary>>).
replace_manifest_entry(Manifest, ManSQN, LevelIdx, Removals, []) ->
remove_manifest_entry(Manifest, ManSQN, LevelIdx, Removals);
replace_manifest_entry(Manifest, ManSQN, LevelIdx, Removals, Additions) ->
Levels = Manifest#manifest.levels,
Level = array:get(LevelIdx, Levels),
@ -753,6 +751,74 @@ keylookup_manifest_test() ->
?assertMatch("pid_y3", key_lookup(Man13, 1, LK1_4)),
?assertMatch("pid_z5", key_lookup(Man13, 2, LK1_4)).
ext_keylookup_manifest_test() ->
RP = "../test",
{_Man0, _Man1, _Man2, _Man3, _Man4, _Man5, Man6} = initial_setup(),
save_manifest(Man6, RP),
E7 = #manifest_entry{start_key={o, "Bucket1", "K997", null},
end_key={o, "Bucket1", "K999", null},
filename="Z7",
owner="pid_z7"},
Man7 = insert_manifest_entry(Man6, 2, 2, E7),
save_manifest(Man7, RP),
ManOpen1 = open_manifest(RP),
?assertMatch(2, get_manifest_sqn(ManOpen1)),
Man7FN = filepath(RP, 2, current_manifest),
{ok, Bin} = file:read_file(Man7FN),
RandPos = random:uniform(bit_size(Bin) - 1),
<<Pre:RandPos/bitstring, BitToFlip:1/integer, Rest/bitstring>> = Bin,
Flipped = BitToFlip bxor 1,
ok = file:write_file(Man7FN,
<<Pre:RandPos/bitstring,
Flipped:1/integer,
Rest/bitstring>>),
?assertMatch(2, get_manifest_sqn(Man7)),
ManOpen2 = open_manifest(RP),
?assertMatch(1, get_manifest_sqn(ManOpen2)),
E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"},
end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"},
filename="Z1",
owner="pid_z1"},
E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"},
end_key={o, "Bucket1", "K71", null},
filename="Z2",
owner="pid_z2"},
E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null},
end_key={o, "Bucket1", "K993", null},
filename="Z3",
owner="pid_z3"},
E1_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld4"}, "K8"},
end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K62"},
owner="pid_y1",
filename="Y1"},
E2_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K67"},
end_key={o, "Bucket1", "K45", null},
owner="pid_y2",
filename="Y2"},
E3_2 = #manifest_entry{start_key={o, "Bucket1", "K47", null},
end_key={o, "Bucket1", "K812", null},
owner="pid_y3",
filename="Y3"},
E4_2 = #manifest_entry{start_key={o, "Bucket1", "K815", null},
end_key={o, "Bucket1", "K998", null},
owner="pid_y4",
filename="Y4"},
Man8 = replace_manifest_entry(ManOpen2, 2, 1, E1, E1_2),
Man9 = remove_manifest_entry(Man8, 2, 1, [E2, E3]),
Man10 = insert_manifest_entry(Man9, 2, 1, [E2_2, E3_2, E4_2]),
?assertMatch(2, get_manifest_sqn(Man10)),
LK1_4 = {o, "Bucket1", "K75", null},
?assertMatch("pid_y3", key_lookup(Man10, 1, LK1_4)),
?assertMatch("pid_z5", key_lookup(Man10, 2, LK1_4)).
rangequery_manifest_test() ->
{_Man0, _Man1, _Man2, _Man3, _Man4, _Man5, Man6} = initial_setup(),

86
src/leveled_tree.erl
View file

@ -91,7 +91,7 @@ search(Key, {tree, _L, Tree}, StartKeyFun) ->
none;
{_NK, SL, _Iter} ->
{K, V} = lookup_best(Key, SL),
case K < StartKeyFun(V) of
case Key < StartKeyFun(V) of
true ->
none;
false ->
@ -105,7 +105,7 @@ search(Key, {idxt, _L, {TLI, IDX}}, StartKeyFun) ->
none;
{_NK, ListID, _Iter} ->
{K, V} = lookup_best(Key, element(ListID, TLI)),
case K < StartKeyFun(V) of
case Key < StartKeyFun(V) of
true ->
none;
false ->
@ -114,15 +114,18 @@ search(Key, {idxt, _L, {TLI, IDX}}, StartKeyFun) ->
end;
search(Key, {skpl, _L, SkipList}, StartKeyFun) ->
SL0 = skpl_getsublist(Key, SkipList),
{K, V} = lookup_best(Key, SL0),
case K < StartKeyFun(V) of
true ->
none;
false ->
{K, V}
case lookup_best(Key, SL0) of
{K, V} ->
case Key < StartKeyFun(V) of
true ->
none;
false ->
{K, V}
end;
none ->
none
end.
match_range(StartRange, EndRange, Tree) ->
EndRangeFun =
fun(ER, FirstRHSKey, _FirstRHSValue) ->
@ -244,6 +247,8 @@ lookup_match(Key, KVList) ->
{value, Value}
end.
lookup_best(_Key, []) ->
none;
lookup_best(Key, [{EK, EV}|_Tail]) when EK >= Key ->
{EK, EV};
lookup_best(Key, [_Top|Tail]) ->
@ -489,14 +494,8 @@ generate_randomkeys(Seqn, Count, BucketRangeLow, BucketRangeHigh) ->
generate_randomkeys(_Seqn, 0, Acc, _BucketLow, _BucketHigh) ->
Acc;
generate_randomkeys(Seqn, Count, Acc, BucketLow, BRange) ->
BNumber =
case BRange of
0 ->
string:right(integer_to_list(BucketLow), 4, $0);
_ ->
BRand = random:uniform(BRange),
string:right(integer_to_list(BucketLow + BRand), 4, $0)
end,
BRand = random:uniform(BRange),
BNumber = string:right(integer_to_list(BucketLow + BRand), 4, $0),
KNumber = string:right(integer_to_list(random:uniform(1000)), 4, $0),
{K, V} = {{o, "Bucket" ++ BNumber, "Key" ++ KNumber, null},
{Seqn, {active, infinity}, null}},
@ -525,7 +524,6 @@ search_test_by_type(Type) ->
T = from_orderedlist(KL, Type),
StartKeyFun = fun(V) -> V end,
SW = os:timestamp(),
?assertMatch([], search_range(0, 1, T, StartKeyFun)),
?assertMatch([], search_range(201, 202, T, StartKeyFun)),
@ -541,6 +539,57 @@ search_test_by_type(Type) ->
[Type, timer:now_diff(os:timestamp(), SW)]).
tree_oor_test() ->
outofrange_test_by_type(tree).
idxt_oor_test() ->
outofrange_test_by_type(idxt).
skpl_oor_test() ->
outofrange_test_by_type(skpl).
outofrange_test_by_type(Type) ->
MapFun =
fun(N) ->
{N * 4, N * 4 - 2}
end,
KL = lists:map(MapFun, lists:seq(1, 50)),
T = from_orderedlist(KL, Type),
io:format("Out of range searches~n"),
?assertMatch(none, match(0, T)),
?assertMatch(none, match(5, T)),
?assertMatch(none, match(97, T)),
?assertMatch(none, match(197, T)),
?assertMatch(none, match(201, T)),
StartKeyFun = fun(V) -> V end,
?assertMatch(none, search(0, T, StartKeyFun)),
?assertMatch(none, search(5, T, StartKeyFun)),
?assertMatch(none, search(97, T, StartKeyFun)),
?assertMatch(none, search(197, T, StartKeyFun)),
?assertMatch(none, search(201, T, StartKeyFun)).
tree_tolist_test() ->
tolist_test_by_type(tree).
idxt_tolist_test() ->
tolist_test_by_type(idxt).
skpl_tolist_test() ->
tolist_test_by_type(skpl).
tolist_test_by_type(Type) ->
MapFun =
fun(N) ->
{N * 4, N * 4 - 2}
end,
KL = lists:map(MapFun, lists:seq(1, 50)),
T = from_orderedlist(KL, Type),
T_Reverse = to_list(T),
?assertMatch(KL, T_Reverse).
tree_timing_test() ->
tree_test_by_(16, tree, 4000),
tree_test_by_(8, tree, 1000),
@ -552,6 +601,7 @@ idxt_timing_test() ->
tree_test_by_(4, idxt, 256).
skpl_timing_test() ->
tree_test_by_(auto, skpl, 6000),
tree_test_by_(auto, skpl, 4000),
tree_test_by_(auto, skpl, 1000),
tree_test_by_(auto, skpl, 256).