From 90c920fe86f2f4511bd838ab120717071fbd7767 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 23 Jan 2017 15:15:40 +0000 Subject: [PATCH] Additional unit test work Reverts a previous ct test fix --- src/leveled_codec.erl | 35 ---------------- src/leveled_log.erl | 4 +- src/leveled_pmanifest.erl | 70 ++++++++++++++++++++++++++++++- src/leveled_tree.erl | 88 ++++++++++++++++++++++++++++++--------- 4 files changed, 139 insertions(+), 58 deletions(-) diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index 6dbbff4..6360e2b 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -34,7 +34,6 @@ -export([ inker_reload_strategy/1, - strip_to_keyonly/1, strip_to_seqonly/1, strip_to_statusonly/1, strip_to_keyseqonly/1, @@ -44,7 +43,6 @@ endkey_passed/2, key_dominates/2, maybe_reap_expiredkey/2, - print_key/1, to_ledgerkey/3, to_ledgerkey/5, from_ledgerkey/1, @@ -108,8 +106,6 @@ inker_reload_strategy(AltList) -> ReloadStrategy0, AltList). -strip_to_keyonly({K, _V}) -> K. - strip_to_statusonly({_, {_, St, _, _}}) -> St. strip_to_seqonly({_, {SeqN, _, _, _}}) -> SeqN. @@ -252,33 +248,6 @@ create_value_for_journal(Value) -> hash(Obj) -> erlang:phash2(term_to_binary(Obj)). -% Return a tuple of strings to ease the printing of keys to logs -print_key(Key) -> - {A_STR, B_TERM, C_TERM} = case Key of - {?STD_TAG, B, K, _SK} -> - {"Object", B, K}; - {?RIAK_TAG, B, K, _SK} -> - {"RiakObject", B, K}; - {?IDX_TAG, B, {F, _V}, _K} -> - {"Index", B, F} - end, - B_STR = turn_to_string(B_TERM), - C_STR = turn_to_string(C_TERM), - {A_STR, B_STR, C_STR}. - -turn_to_string(Item) -> - if - is_binary(Item) == true -> - binary_to_list(Item); - is_integer(Item) == true -> - integer_to_list(Item); - is_list(Item) == true -> - Item; - true -> - [Output] = io_lib:format("~w", [Item]), - Output - end. - % Compare a key against a query key, only comparing elements that are non-null % in the Query key. This is used for comparing against end keys in queries. @@ -461,10 +430,6 @@ endkey_passed_test() -> ?assertMatch(false, endkey_passed(TestKey, K1)), ?assertMatch(true, endkey_passed(TestKey, K2)). -stringcheck_test() -> - ?assertMatch("Bucket", turn_to_string("Bucket")), - ?assertMatch("Bucket", turn_to_string(<<"Bucket">>)), - ?assertMatch("bucket", turn_to_string(bucket)). %% Test below proved that the overhead of performing hashes was trivial %% Maybe 5 microseconds per hash diff --git a/src/leveled_log.erl b/src/leveled_log.erl index f3dbb25..618c871 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -15,8 +15,8 @@ sst_timing/3]). -define(PUT_LOGPOINT, 20000). --define(HEAD_LOGPOINT, 160000). --define(GET_LOGPOINT, 160000). +-define(HEAD_LOGPOINT, 50000). +-define(GET_LOGPOINT, 50000). -define(SST_LOGPOINT, 20000). -define(LOG_LEVEL, [info, warn, error, critical]). -define(SAMPLE_RATE, 16). diff --git a/src/leveled_pmanifest.erl b/src/leveled_pmanifest.erl index d8e2ba4..18e80c4 100644 --- a/src/leveled_pmanifest.erl +++ b/src/leveled_pmanifest.erl @@ -151,8 +151,6 @@ save_manifest(Manifest, RootPath) -> ok = file:write_file(FP, <>). -replace_manifest_entry(Manifest, ManSQN, LevelIdx, Removals, []) -> - remove_manifest_entry(Manifest, ManSQN, LevelIdx, Removals); replace_manifest_entry(Manifest, ManSQN, LevelIdx, Removals, Additions) -> Levels = Manifest#manifest.levels, Level = array:get(LevelIdx, Levels), @@ -753,6 +751,74 @@ keylookup_manifest_test() -> ?assertMatch("pid_y3", key_lookup(Man13, 1, LK1_4)), ?assertMatch("pid_z5", key_lookup(Man13, 2, LK1_4)). +ext_keylookup_manifest_test() -> + RP = "../test", + {_Man0, _Man1, _Man2, _Man3, _Man4, _Man5, Man6} = initial_setup(), + save_manifest(Man6, RP), + + E7 = #manifest_entry{start_key={o, "Bucket1", "K997", null}, + end_key={o, "Bucket1", "K999", null}, + filename="Z7", + owner="pid_z7"}, + Man7 = insert_manifest_entry(Man6, 2, 2, E7), + save_manifest(Man7, RP), + ManOpen1 = open_manifest(RP), + ?assertMatch(2, get_manifest_sqn(ManOpen1)), + + Man7FN = filepath(RP, 2, current_manifest), + {ok, Bin} = file:read_file(Man7FN), + RandPos = random:uniform(bit_size(Bin) - 1), + <> = Bin, + Flipped = BitToFlip bxor 1, + ok = file:write_file(Man7FN, + <>), + + ?assertMatch(2, get_manifest_sqn(Man7)), + + ManOpen2 = open_manifest(RP), + ?assertMatch(1, get_manifest_sqn(ManOpen2)), + + E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, + end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, + filename="Z1", + owner="pid_z1"}, + E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"}, + end_key={o, "Bucket1", "K71", null}, + filename="Z2", + owner="pid_z2"}, + E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null}, + end_key={o, "Bucket1", "K993", null}, + filename="Z3", + owner="pid_z3"}, + + E1_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld4"}, "K8"}, + end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K62"}, + owner="pid_y1", + filename="Y1"}, + E2_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K67"}, + end_key={o, "Bucket1", "K45", null}, + owner="pid_y2", + filename="Y2"}, + E3_2 = #manifest_entry{start_key={o, "Bucket1", "K47", null}, + end_key={o, "Bucket1", "K812", null}, + owner="pid_y3", + filename="Y3"}, + E4_2 = #manifest_entry{start_key={o, "Bucket1", "K815", null}, + end_key={o, "Bucket1", "K998", null}, + owner="pid_y4", + filename="Y4"}, + + Man8 = replace_manifest_entry(ManOpen2, 2, 1, E1, E1_2), + Man9 = remove_manifest_entry(Man8, 2, 1, [E2, E3]), + Man10 = insert_manifest_entry(Man9, 2, 1, [E2_2, E3_2, E4_2]), + ?assertMatch(2, get_manifest_sqn(Man10)), + + LK1_4 = {o, "Bucket1", "K75", null}, + ?assertMatch("pid_y3", key_lookup(Man10, 1, LK1_4)), + ?assertMatch("pid_z5", key_lookup(Man10, 2, LK1_4)). + rangequery_manifest_test() -> {_Man0, _Man1, _Man2, _Man3, _Man4, _Man5, Man6} = initial_setup(), diff --git a/src/leveled_tree.erl b/src/leveled_tree.erl index 23404f5..8804847 100644 --- a/src/leveled_tree.erl +++ b/src/leveled_tree.erl @@ -91,7 +91,7 @@ search(Key, {tree, _L, Tree}, StartKeyFun) -> none; {_NK, SL, _Iter} -> {K, V} = lookup_best(Key, SL), - case K < StartKeyFun(V) of + case Key < StartKeyFun(V) of true -> none; false -> @@ -105,7 +105,7 @@ search(Key, {idxt, _L, {TLI, IDX}}, StartKeyFun) -> none; {_NK, ListID, _Iter} -> {K, V} = lookup_best(Key, element(ListID, TLI)), - case K < StartKeyFun(V) of + case Key < StartKeyFun(V) of true -> none; false -> @@ -114,15 +114,18 @@ search(Key, {idxt, _L, {TLI, IDX}}, StartKeyFun) -> end; search(Key, {skpl, _L, SkipList}, StartKeyFun) -> SL0 = skpl_getsublist(Key, SkipList), - {K, V} = lookup_best(Key, SL0), - case K < StartKeyFun(V) of - true -> - none; - false -> - {K, V} + case lookup_best(Key, SL0) of + {K, V} -> + case Key < StartKeyFun(V) of + true -> + none; + false -> + {K, V} + end; + none -> + none end. - match_range(StartRange, EndRange, Tree) -> EndRangeFun = fun(ER, FirstRHSKey, _FirstRHSValue) -> @@ -244,6 +247,8 @@ lookup_match(Key, KVList) -> {value, Value} end. +lookup_best(_Key, []) -> + none; lookup_best(Key, [{EK, EV}|_Tail]) when EK >= Key -> {EK, EV}; lookup_best(Key, [_Top|Tail]) -> @@ -489,14 +494,8 @@ generate_randomkeys(Seqn, Count, BucketRangeLow, BucketRangeHigh) -> generate_randomkeys(_Seqn, 0, Acc, _BucketLow, _BucketHigh) -> Acc; generate_randomkeys(Seqn, Count, Acc, BucketLow, BRange) -> - BNumber = - case BRange of - 0 -> - string:right(integer_to_list(BucketLow), 4, $0); - _ -> - BRand = random:uniform(BRange), - string:right(integer_to_list(BucketLow + BRand), 4, $0) - end, + BRand = random:uniform(BRange), + BNumber = string:right(integer_to_list(BucketLow + BRand), 4, $0), KNumber = string:right(integer_to_list(random:uniform(1000)), 4, $0), {K, V} = {{o, "Bucket" ++ BNumber, "Key" ++ KNumber, null}, {Seqn, {active, infinity}, null}}, @@ -525,7 +524,6 @@ search_test_by_type(Type) -> T = from_orderedlist(KL, Type), StartKeyFun = fun(V) -> V end, - SW = os:timestamp(), ?assertMatch([], search_range(0, 1, T, StartKeyFun)), ?assertMatch([], search_range(201, 202, T, StartKeyFun)), @@ -539,8 +537,59 @@ search_test_by_type(Type) -> ?assertMatch(48, length(search_range(5, 197, T, StartKeyFun))), io:format(user, "10 range tests with type ~w in ~w microseconds~n", [Type, timer:now_diff(os:timestamp(), SW)]). - + +tree_oor_test() -> + outofrange_test_by_type(tree). + +idxt_oor_test() -> + outofrange_test_by_type(idxt). + +skpl_oor_test() -> + outofrange_test_by_type(skpl). + +outofrange_test_by_type(Type) -> + MapFun = + fun(N) -> + {N * 4, N * 4 - 2} + end, + KL = lists:map(MapFun, lists:seq(1, 50)), + T = from_orderedlist(KL, Type), + + io:format("Out of range searches~n"), + ?assertMatch(none, match(0, T)), + ?assertMatch(none, match(5, T)), + ?assertMatch(none, match(97, T)), + ?assertMatch(none, match(197, T)), + ?assertMatch(none, match(201, T)), + + StartKeyFun = fun(V) -> V end, + + ?assertMatch(none, search(0, T, StartKeyFun)), + ?assertMatch(none, search(5, T, StartKeyFun)), + ?assertMatch(none, search(97, T, StartKeyFun)), + ?assertMatch(none, search(197, T, StartKeyFun)), + ?assertMatch(none, search(201, T, StartKeyFun)). + +tree_tolist_test() -> + tolist_test_by_type(tree). + +idxt_tolist_test() -> + tolist_test_by_type(idxt). + +skpl_tolist_test() -> + tolist_test_by_type(skpl). + +tolist_test_by_type(Type) -> + MapFun = + fun(N) -> + {N * 4, N * 4 - 2} + end, + KL = lists:map(MapFun, lists:seq(1, 50)), + T = from_orderedlist(KL, Type), + T_Reverse = to_list(T), + ?assertMatch(KL, T_Reverse). + tree_timing_test() -> tree_test_by_(16, tree, 4000), tree_test_by_(8, tree, 1000), @@ -552,6 +601,7 @@ idxt_timing_test() -> tree_test_by_(4, idxt, 256). skpl_timing_test() -> + tree_test_by_(auto, skpl, 6000), tree_test_by_(auto, skpl, 4000), tree_test_by_(auto, skpl, 1000), tree_test_by_(auto, skpl, 256).