Additional unit test work

Reverts a previous ct test fix

src/leveled_codec.erl

@@ -34,7 +34,6 @@
 
 -export([
         inker_reload_strategy/1,
-        strip_to_keyonly/1,
         strip_to_seqonly/1,
         strip_to_statusonly/1,
         strip_to_keyseqonly/1,
@@ -44,7 +43,6 @@
         endkey_passed/2,
         key_dominates/2,
         maybe_reap_expiredkey/2,
-        print_key/1,
         to_ledgerkey/3,
         to_ledgerkey/5,
         from_ledgerkey/1,
@@ -108,8 +106,6 @@ inker_reload_strategy(AltList) ->
                     ReloadStrategy0,
                     AltList).
 
-strip_to_keyonly({K, _V}) -> K.
-
 strip_to_statusonly({_, {_, St, _, _}}) -> St.
 
 strip_to_seqonly({_, {SeqN, _, _, _}}) -> SeqN.
@@ -252,33 +248,6 @@ create_value_for_journal(Value) ->
 hash(Obj) ->
     erlang:phash2(term_to_binary(Obj)).
 
-% Return a tuple of strings to ease the printing of keys to logs
-print_key(Key) ->
-    {A_STR, B_TERM, C_TERM} = case Key of
-                                    {?STD_TAG, B, K, _SK} ->
-                                        {"Object", B, K};
-                                    {?RIAK_TAG, B, K, _SK} ->
-                                        {"RiakObject", B, K};
-                                    {?IDX_TAG, B, {F, _V}, _K} ->
-                                        {"Index", B, F}
-                                end,
-    B_STR = turn_to_string(B_TERM),
-    C_STR = turn_to_string(C_TERM),
-    {A_STR, B_STR, C_STR}.
-
-turn_to_string(Item) ->
-    if
-        is_binary(Item) == true ->
-            binary_to_list(Item);
-        is_integer(Item) == true ->
-            integer_to_list(Item);
-        is_list(Item) == true ->
-            Item;
-        true ->
-            [Output] = io_lib:format("~w", [Item]),
-            Output
-    end.
-                    
 
 % Compare a key against a query key, only comparing elements that are non-null
 % in the Query key.  This is used for comparing against end keys in queries.
@@ -461,10 +430,6 @@ endkey_passed_test() ->
     ?assertMatch(false, endkey_passed(TestKey, K1)),
     ?assertMatch(true, endkey_passed(TestKey, K2)).
 
-stringcheck_test() ->
-    ?assertMatch("Bucket", turn_to_string("Bucket")),
-    ?assertMatch("Bucket", turn_to_string(<<"Bucket">>)),
-    ?assertMatch("bucket", turn_to_string(bucket)).
 
 %% Test below proved that the overhead of performing hashes was trivial
 %% Maybe 5 microseconds per hash

src/leveled_log.erl

@@ -15,8 +15,8 @@
             sst_timing/3]).         
 
 -define(PUT_LOGPOINT, 20000).
--define(HEAD_LOGPOINT, 160000).
+-define(HEAD_LOGPOINT, 50000).
--define(GET_LOGPOINT, 160000).
+-define(GET_LOGPOINT, 50000).
 -define(SST_LOGPOINT, 20000).
 -define(LOG_LEVEL, [info, warn, error, critical]).
 -define(SAMPLE_RATE, 16).

src/leveled_pmanifest.erl

@@ -151,8 +151,6 @@ save_manifest(Manifest, RootPath) ->
     ok = file:write_file(FP, <<CRC:32/integer, ManBin/binary>>).
 
 
-replace_manifest_entry(Manifest, ManSQN, LevelIdx, Removals, []) ->
-    remove_manifest_entry(Manifest, ManSQN, LevelIdx, Removals);
 replace_manifest_entry(Manifest, ManSQN, LevelIdx, Removals, Additions) ->
     Levels = Manifest#manifest.levels,
     Level = array:get(LevelIdx, Levels),
@@ -753,6 +751,74 @@ keylookup_manifest_test() ->
     ?assertMatch("pid_y3", key_lookup(Man13, 1, LK1_4)),
     ?assertMatch("pid_z5", key_lookup(Man13, 2, LK1_4)).
 
+ext_keylookup_manifest_test() ->
+    RP = "../test",
+    {_Man0, _Man1, _Man2, _Man3, _Man4, _Man5, Man6} = initial_setup(),
+    save_manifest(Man6, RP),
+    
+    E7 = #manifest_entry{start_key={o, "Bucket1", "K997", null},
+                            end_key={o, "Bucket1", "K999", null},
+                            filename="Z7",
+                            owner="pid_z7"},
+    Man7 = insert_manifest_entry(Man6, 2, 2, E7),
+    save_manifest(Man7, RP),
+    ManOpen1 = open_manifest(RP),
+    ?assertMatch(2, get_manifest_sqn(ManOpen1)),
+    
+    Man7FN = filepath(RP, 2, current_manifest),
+    {ok, Bin} = file:read_file(Man7FN),
+    RandPos = random:uniform(bit_size(Bin) - 1),
+    <<Pre:RandPos/bitstring, BitToFlip:1/integer, Rest/bitstring>> = Bin,
+    Flipped = BitToFlip bxor 1,
+    ok  = file:write_file(Man7FN,
+                            <<Pre:RandPos/bitstring,
+                                Flipped:1/integer,
+                                Rest/bitstring>>),
+    
+    ?assertMatch(2, get_manifest_sqn(Man7)),
+    
+    ManOpen2 = open_manifest(RP),
+    ?assertMatch(1, get_manifest_sqn(ManOpen2)),
+    
+    E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"},
+                            end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"},
+                            filename="Z1",
+                            owner="pid_z1"},
+    E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"},
+                                end_key={o, "Bucket1", "K71", null},
+                                filename="Z2",
+                                owner="pid_z2"},
+    E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null},
+                            end_key={o, "Bucket1", "K993", null},
+                            filename="Z3",
+                            owner="pid_z3"},
+    
+    E1_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld4"}, "K8"},
+                            end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K62"},
+                            owner="pid_y1",
+                            filename="Y1"},
+    E2_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K67"},
+                            end_key={o, "Bucket1", "K45", null},
+                            owner="pid_y2",
+                            filename="Y2"},
+    E3_2 = #manifest_entry{start_key={o, "Bucket1", "K47", null},
+                            end_key={o, "Bucket1", "K812", null},
+                            owner="pid_y3",
+                            filename="Y3"},
+    E4_2 = #manifest_entry{start_key={o, "Bucket1", "K815", null},
+                            end_key={o, "Bucket1", "K998", null},
+                            owner="pid_y4",
+                            filename="Y4"},
+    
+    Man8 = replace_manifest_entry(ManOpen2, 2, 1, E1, E1_2),
+    Man9 = remove_manifest_entry(Man8, 2, 1, [E2, E3]),
+    Man10 = insert_manifest_entry(Man9, 2, 1, [E2_2, E3_2, E4_2]),
+    ?assertMatch(2, get_manifest_sqn(Man10)),
+    
+    LK1_4 = {o, "Bucket1", "K75", null},
+    ?assertMatch("pid_y3", key_lookup(Man10, 1, LK1_4)),
+    ?assertMatch("pid_z5", key_lookup(Man10, 2, LK1_4)).
+
 
 rangequery_manifest_test() ->
     {_Man0, _Man1, _Man2, _Man3, _Man4, _Man5, Man6} = initial_setup(),

src/leveled_tree.erl

@@ -91,7 +91,7 @@ search(Key, {tree, _L, Tree}, StartKeyFun) ->
             none;
         {_NK, SL, _Iter} ->
             {K, V} = lookup_best(Key, SL),
-            case K < StartKeyFun(V) of
+            case Key < StartKeyFun(V) of
                 true ->
                     none;
                 false ->
@@ -105,7 +105,7 @@ search(Key, {idxt, _L, {TLI, IDX}}, StartKeyFun) ->
             none;
         {_NK, ListID, _Iter} ->
             {K, V} = lookup_best(Key, element(ListID, TLI)),
-            case K < StartKeyFun(V) of
+            case Key < StartKeyFun(V) of
                 true ->
                     none;
                 false ->
@@ -114,15 +114,18 @@ search(Key, {idxt, _L, {TLI, IDX}}, StartKeyFun) ->
     end;
 search(Key, {skpl, _L, SkipList}, StartKeyFun) ->
     SL0 = skpl_getsublist(Key, SkipList),
-    {K, V} = lookup_best(Key, SL0),
+    case lookup_best(Key, SL0) of
-    case K < StartKeyFun(V) of
+        {K, V} ->
-        true ->
+            case Key < StartKeyFun(V) of
-            none;
+                true ->
-        false ->
+                    none;
-            {K, V}
+                false ->
+                    {K, V}
+            end;
+        none ->
+            none
     end.
 
-
 match_range(StartRange, EndRange, Tree) ->
     EndRangeFun =
         fun(ER, FirstRHSKey, _FirstRHSValue) ->
@@ -244,6 +247,8 @@ lookup_match(Key, KVList) ->
             {value, Value}
     end.
 
+lookup_best(_Key, []) ->
+    none;
 lookup_best(Key, [{EK, EV}|_Tail]) when EK >= Key ->
     {EK, EV};
 lookup_best(Key, [_Top|Tail]) ->
@@ -489,14 +494,8 @@ generate_randomkeys(Seqn, Count, BucketRangeLow, BucketRangeHigh) ->
 generate_randomkeys(_Seqn, 0, Acc, _BucketLow, _BucketHigh) ->
     Acc;
 generate_randomkeys(Seqn, Count, Acc, BucketLow, BRange) ->
-    BNumber =
+    BRand = random:uniform(BRange),
-        case BRange of
+    BNumber = string:right(integer_to_list(BucketLow + BRand), 4, $0),
-            0 ->
-                string:right(integer_to_list(BucketLow), 4, $0);
-            _ ->
-                BRand = random:uniform(BRange),
-                string:right(integer_to_list(BucketLow + BRand), 4, $0)
-        end,
     KNumber = string:right(integer_to_list(random:uniform(1000)), 4, $0),
     {K, V} = {{o, "Bucket" ++ BNumber, "Key" ++ KNumber, null},
                 {Seqn, {active, infinity}, null}},
@@ -525,7 +524,6 @@ search_test_by_type(Type) ->
     T = from_orderedlist(KL, Type),
     
     StartKeyFun = fun(V) -> V end,
-    
     SW = os:timestamp(),
     ?assertMatch([], search_range(0, 1, T, StartKeyFun)),
     ?assertMatch([], search_range(201, 202, T, StartKeyFun)),
@@ -539,8 +537,59 @@ search_test_by_type(Type) ->
     ?assertMatch(48, length(search_range(5, 197, T, StartKeyFun))),
     io:format(user, "10 range tests with type ~w in ~w microseconds~n",
                 [Type, timer:now_diff(os:timestamp(), SW)]).
-    
 
+
+tree_oor_test() ->
+    outofrange_test_by_type(tree).
+
+idxt_oor_test() ->
+    outofrange_test_by_type(idxt).
+
+skpl_oor_test() ->
+    outofrange_test_by_type(skpl).
+
+outofrange_test_by_type(Type) ->
+    MapFun =
+        fun(N) ->
+            {N * 4, N * 4 - 2}
+        end,
+    KL = lists:map(MapFun, lists:seq(1, 50)),
+    T = from_orderedlist(KL, Type),
+
+    io:format("Out of range searches~n"),
+    ?assertMatch(none, match(0, T)),
+    ?assertMatch(none, match(5, T)),
+    ?assertMatch(none, match(97, T)),
+    ?assertMatch(none, match(197, T)),
+    ?assertMatch(none, match(201, T)),
+    
+    StartKeyFun = fun(V) -> V end,
+    
+    ?assertMatch(none, search(0, T, StartKeyFun)),
+    ?assertMatch(none, search(5, T, StartKeyFun)),
+    ?assertMatch(none, search(97, T, StartKeyFun)),
+    ?assertMatch(none, search(197, T, StartKeyFun)),
+    ?assertMatch(none, search(201, T, StartKeyFun)).
+
+tree_tolist_test() ->
+    tolist_test_by_type(tree).
+
+idxt_tolist_test() ->
+    tolist_test_by_type(idxt).
+
+skpl_tolist_test() ->
+    tolist_test_by_type(skpl).
+
+tolist_test_by_type(Type) ->
+    MapFun =
+        fun(N) ->
+            {N * 4, N * 4 - 2}
+        end,
+    KL = lists:map(MapFun, lists:seq(1, 50)),
+    T = from_orderedlist(KL, Type),
+    T_Reverse = to_list(T),
+    ?assertMatch(KL, T_Reverse).
+    
 tree_timing_test() ->
     tree_test_by_(16, tree, 4000),
     tree_test_by_(8, tree, 1000),
@@ -552,6 +601,7 @@ idxt_timing_test() ->
     tree_test_by_(4, idxt, 256).
 
 skpl_timing_test() ->
+    tree_test_by_(auto, skpl, 6000),
     tree_test_by_(auto, skpl, 4000),
     tree_test_by_(auto, skpl, 1000),
     tree_test_by_(auto, skpl, 256).