umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Tighten SkipList range

Browse source 
Make SkipList range accurate - trim the edges
This commit is contained in:
martinsumner 2016-11-24 23:09:17 +00:00
parent 2d3b1bbf2c
commit 3bbfd8b923
1 changed files with 107 additions and 78 deletions
Download patch file Download diff file Expand all files Collapse all files

185
src/leveled_pmem.erl
View file

@ -158,8 +158,7 @@ merge_trees(StartKey, EndKey, TreeList, LevelMinus1) ->
%%%============================================================================ %%%============================================================================
addkey_to_index(HashIndex, Key, Count) -> addhash_to_index(HashIndex, Hash, Slot, Count) ->
{Hash, Slot} = hash_to_slot(Key),
L = array:get(Slot, HashIndex), L = array:get(Slot, HashIndex),
case lists:member(Hash, L) of case lists:member(Hash, L) of
true -> true ->
@ -197,7 +196,7 @@ merge_indexes_singleslot(HashList, IndexSlot, MergedIndex, L0Slot, Count) ->
HashList), HashList),
{array:set(IndexSlot, UpdHL, MergedIndex), UpdCount}. {array:set(IndexSlot, UpdHL, MergedIndex), UpdCount}.
load_dynamic_skiplist(SkipList, Key, Value, Hash) -> skiplist_put(SkipList, Key, Value, Hash) ->
{MarkerKey, SubList} = lists:foldl(fun({Marker, SL}, Acc) -> {MarkerKey, SubList} = lists:foldl(fun({Marker, SL}, Acc) ->
case Acc of case Acc of
false -> false ->
@ -223,7 +222,7 @@ load_dynamic_skiplist(SkipList, Key, Value, Hash) ->
lists:keyreplace(MarkerKey, 1, SkipList, {MarkerKey, UpdSubList}) lists:keyreplace(MarkerKey, 1, SkipList, {MarkerKey, UpdSubList})
end. end.
generate_balanced_skiplist(UnsortedKVL) -> skiplist_generate(UnsortedKVL) ->
KVL = lists:ukeysort(1, UnsortedKVL), KVL = lists:ukeysort(1, UnsortedKVL),
Slots = length(KVL) div ?SKIP_WIDTH, Slots = length(KVL) div ?SKIP_WIDTH,
SkipList0 = lists:map(fun(X) -> SkipList0 = lists:map(fun(X) ->
@ -242,7 +241,7 @@ generate_balanced_skiplist(UnsortedKVL) ->
SkipList0 SkipList0
end. end.
fetchkey_from_skiplist(SkipList, Key) -> skiplist_get(SkipList, Key) ->
SubList = lists:foldl(fun({SkipKey, SL}, Acc) -> SubList = lists:foldl(fun({SkipKey, SL}, Acc) ->
case {Acc, SkipKey} of case {Acc, SkipKey} of
{null, SkipKey} when SkipKey >= Key -> {null, SkipKey} when SkipKey >= Key ->
@ -264,33 +263,53 @@ fetchkey_from_skiplist(SkipList, Key) ->
end end
end. end.
fetchrange_from_skiplist(SkipList, StartKey, EndKey) -> skiplist_range(SkipList, Start, End) ->
R = lists:foldl(fun({SkipKey, SL}, {Continue, Acc}) -> R = lists:foldl(fun({Mark, SL}, {PassedStart, PassedEnd, Acc, PrevList}) ->
% io:format("SkipKey ~w StartKey ~w EndKey ~w~n", [SkipKey, StartKey, EndKey]),
case Continue of case {PassedStart, PassedEnd} of
true -> {true, true} ->
case StartKey > SkipKey of {true, true, Acc, null};
true -> {false, false} ->
% io:format("StartKey after SkipKey~n"), case Start > Mark of
{true, Acc}; true ->
false -> {false, false, Acc, SL};
case leveled_codec:endkey_passed(EndKey, SkipKey) of false ->
true -> RHS = splitlist_start(Start, PrevList ++ SL),
% io:format("EndKey after SkipKey~n"), case leveled_codec:endkey_passed(End, Mark) of
{false, Acc ++ SL}; true ->
false -> EL = splitlist_end(End, RHS),
% io:format("EndKey before SkipKey~n"), {true, true, EL, null};
{true, Acc ++ SL} false ->
end {true, false, RHS, null}
end; end
false -> end;
{false, Acc} {true, false} ->
end end, case leveled_codec:endkey_passed(End, Mark) of
{true, []}, true ->
SkipList), EL = splitlist_end(End, SL),
{_Bool, SubList} = R, {true, true, Acc ++ EL, null};
false ->
{true, false, Acc ++ SL, null}
end
end end,
{false, false, [], []},
SkipList),
{_Bool1, _Bool2, SubList, _PrevList} = R,
SubList. SubList.
splitlist_start(StartKey, SL) ->
{_LHS, RHS} = lists:splitwith(fun({K, _V}) -> K < StartKey end, SL),
RHS.
splitlist_end(EndKey, SL) ->
{LHS, _RHS} = lists:splitwith(fun({K, _V}) ->
not leveled_codec:endkey_passed(EndKey, K)
end,
SL),
LHS.
%%%============================================================================ %%%============================================================================
%%% Internal Functions %%% Internal Functions
@ -433,7 +452,7 @@ skiplist_test() ->
[timer:now_diff(os:timestamp(), SWaD)]), [timer:now_diff(os:timestamp(), SWaD)]),
SWa = os:timestamp(), SWa = os:timestamp(),
SkipList = generate_balanced_skiplist(KL), SkipList = skiplist_generate(KL),
io:format(user, "Generating skip list with 4000 keys in ~w microseconds~n", io:format(user, "Generating skip list with 4000 keys in ~w microseconds~n",
[timer:now_diff(os:timestamp(), SWa)]), [timer:now_diff(os:timestamp(), SWa)]),
@ -444,56 +463,61 @@ skiplist_test() ->
CheckList5 = lists:sublist(KL, 2800, 100), CheckList5 = lists:sublist(KL, 2800, 100),
CheckList6 = lists:sublist(KL, 1, 10), CheckList6 = lists:sublist(KL, 1, 10),
CheckList7 = lists:nthtail(3800, KL), CheckList7 = lists:nthtail(3800, KL),
CheckList8 = lists:sublist(KL, 3000, 1),
CheckAll = CheckList1 ++ CheckList2 ++ CheckList3 ++ CheckAll = CheckList1 ++ CheckList2 ++ CheckList3 ++
CheckList4 ++ CheckList5 ++ CheckList6 ++ CheckList7, CheckList4 ++ CheckList5 ++ CheckList6 ++ CheckList7,
SWb = os:timestamp(), SWb = os:timestamp(),
lists:foreach(fun({K, V}) -> lists:foreach(fun({K, V}) ->
?assertMatch({K, V}, ?assertMatch({K, V}, skiplist_get(SkipList, K))
fetchkey_from_skiplist(SkipList, K))
end, end,
CheckAll), CheckAll),
io:format(user, "Finding 520 keys took ~w microseconds~n", io:format(user, "Finding 520 keys took ~w microseconds~n",
[timer:now_diff(os:timestamp(), SWb)]), [timer:now_diff(os:timestamp(), SWb)]),
SWc = os:timestamp(), SWc = os:timestamp(),
KR1 = fetchrange_from_skiplist(SkipList, KR1 = skiplist_range(SkipList,
element(1, lists:nth(1, CheckList1)), element(1, lists:nth(1, CheckList1)),
element(1, lists:last(CheckList1))), element(1, lists:last(CheckList1))),
io:format("Result length ~w ~n", [length(KR1)]), io:format("Result length ~w ~n", [length(KR1)]),
?assertMatch(true, length(KR1) >= 100), CompareL1 = length(lists:usort(CheckList1)),
?assertMatch(true, length(KR1) < 400), ?assertMatch(CompareL1, length(KR1)),
KR2 = fetchrange_from_skiplist(SkipList, KR2 = skiplist_range(SkipList,
element(1, lists:nth(1, CheckList2)), element(1, lists:nth(1, CheckList2)),
element(1, lists:last(CheckList2))), element(1, lists:last(CheckList2))),
?assertMatch(true, length(KR2) >= 100), CompareL2 = length(lists:usort(CheckList2)),
?assertMatch(true, length(KR2) < 400), ?assertMatch(CompareL2, length(KR2)),
KR3 = fetchrange_from_skiplist(SkipList, KR3 = skiplist_range(SkipList,
element(1, lists:nth(1, CheckList3)), element(1, lists:nth(1, CheckList3)),
element(1, lists:last(CheckList3))), element(1, lists:last(CheckList3))),
?assertMatch(true, length(KR3) >= 100), CompareL3 = length(lists:usort(CheckList3)),
?assertMatch(true, length(KR3) < 400), ?assertMatch(CompareL3, length(KR3)),
KR4 = fetchrange_from_skiplist(SkipList, KR4 = skiplist_range(SkipList,
element(1, lists:nth(1, CheckList4)), element(1, lists:nth(1, CheckList4)),
element(1, lists:last(CheckList4))), element(1, lists:last(CheckList4))),
?assertMatch(true, length(KR4) >= 100), CompareL4 = length(lists:usort(CheckList4)),
?assertMatch(true, length(KR4) < 400), ?assertMatch(CompareL4, length(KR4)),
KR5 = fetchrange_from_skiplist(SkipList, KR5 = skiplist_range(SkipList,
element(1, lists:nth(1, CheckList5)), element(1, lists:nth(1, CheckList5)),
element(1, lists:last(CheckList5))), element(1, lists:last(CheckList5))),
?assertMatch(true, length(KR5) >= 100), CompareL5 = length(lists:usort(CheckList5)),
?assertMatch(true, length(KR5) < 400), ?assertMatch(CompareL5, length(KR5)),
KR6 = fetchrange_from_skiplist(SkipList, KR6 = skiplist_range(SkipList,
element(1, lists:nth(1, CheckList6)), element(1, lists:nth(1, CheckList6)),
element(1, lists:last(CheckList6))), element(1, lists:last(CheckList6))),
?assertMatch(true, length(KR6) >= 10), CompareL6 = length(lists:usort(CheckList6)),
?assertMatch(true, length(KR6) < 200), ?assertMatch(CompareL6, length(KR6)),
KR7 = fetchrange_from_skiplist(SkipList, KR7 = skiplist_range(SkipList,
element(1, lists:nth(1, CheckList7)), element(1, lists:nth(1, CheckList7)),
element(1, lists:last(CheckList7))), element(1, lists:last(CheckList7))),
?assertMatch(true, length(KR7) >= 10), CompareL7 = length(lists:usort(CheckList7)),
?assertMatch(true, length(KR7) < 200), ?assertMatch(CompareL7, length(KR7)),
io:format(user, "Finding 7 ranges took ~w microseconds~n", KR8 = skiplist_range(SkipList,
element(1, lists:nth(1, CheckList8)),
element(1, lists:last(CheckList8))),
CompareL8 = length(lists:usort(CheckList8)),
?assertMatch(CompareL8, length(KR8)),
io:format(user, "Finding 8 ranges took ~w microseconds~n",
[timer:now_diff(os:timestamp(), SWc)]). [timer:now_diff(os:timestamp(), SWc)]).
hash_index_test() -> hash_index_test() ->
@ -513,11 +537,13 @@ hash_index_test() ->
CheckList4 ++ CheckList5 ++ CheckList6 ++ CheckList7, CheckList4 ++ CheckList5 ++ CheckList6 ++ CheckList7,
SWa = os:timestamp(), SWa = os:timestamp(),
SkipList1 = {HashIndex1, SkipList1, _TC} =
lists:foldl(fun({K, V}, Acc) -> lists:foldl(fun({K, V}, {HI, SL, C}) ->
{H, _Slot} = hash_to_slot(K), {H, S} = hash_to_slot(K),
load_dynamic_skiplist(Acc, K, V, H) end, {UpdHI, UpdC} = addhash_to_index(HI, H, S, C),
?EMPTY_SKIPLIST, UpdSL = skiplist_put(SL, K, V, H),
{UpdHI, UpdSL, UpdC} end,
{HI0, ?EMPTY_SKIPLIST, 0},
KL0), KL0),
io:format(user, "Dynamic load of skiplist took ~w microseconds~n", io:format(user, "Dynamic load of skiplist took ~w microseconds~n",
[timer:now_diff(os:timestamp(), SWa)]), [timer:now_diff(os:timestamp(), SWa)]),
@ -535,7 +561,7 @@ hash_index_test() ->
SWb = os:timestamp(), SWb = os:timestamp(),
lists:foreach(fun({K, V}) -> lists:foreach(fun({K, V}) ->
?assertMatch({K, V}, ?assertMatch({K, V},
fetchkey_from_skiplist(SkipList1, K)) skiplist_get(SkipList1, K))
end, end,
CheckAll), CheckAll),
io:format(user, "Fetching ~w keys from skiplist took ~w microseconds~n", io:format(user, "Fetching ~w keys from skiplist took ~w microseconds~n",
@ -543,7 +569,8 @@ hash_index_test() ->
SWc = os:timestamp(), SWc = os:timestamp(),
{HI1, _C1} = lists:foldl(fun({K, _V}, {HI, C}) -> {HI1, _C1} = lists:foldl(fun({K, _V}, {HI, C}) ->
addkey_to_index(HI, K, C) end, {H, S} = hash_to_slot(K),
addhash_to_index(HI, H, S, C) end,
{HI0, 0}, {HI0, 0},
KL0), KL0),
io:format(user, "Adding ~w keys to hashindex took ~w microseconds~n", io:format(user, "Adding ~w keys to hashindex took ~w microseconds~n",
@ -561,7 +588,8 @@ hash_index_test() ->
SWe = os:timestamp(), SWe = os:timestamp(),
HI2 = new_index(), HI2 = new_index(),
{HI3, _C2} = lists:foldl(fun({K, _V}, {HI, C}) -> {HI3, _C2} = lists:foldl(fun({K, _V}, {HI, C}) ->
addkey_to_index(HI, K, C) end, {H, S} = hash_to_slot(K),
addhash_to_index(HI, H, S, C) end,
{HI2, 0}, {HI2, 0},
KL1), KL1),
io:format(user, "Adding ~w keys to hashindex took ~w microseconds~n", io:format(user, "Adding ~w keys to hashindex took ~w microseconds~n",
@ -576,7 +604,8 @@ hash_index_test() ->
SWg = os:timestamp(), SWg = os:timestamp(),
HI4 = new_index(), HI4 = new_index(),
{HI5, _C3} = lists:foldl(fun({K, _V}, {HI, C}) -> {HI5, _C3} = lists:foldl(fun({K, _V}, {HI, C}) ->
addkey_to_index(HI, K, C) end, {H, S} = hash_to_slot(K),
addhash_to_index(HI, H, S, C) end,
{HI4, 0}, {HI4, 0},
KL1), KL1),
io:format(user, "Adding ~w keys to hashindex took ~w microseconds~n", io:format(user, "Adding ~w keys to hashindex took ~w microseconds~n",