From b1a3b4ad137b94646acebe690ea9b1ec932f5911 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 24 Dec 2016 00:02:06 +0000 Subject: [PATCH] Switch slot to gb_trees and size of 128 --- src/leveled_sst.erl | 82 ++++++++++++++++++--------------------- src/leveled_tinybloom.erl | 28 +++---------- 2 files changed, 43 insertions(+), 67 deletions(-) diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index f728643..948972f 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -3,7 +3,34 @@ %% A FSM module intended to wrap a persisted, ordered view of Keys and Values %% %% The persisted view is built from a list (which may be created by merging -%% multiple lists) +%% multiple lists). +%% +%% +%% -------- Slots --------- +%% +%% The view is built from sublists referred to as slot. Each slot is up to 128 +%% keys and values in size. The slots are each themselves a gb_tree. The +%% gb_tree is slightly slower than the skiplist at fetch time, and doesn't +%% support directly the useful to_range function. However the from_orddict +%% capability is much faster than from_sortedlist in skiplist, saving on CPU +%% at sst build time: +%% +%% Skiplist: +%% build and serialise slot 3233 microseconds +%% de-serialise and check * 128 - 14669 microseconds +%% flatten back to list - 164 microseconds +%% +%% GBTree: +%% build and serialise tree 402 microseconds +%% de-serialise and check * 128 - 15263 microseconds +%% flatten back to list - 175 microseconds +%% +%% The performance advantage at lookup time is no negligible as the time to +%% de-deserialise for each check is dominant. This time grows linearly with +%% the size of the slot, wherease the serialisation time is relatively constant +%% with growth. So bigger slots would be quicker to build, but the penalty for +%% that speed is too high at lookup time. + -module(leveled_sst). @@ -16,7 +43,6 @@ -record(slot_index_value, {slot_id :: integer(), bloom :: dict:dict(), - cache :: tuple(), start_position :: integer(), length :: integer()}). @@ -63,13 +89,12 @@ build_all_slots(KVList, StartPosition, AllHashes, SlotID, SlotIndex) -> [{LastKey, SlotIndexV}|SlotIndex]). - build_slot(KVList, HashList) when length(KVList) =< ?SLOT_SIZE -> - SkipList = leveled_skiplist:to_sstlist(KVList), + Tree = gb_trees:from_orddict(KVList), Bloom = lists:foldr(fun leveled_tinybloom:tiny_enter/2, leveled_tinybloom:tiny_empty(), HashList), - SlotBin = term_to_binary(SkipList, [{compressed, ?COMPRESSION_LEVEL}]), + SlotBin = term_to_binary(Tree, [{compressed, ?COMPRESSION_LEVEL}]), {SlotBin, Bloom}. is_check_slot_required(_Hash, none) -> @@ -80,20 +105,14 @@ is_check_slot_required(Hash, Bloom) -> lookup_in_slot(Key, {pointer, Handle, Pos, Length}) -> lookup_in_slot(Key, read_slot(Handle, Pos, Length)); lookup_in_slot(Key, SlotBin) -> - SkipList = binary_to_term(SlotBin), - leveled_skiplist:lookup(Key, SkipList). - -range_from_slot(StartKey, EndKey, {pointer, Handle, Pos, Length}) -> - range_from_slot(StartKey, EndKey, read_slot(Handle, Pos, Length)); -range_from_slot(StartKey, EndKey, SlotBin) -> - SkipList = binary_to_term(SlotBin), - leveled_skiplist:to_range(SkipList, StartKey, EndKey). + Tree = binary_to_term(SlotBin), + gb_trees:lookup(Key, Tree). all_from_slot({pointer, Handle, Pos, Length}) -> all_from_slot(read_slot(Handle, Pos, Length)); all_from_slot(SlotBin) -> SkipList = binary_to_term(SlotBin), - leveled_skiplist:to_list(SkipList). + gb_trees:to_list(SkipList). read_slot(_Handle, _Pos, _Length) -> @@ -148,7 +167,6 @@ simple_slotbin_test() -> {_SQN, H} = leveled_codec:strip_to_seqnhashonly({K, V}), {hash, H} end, HashList = lists:map(ExtractHashFun, KVList1), - io:format(user, "~nSkiplist with bloom timing test~n", []), SW0 = os:timestamp(), {SlotBin0, Bloom0} = build_slot(KVList1, HashList), io:format(user, "Slot built in ~w microseconds with size ~w~n", @@ -166,34 +184,10 @@ simple_slotbin_test() -> KVList1), io:format(user, "Slot checked for all keys in ~w microseconds~n", [timer:now_diff(os:timestamp(), SW1)]), - SW5 = os:timestamp(), - leveled_skiplist:to_list(binary_to_term(SlotBin0)), - io:format(user, "Skiplist flattened in ~w microseconds~n", - [timer:now_diff(os:timestamp(), SW5)]), - - - io:format(user, "~ngb_tree comparison~n", []), SW2 = os:timestamp(), - Bloom = lists:foldr(fun leveled_tinybloom:tiny_enter/2, - leveled_tinybloom:tiny_empty(), - HashList), - Tree0 = gb_trees:from_orddict(KVList1), - TreeBin = term_to_binary(Tree0, [{compressed, ?COMPRESSION_LEVEL}]), - io:format(user, "Bloom and Tree created for all keys in ~w microseconds " ++ - "with size ~w~n", - [timer:now_diff(os:timestamp(), SW2), byte_size(TreeBin)]), - SW3 = os:timestamp(), - lists:foreach(fun({K, V}) -> - ?assertMatch({value, V}, - gb_trees:lookup(K, binary_to_term(TreeBin))) - end, - KVList1), - io:format(user, "Tree checked for all keys in ~w microseconds~n", - [timer:now_diff(os:timestamp(), SW3)]), - SW4 = os:timestamp(), - gb_trees:to_list(binary_to_term(TreeBin)), - io:format(user, "Tree flattened in ~w microseconds~n", - [timer:now_diff(os:timestamp(), SW4)]). - - + ?assertMatch(KVList1, all_from_slot(SlotBin0)), + io:format(user, "Slot flattened in ~w microseconds~n", + [timer:now_diff(os:timestamp(), SW2)]). + + -endif. \ No newline at end of file diff --git a/src/leveled_tinybloom.erl b/src/leveled_tinybloom.erl index 8868479..a0af8a5 100644 --- a/src/leveled_tinybloom.erl +++ b/src/leveled_tinybloom.erl @@ -73,35 +73,17 @@ check(Key, Bloom) -> check({hash, Hash}, Bloom). tiny_empty() -> - <<0:2048>>. + <<0:1024>>. tiny_enter({hash, no_lookup}, Bloom) -> Bloom; tiny_enter({hash, Hash}, Bloom) -> - {Q, Bit0, Bit1, Bit2} = split_hash_for_tinybloom(Hash), + {_Q, Bit0, Bit1, Bit2} = split_hash_for_tinybloom(Hash), AddFun = fun(Bit, Arr0) -> add_to_array(Bit, Arr0, 1024) end, - case Q of - N when N < 2 -> - <> = Bloom, - NewBin = lists:foldl(AddFun, Bin1, [Bit0, Bit1, Bit2]), - <>; - _N -> - <> = Bloom, - NewBin = lists:foldl(AddFun, Bin2, [Bit0, Bit1, Bit2]), - <> - end. + lists:foldl(AddFun, Bloom, [Bit0, Bit1, Bit2]). -tiny_check({hash, Hash}, FullBloom) -> - {Q, Bit0, Bit1, Bit2} = split_hash_for_tinybloom(Hash), - Bloom = - case Q of - N when N < 2 -> - <> = FullBloom, - Bin1; - _N -> - <<_Bin1:1024/bitstring, Bin2:1024/bitstring>> = FullBloom, - Bin2 - end, +tiny_check({hash, Hash}, Bloom) -> + {_Q, Bit0, Bit1, Bit2} = split_hash_for_tinybloom(Hash), case getbit(Bit0, Bloom, 1024) of <<0:1>> -> false;