Switch to using ets set as index of L0 cache

Hope is that this will cause less garbage collection, and also will be
slightly faster.

Note that snapshots don't now get an index - they get the special index
'snap'.  However, the SkipLists have bloom protection, and most
snapshots are iterators not fetchers.

src/leveled_penciller.erl

@@ -395,7 +395,7 @@ handle_call({register_snapshot, Snapshot}, _From, State) ->
     Rs = [{Snapshot, State#state.manifest_sqn}|State#state.registered_snapshots],
     {reply, {ok, State}, State#state{registered_snapshots = Rs}};
 handle_call({load_snapshot, BookieIncrTree}, _From, State) ->
-    L0D = leveled_pmem:add_to_index(State#state.levelzero_index,
+    L0D = leveled_pmem:add_to_index(snap,
                                         State#state.levelzero_size,
                                         BookieIncrTree,
                                         State#state.ledger_sqn,

src/leveled_pmem.erl

@@ -51,42 +51,55 @@
 
 -include_lib("eunit/include/eunit.hrl").
 
--define(SLOT_WIDTH, {4096, 12}).
-
 
 %%%============================================================================
 %%% API
 %%%============================================================================
 
+add_to_index(snap, L0Size, LevelMinus1, LedgerSQN, TreeList) ->
+    FoldFun = fun({K, V}, {AccMinSQN, AccMaxSQN, AccCount}) ->
+                    SQN = leveled_codec:strip_to_seqonly({K, V}),
+                    {min(SQN, AccMinSQN),
+                        max(SQN, AccMaxSQN),
+                        AccCount + 1}
+                    end,
+    LM1List = leveled_skiplist:to_list(LevelMinus1),
+    StartingT = {infinity, 0, L0Size},
+    {MinSQN, MaxSQN, NewL0Size} = lists:foldl(FoldFun, StartingT, LM1List),
+    if
+        MinSQN > LedgerSQN ->
+            {MaxSQN,
+                NewL0Size,
+                snap,
+                lists:append(TreeList, [LevelMinus1])}
+    end;
 add_to_index(L0Index, L0Size, LevelMinus1, LedgerSQN, TreeList) ->
     SW = os:timestamp(),
     SlotInTreeList = length(TreeList) + 1,
-    FoldFun = fun({K, V}, {AccMinSQN, AccMaxSQN, AccCount, HashIndex}) ->
+    FoldFun = fun({K, V}, {AccMinSQN, AccMaxSQN, AccCount}) ->
                     SQN = leveled_codec:strip_to_seqonly({K, V}),
-                    {Hash, Slot} = hash_to_slot(K),
+                    Hash = erlang:phash2(K),
-                    L = array:get(Slot, HashIndex),
+                    Count0 = case ets:lookup(L0Index, Hash) of
-                    Count0 = case lists:keymember(Hash, 1, L) of
+                                    [] ->
-                                    true ->
+                                        ets:insert(L0Index, {Hash, [SlotInTreeList]}),
-                                        AccCount;
+                                        AccCount + 1;
-                                    false ->
+                                    [{Hash, L}] ->
-                                        AccCount + 1
+                                        ets:insert(L0Index, {Hash, [SlotInTreeList|L]}),
+                                        AccCount
                                 end,
                     {min(SQN, AccMinSQN),
                         max(SQN, AccMaxSQN),
-                        Count0,
+                        Count0}
-                        array:set(Slot, [{Hash, SlotInTreeList}|L], HashIndex)}
                     end,
     LM1List = leveled_skiplist:to_list(LevelMinus1),
-    StartingT = {infinity, 0, L0Size, L0Index},
+    StartingT = {infinity, 0, L0Size},
-    {MinSQN, MaxSQN, NewL0Size, UpdL0Index} = lists:foldl(FoldFun,
+    {MinSQN, MaxSQN, NewL0Size} = lists:foldl(FoldFun, StartingT, LM1List),
-                                                            StartingT,
-                                                            LM1List),
     leveled_log:log_timer("PM001", [NewL0Size], SW),
     if
         MinSQN > LedgerSQN ->
             {MaxSQN,
                 NewL0Size,
-                UpdL0Index,
+                L0Index,
                 lists:append(TreeList, [LevelMinus1])}
     end.
     
@@ -106,38 +119,20 @@ to_list(Slots, FetchFun) ->
 
 
 new_index() ->
-    array:new(element(1, ?SLOT_WIDTH), [{default, []}, fixed]).
+    ets:new(index, [set, private]).
-
 
+check_levelzero(_Key, _L0Index, []) ->
+    {false, not_found};
+check_levelzero(Key, snap, TreeList) ->
+    check_slotlist(Key, lists:seq(1, length(TreeList)), TreeList);
 check_levelzero(Key, L0Index, TreeList) ->
-    {Hash, Slot} = hash_to_slot(Key),
+    Hash = erlang:phash2(Key),
-    CheckList = array:get(Slot, L0Index),
+    case ets:lookup(L0Index, Hash) of
-    SlotList = lists:foldl(fun({H0, S0}, SL) ->
+        [] ->
-                                case H0 of
+            {false, not_found};
-                                    Hash ->
+        [{Hash, SlotList}] ->
-                                        [S0|SL];
+            check_slotlist(Key, SlotList, TreeList)
-                                    _ ->
+    end.
-                                        SL
-                                end
-                                end,
-                            [],
-                            CheckList),
-    lists:foldl(fun(SlotToCheck, {Found, KV}) ->
-                        case Found of
-                            true ->
-                                {Found, KV};
-                            false ->
-                                CheckTree = lists:nth(SlotToCheck, TreeList),
-                                case leveled_skiplist:lookup(Key, CheckTree) of
-                                    none ->
-                                        {Found, KV};
-                                    {value, Value} ->
-                                        {true, {Key, Value}}
-                                end
-                        end
-                        end,
-                    {false, not_found},
-                    lists:reverse(lists:usort(SlotList))).
 
 
 merge_trees(StartKey, EndKey, SkipListList, LevelMinus1) ->
@@ -153,11 +148,25 @@ merge_trees(StartKey, EndKey, SkipListList, LevelMinus1) ->
 %%% Internal Functions
 %%%============================================================================
 
-
+check_slotlist(Key, CheckList, TreeList) ->
-hash_to_slot(Key) ->
+    SlotCheckFun =
-    H = erlang:phash2(Key),
+                fun(SlotToCheck, {Found, KV}) ->
-    {H bsr element(2, ?SLOT_WIDTH), H band (element(1, ?SLOT_WIDTH) - 1)}.
+                    case Found of
-
+                        true ->
+                            {Found, KV};
+                        false ->
+                            CheckTree = lists:nth(SlotToCheck, TreeList),
+                            case leveled_skiplist:lookup(Key, CheckTree) of
+                                none ->
+                                    {Found, KV};
+                                {value, Value} ->
+                                    {true, {Key, Value}}
+                            end
+                    end
+                    end,
+    lists:foldl(SlotCheckFun,
+                    {false, not_found},
+                    lists:reverse(lists:usort(CheckList))).
 
 %%%============================================================================
 %%% Test
@@ -231,8 +240,15 @@ compare_method_test() ->
                             end,
                         [],
                         TestList),
+    S2 = lists:foldl(fun({Key, _V}, Acc) ->
+                            R0 = check_levelzero(Key, snap, TreeList),
+                            [R0|Acc]
+                            end,
+                        [],
+                        TestList),
     
     ?assertMatch(S0, S1),
+    ?assertMatch(S0, S2),
     
     StartKey = {o, "Bucket0100", null, null},
     EndKey = {o, "Bucket0200", null, null},