L0 cache size counter improved

This is now accurate save for hash collisions.  It may now be a small
under-estimate, whereas previously it could be a large over-estimate.
This commit is contained in:
martinsumner 2016-11-03 16:22:51 +00:00
parent 4e46c9735d
commit 37e20ccdfe

View file

@ -30,7 +30,11 @@
%% Total time for array_filter 69000 microseconds %% Total time for array_filter 69000 microseconds
%% List of 2000 checked without array - success count of 90 in 36000 microseconds %% List of 2000 checked without array - success count of 90 in 36000 microseconds
%% List of 2000 checked with array - success count of 90 in 1000 microseconds %% List of 2000 checked with array - success count of 90 in 1000 microseconds
%%
%% The trade-off taken with the approach is that the size of the L0Cache is
%% uncertain. The Size count is incremented if the hash is not already
%% present, so the size may be lower than the actual size due to hash
%% collisions
-module(leveled_pmem). -module(leveled_pmem).
@ -56,19 +60,26 @@
add_to_index(L0Index, L0Size, LevelMinus1, LedgerSQN, TreeList) -> add_to_index(L0Index, L0Size, LevelMinus1, LedgerSQN, TreeList) ->
SW = os:timestamp(), SW = os:timestamp(),
SlotInTreeList = length(TreeList) + 1, SlotInTreeList = length(TreeList) + 1,
FoldFun = fun({K, V}, {AccMinSQN, AccMaxSQN, HashIndex}) -> FoldFun = fun({K, V}, {AccMinSQN, AccMaxSQN, AccCount, HashIndex}) ->
SQN = leveled_codec:strip_to_seqonly({K, V}), SQN = leveled_codec:strip_to_seqonly({K, V}),
{Hash, Slot} = hash_to_slot(K), {Hash, Slot} = hash_to_slot(K),
L = array:get(Slot, HashIndex), L = array:get(Slot, HashIndex),
Count0 = case lists:keymember(Hash, 1, L) of
true ->
AccCount;
false ->
AccCount + 1
end,
{min(SQN, AccMinSQN), {min(SQN, AccMinSQN),
max(SQN, AccMaxSQN), max(SQN, AccMaxSQN),
Count0,
array:set(Slot, [{Hash, SlotInTreeList}|L], HashIndex)} array:set(Slot, [{Hash, SlotInTreeList}|L], HashIndex)}
end, end,
LM1List = gb_trees:to_list(LevelMinus1), LM1List = gb_trees:to_list(LevelMinus1),
{MinSQN, MaxSQN, UpdL0Index} = lists:foldl(FoldFun, StartingT = {infinity, 0, L0Size, L0Index},
{infinity, 0, L0Index}, {MinSQN, MaxSQN, NewL0Size, UpdL0Index} = lists:foldl(FoldFun,
LM1List), StartingT,
NewL0Size = length(LM1List) + L0Size, LM1List),
leveled_log:log_timer("PM001", [NewL0Size], SW), leveled_log:log_timer("PM001", [NewL0Size], SW),
if if
MinSQN > LedgerSQN -> MinSQN > LedgerSQN ->
@ -198,8 +209,9 @@ compare_method_test() ->
{0, 0, new_index(), []}, {0, 0, new_index(), []},
lists:seq(1, 16)), lists:seq(1, 16)),
{SQN, _Size, Index, TreeList} = R, {SQN, Size, Index, TreeList} = R,
?assertMatch(32000, SQN), ?assertMatch(32000, SQN),
?assertMatch(true, Size =< 32000),
TestList = gb_trees:to_list(generate_randomkeys(1, 2000, 1, 800)), TestList = gb_trees:to_list(generate_randomkeys(1, 2000, 1, 800)),