umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Minor quibbles

Browse source 
Move legacy CDB code used only in unit tests into test area.  Fix column
width in  pmem and comment out the unused case statement (in healthy
tests) from the penciller test code
This commit is contained in:
martinsumner 2016-11-03 16:46:25 +00:00
parent 37e20ccdfe
commit f41c788bff
3 changed files with 64 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

102
src/leveled_cdb.erl
View file

@ -552,41 +552,6 @@ create(FileName,KeyValueList) ->
{BasePos, HashTree} = write_key_value_pairs(Handle, KeyValueList),
close_file(Handle, HashTree, BasePos).
%%
%% dump(FileName) -> List
%% Given a file name, this function returns a list
%% of {key,value} tuples from the CDB.
%%
dump(FileName) ->
{ok, Handle} = file:open(FileName, [binary, raw, read]),
Fn = fun(Index, Acc) ->
{ok, _} = file:position(Handle, ?DWORD_SIZE * Index),
{_, Count} = read_next_2_integers(Handle),
Acc + Count
end,
NumberOfPairs = lists:foldl(Fn, 0, lists:seq(0,255)) bsr 1,
io:format("Count of keys in db is ~w~n", [NumberOfPairs]),
{ok, _} = file:position(Handle, {bof, 2048}),
Fn1 = fun(_I,Acc) ->
{KL,VL} = read_next_2_integers(Handle),
Key = read_next_term(Handle, KL),
case read_next_term(Handle, VL, crc) of
{false, _} ->
{ok, CurrLoc} = file:position(Handle, cur),
Return = {crc_wonky, get(Handle, Key)};
{_, Value} ->
{ok, CurrLoc} = file:position(Handle, cur),
Return =
case get(Handle, Key) of
{Key,Value} -> {Key ,Value};
X -> {wonky, X}
end
end,
{ok, _} = file:position(Handle, CurrLoc),
[Return | Acc]
end,
lists:foldr(Fn1, [], lists:seq(0, NumberOfPairs-1)).
%% Open an active file - one for which it is assumed the hash tables have not
%% yet been written
@ -1069,20 +1034,6 @@ calc_crc(Value) ->
erlang:crc32(<<Value/bitstring,0:M>>)
end.
%%
%% to_dict(FileName)
%% Given a filename returns a dict containing
%% the key value pairs from the dict.
%%
%% @spec to_dict(filename()) -> dictionary()
%% where
%% filename() = string(),
%% dictionary() = dict()
%%
to_dict(FileName) ->
KeyValueList = dump(FileName),
dict:from_list(KeyValueList).
read_next_term(Handle, Length) ->
case file:read(Handle, Length) of
{ok, Bin} ->
@ -1378,6 +1329,59 @@ multi_key_value_to_record(KVList, BinaryMode, LastPosition) ->
%%%%%%%%%%%%%%%
-ifdef(TEST).
%%
%% dump(FileName) -> List
%% Given a file name, this function returns a list
%% of {key,value} tuples from the CDB.
%%
dump(FileName) ->
{ok, Handle} = file:open(FileName, [binary, raw, read]),
Fn = fun(Index, Acc) ->
{ok, _} = file:position(Handle, ?DWORD_SIZE * Index),
{_, Count} = read_next_2_integers(Handle),
Acc + Count
end,
NumberOfPairs = lists:foldl(Fn, 0, lists:seq(0,255)) bsr 1,
io:format("Count of keys in db is ~w~n", [NumberOfPairs]),
{ok, _} = file:position(Handle, {bof, 2048}),
Fn1 = fun(_I,Acc) ->
{KL,VL} = read_next_2_integers(Handle),
Key = read_next_term(Handle, KL),
case read_next_term(Handle, VL, crc) of
{false, _} ->
{ok, CurrLoc} = file:position(Handle, cur),
Return = {crc_wonky, get(Handle, Key)};
{_, Value} ->
{ok, CurrLoc} = file:position(Handle, cur),
Return =
case get(Handle, Key) of
{Key,Value} -> {Key ,Value};
X -> {wonky, X}
end
end,
{ok, _} = file:position(Handle, CurrLoc),
[Return | Acc]
end,
lists:foldr(Fn1, [], lists:seq(0, NumberOfPairs-1)).
%%
%% to_dict(FileName)
%% Given a filename returns a dict containing
%% the key value pairs from the dict.
%%
%% @spec to_dict(filename()) -> dictionary()
%% where
%% filename() = string(),
%% dictionary() = dict()
%%
to_dict(FileName) ->
KeyValueList = dump(FileName),
dict:from_list(KeyValueList).
write_key_value_pairs_1_test() ->
{ok,Handle} = file:open("../test/test.cdb",[write]),
{_, HashTree} = write_key_value_pairs(Handle,

7
src/leveled_penciller.erl
View file

@ -1260,10 +1260,9 @@ clean_subdir(DirPath) ->
{ok, Files} = file:list_dir(DirPath),
lists:foreach(fun(FN) ->
File = filename:join(DirPath, FN),
case file:delete(File) of
ok -> io:format("Success deleting ~s~n", [File]);
_ -> io:format("Error deleting ~s~n", [File])
end end,
ok = file:delete(File),
io:format("Success deleting ~s~n", [File])
end,
Files);
false ->
ok

13
src/leveled_pmem.erl
View file

@ -7,7 +7,8 @@
%% whilst maintaining the capability to quickly snapshot the memory for clones
%% of the Penciller.
%%
%% ETS tables are not used due to complications with managing their mutability.
%% ETS tables are not used due to complications with managing their mutability,
%% as the database is snapshotted.
%%
%% An attempt was made to merge all trees into a single tree on push (in a
%% spawned process), but this proved to have an expensive impact as the tree
@ -28,8 +29,8 @@
%% Total time for array_tree 209000 microseconds
%% Total time for array_list 142000 microseconds
%% Total time for array_filter 69000 microseconds
%% List of 2000 checked without array - success count of 90 in 36000 microseconds
%% List of 2000 checked with array - success count of 90 in 1000 microseconds
%% List of 2000 checked without array - success count of 90 in 36000 microsecs
%% List of 2000 checked with array - success count of 90 in 1000 microsecs
%%
%% The trade-off taken with the approach is that the size of the L0Cache is
%% uncertain. The Size count is incremented if the hash is not already
@ -203,8 +204,10 @@ generate_randomkeys(Seqn, Count, Acc, BucketLow, BRange) ->
compare_method_test() ->
R = lists:foldl(fun(_X, {LedgerSQN, L0Size, L0Index, L0TreeList}) ->
LM1 = generate_randomkeys(LedgerSQN + 1, 2000, 1, 500),
add_to_index(L0Index, L0Size, LM1, LedgerSQN, L0TreeList)
LM1 = generate_randomkeys(LedgerSQN + 1,
2000, 1, 500),
add_to_index(L0Index, L0Size, LM1, LedgerSQN,
L0TreeList)
end,
{0, 0, new_index(), []},
lists:seq(1, 16)),