From f41c788bffed419fe68da0f93fb4532c8df5a9f2 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 16:46:25 +0000 Subject: [PATCH] Minor quibbles Move legacy CDB code used only in unit tests into test area. Fix column width in pmem and comment out the unused case statement (in healthy tests) from the penciller test code --- src/leveled_cdb.erl | 102 ++++++++++++++++++++------------------ src/leveled_penciller.erl | 7 ++- src/leveled_pmem.erl | 13 +++-- 3 files changed, 64 insertions(+), 58 deletions(-) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index fc8af19..59b8d77 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -552,41 +552,6 @@ create(FileName,KeyValueList) -> {BasePos, HashTree} = write_key_value_pairs(Handle, KeyValueList), close_file(Handle, HashTree, BasePos). -%% -%% dump(FileName) -> List -%% Given a file name, this function returns a list -%% of {key,value} tuples from the CDB. -%% - -dump(FileName) -> - {ok, Handle} = file:open(FileName, [binary, raw, read]), - Fn = fun(Index, Acc) -> - {ok, _} = file:position(Handle, ?DWORD_SIZE * Index), - {_, Count} = read_next_2_integers(Handle), - Acc + Count - end, - NumberOfPairs = lists:foldl(Fn, 0, lists:seq(0,255)) bsr 1, - io:format("Count of keys in db is ~w~n", [NumberOfPairs]), - {ok, _} = file:position(Handle, {bof, 2048}), - Fn1 = fun(_I,Acc) -> - {KL,VL} = read_next_2_integers(Handle), - Key = read_next_term(Handle, KL), - case read_next_term(Handle, VL, crc) of - {false, _} -> - {ok, CurrLoc} = file:position(Handle, cur), - Return = {crc_wonky, get(Handle, Key)}; - {_, Value} -> - {ok, CurrLoc} = file:position(Handle, cur), - Return = - case get(Handle, Key) of - {Key,Value} -> {Key ,Value}; - X -> {wonky, X} - end - end, - {ok, _} = file:position(Handle, CurrLoc), - [Return | Acc] - end, - lists:foldr(Fn1, [], lists:seq(0, NumberOfPairs-1)). %% Open an active file - one for which it is assumed the hash tables have not %% yet been written @@ -1069,20 +1034,6 @@ calc_crc(Value) -> erlang:crc32(<>) end. -%% -%% to_dict(FileName) -%% Given a filename returns a dict containing -%% the key value pairs from the dict. -%% -%% @spec to_dict(filename()) -> dictionary() -%% where -%% filename() = string(), -%% dictionary() = dict() -%% -to_dict(FileName) -> - KeyValueList = dump(FileName), - dict:from_list(KeyValueList). - read_next_term(Handle, Length) -> case file:read(Handle, Length) of {ok, Bin} -> @@ -1378,6 +1329,59 @@ multi_key_value_to_record(KVList, BinaryMode, LastPosition) -> %%%%%%%%%%%%%%% -ifdef(TEST). +%% +%% dump(FileName) -> List +%% Given a file name, this function returns a list +%% of {key,value} tuples from the CDB. +%% + +dump(FileName) -> + {ok, Handle} = file:open(FileName, [binary, raw, read]), + Fn = fun(Index, Acc) -> + {ok, _} = file:position(Handle, ?DWORD_SIZE * Index), + {_, Count} = read_next_2_integers(Handle), + Acc + Count + end, + NumberOfPairs = lists:foldl(Fn, 0, lists:seq(0,255)) bsr 1, + io:format("Count of keys in db is ~w~n", [NumberOfPairs]), + {ok, _} = file:position(Handle, {bof, 2048}), + Fn1 = fun(_I,Acc) -> + {KL,VL} = read_next_2_integers(Handle), + Key = read_next_term(Handle, KL), + case read_next_term(Handle, VL, crc) of + {false, _} -> + {ok, CurrLoc} = file:position(Handle, cur), + Return = {crc_wonky, get(Handle, Key)}; + {_, Value} -> + {ok, CurrLoc} = file:position(Handle, cur), + Return = + case get(Handle, Key) of + {Key,Value} -> {Key ,Value}; + X -> {wonky, X} + end + end, + {ok, _} = file:position(Handle, CurrLoc), + [Return | Acc] + end, + lists:foldr(Fn1, [], lists:seq(0, NumberOfPairs-1)). + +%% +%% to_dict(FileName) +%% Given a filename returns a dict containing +%% the key value pairs from the dict. +%% +%% @spec to_dict(filename()) -> dictionary() +%% where +%% filename() = string(), +%% dictionary() = dict() +%% +to_dict(FileName) -> + KeyValueList = dump(FileName), + dict:from_list(KeyValueList). + + + + write_key_value_pairs_1_test() -> {ok,Handle} = file:open("../test/test.cdb",[write]), {_, HashTree} = write_key_value_pairs(Handle, diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index baeec07..f0ed89b 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -1260,10 +1260,9 @@ clean_subdir(DirPath) -> {ok, Files} = file:list_dir(DirPath), lists:foreach(fun(FN) -> File = filename:join(DirPath, FN), - case file:delete(File) of - ok -> io:format("Success deleting ~s~n", [File]); - _ -> io:format("Error deleting ~s~n", [File]) - end end, + ok = file:delete(File), + io:format("Success deleting ~s~n", [File]) + end, Files); false -> ok diff --git a/src/leveled_pmem.erl b/src/leveled_pmem.erl index 3784715..d12425b 100644 --- a/src/leveled_pmem.erl +++ b/src/leveled_pmem.erl @@ -7,7 +7,8 @@ %% whilst maintaining the capability to quickly snapshot the memory for clones %% of the Penciller. %% -%% ETS tables are not used due to complications with managing their mutability. +%% ETS tables are not used due to complications with managing their mutability, +%% as the database is snapshotted. %% %% An attempt was made to merge all trees into a single tree on push (in a %% spawned process), but this proved to have an expensive impact as the tree @@ -28,8 +29,8 @@ %% Total time for array_tree 209000 microseconds %% Total time for array_list 142000 microseconds %% Total time for array_filter 69000 microseconds -%% List of 2000 checked without array - success count of 90 in 36000 microseconds -%% List of 2000 checked with array - success count of 90 in 1000 microseconds +%% List of 2000 checked without array - success count of 90 in 36000 microsecs +%% List of 2000 checked with array - success count of 90 in 1000 microsecs %% %% The trade-off taken with the approach is that the size of the L0Cache is %% uncertain. The Size count is incremented if the hash is not already @@ -203,8 +204,10 @@ generate_randomkeys(Seqn, Count, Acc, BucketLow, BRange) -> compare_method_test() -> R = lists:foldl(fun(_X, {LedgerSQN, L0Size, L0Index, L0TreeList}) -> - LM1 = generate_randomkeys(LedgerSQN + 1, 2000, 1, 500), - add_to_index(L0Index, L0Size, LM1, LedgerSQN, L0TreeList) + LM1 = generate_randomkeys(LedgerSQN + 1, + 2000, 1, 500), + add_to_index(L0Index, L0Size, LM1, LedgerSQN, + L0TreeList) end, {0, 0, new_index(), []}, lists:seq(1, 16)),