Further Journal compaction tests
Improved unit testing
This commit is contained in:
martinsumner
parent
50b50ba486
commit
15f57a0b4a
3 changed files with 125 additions and 27 deletions
|
|
@ -336,8 +336,10 @@ handle_call(cdb_complete, _From, State) ->
|
|||
%% Rename file
|
||||
NewName = filename:rootname(State#state.filename, ".pnd")
|
||||
++ ".cdb",
|
||||
io:format("Renaming file from ~s to ~s~n",
|
||||
[State#state.filename, NewName]),
|
||||
io:format("Renaming file from ~s to ~s " ++
|
||||
"for which existence is ~w~n",
|
||||
[State#state.filename, NewName,
|
||||
filelib:is_file(NewName)]),
|
||||
ok = file:rename(State#state.filename, NewName),
|
||||
{stop, normal, {ok, NewName}, State};
|
||||
false ->
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
handle_info/2,
|
||||
terminate/2,
|
||||
clerk_new/1,
|
||||
clerk_compact/4,
|
||||
clerk_compact/6,
|
||||
clerk_remove/2,
|
||||
clerk_stop/1,
|
||||
code_change/3]).
|
||||
|
|
@ -51,8 +51,14 @@ clerk_remove(Pid, Removals) ->
|
|||
gen_server:cast(Pid, {remove, Removals}),
|
||||
ok.
|
||||
|
||||
clerk_compact(Pid, Penciller, Inker, Timeout) ->
|
||||
clerk_compact(Pid, Penciller, Inker, Timeout).
|
||||
clerk_compact(Pid, Checker, InitiateFun, FilterFun, Inker, Timeout) ->
|
||||
gen_server:cast(Pid,
|
||||
{compact,
|
||||
Checker,
|
||||
InitiateFun,
|
||||
FilterFun,
|
||||
Inker,
|
||||
Timeout}).
|
||||
|
||||
clerk_stop(Pid) ->
|
||||
gen_server:cast(Pid, stop).
|
||||
|
|
@ -76,25 +82,18 @@ init([IClerkOpts]) ->
|
|||
handle_call(_Msg, _From, State) ->
|
||||
{reply, not_supprted, State}.
|
||||
|
||||
handle_cast({compact, Penciller, Inker, _Timeout}, State) ->
|
||||
handle_cast({compact, Checker, InitiateFun, FilterFun, Inker, _Timeout},
|
||||
State) ->
|
||||
% Need to fetch manifest at start rather than have it be passed in
|
||||
% Don't want to process a queued call waiting on an old manifest
|
||||
Manifest = leveled_inker:ink_getmanifest(Inker),
|
||||
MaxRunLength = State#state.max_run_length,
|
||||
PclOpts = #penciller_options{start_snapshot = true,
|
||||
source_penciller = Penciller,
|
||||
requestor = self()},
|
||||
FilterFun = fun leveled_penciller:pcl_checksequencenumber/3,
|
||||
FilterServer = leveled_penciller:pcl_start(PclOpts),
|
||||
ok = leveled_penciller:pcl_loadsnapshot(FilterServer, []),
|
||||
|
||||
FilterServer = InitiateFun(Checker),
|
||||
CDBopts = State#state.cdb_options,
|
||||
FP = CDBopts#cdb_options.file_path,
|
||||
ok = filelib:ensure_dir(FP),
|
||||
|
||||
Candidates = scan_all_files(Manifest,
|
||||
FilterFun,
|
||||
FilterServer),
|
||||
Candidates = scan_all_files(Manifest, FilterFun, FilterServer),
|
||||
BestRun = assess_candidates(Candidates, MaxRunLength),
|
||||
case score_run(BestRun, MaxRunLength) of
|
||||
Score when Score > 0 ->
|
||||
|
|
@ -278,6 +277,10 @@ compact_files(BestRun, CDBopts, FilterFun, FilterServer) ->
|
|||
[],
|
||||
true).
|
||||
|
||||
|
||||
compact_files([], _CDBopts, null, _FilterFun, _FilterServer,
|
||||
ManSlice0, PromptDelete0) ->
|
||||
{ManSlice0, PromptDelete0};
|
||||
compact_files([], _CDBopts, ActiveJournal0, _FilterFun, _FilterServer,
|
||||
ManSlice0, PromptDelete0) ->
|
||||
ManSlice1 = ManSlice0 ++ generate_manifest_entry(ActiveJournal0),
|
||||
|
|
@ -538,7 +541,8 @@ compact_single_file_test() ->
|
|||
?assertMatch(missing, leveled_cdb:cdb_get(PidR, {7, "Key1"})),
|
||||
?assertMatch(missing, leveled_cdb:cdb_get(PidR, {1, "Key1"})),
|
||||
{_RK1, RV1} = leveled_cdb:cdb_get(PidR, {2, "Key2"}),
|
||||
?assertMatch("Value2", binary_to_term(RV1)).
|
||||
?assertMatch("Value2", binary_to_term(RV1)),
|
||||
ok = leveled_cdb:cdb_destroy(CDB).
|
||||
|
||||
|
||||
-endif.
|
||||
|
|
@ -164,7 +164,26 @@ ink_loadpcl(Pid, MinSQN, FilterFun, Penciller) ->
|
|||
gen_server:call(Pid, {load_pcl, MinSQN, FilterFun, Penciller}, infinity).
|
||||
|
||||
ink_compactjournal(Pid, Penciller, Timeout) ->
|
||||
gen_server:call(Pid, {compact_journal, Penciller, Timeout}, infinty).
|
||||
CheckerInitiateFun = fun initiate_penciller_snapshot/1,
|
||||
CheckerFilterFun = fun leveled_penciller:pcl_checksequencenumber/3,
|
||||
gen_server:call(Pid,
|
||||
{compact,
|
||||
Penciller,
|
||||
CheckerInitiateFun,
|
||||
CheckerFilterFun,
|
||||
Timeout},
|
||||
infiniy).
|
||||
|
||||
%% Allows the Checker to be overriden in test, use something other than a
|
||||
%% penciller
|
||||
ink_compactjournal(Pid, Checker, InitiateFun, FilterFun, Timeout) ->
|
||||
gen_server:call(Pid,
|
||||
{compact,
|
||||
Checker,
|
||||
InitiateFun,
|
||||
FilterFun,
|
||||
Timeout},
|
||||
infinity).
|
||||
|
||||
ink_getmanifest(Pid) ->
|
||||
gen_server:call(Pid, get_manifest, infinity).
|
||||
|
|
@ -259,21 +278,23 @@ handle_call({update_manifest,
|
|||
Check = lists:member(ManEntry, DeletedFiles),
|
||||
if
|
||||
Check == false ->
|
||||
lists:append(AccMan, ManEntry)
|
||||
AccMan ++ [ManEntry];
|
||||
true ->
|
||||
AccMan
|
||||
end
|
||||
end,
|
||||
[],
|
||||
State#state.manifest),
|
||||
Man1 = lists:foldl(fun(ManEntry, AccMan) ->
|
||||
add_to_manifest(AccMan, ManEntry) end,
|
||||
Man0,
|
||||
ManifestSnippet),
|
||||
add_to_manifest(AccMan, ManEntry) end,
|
||||
Man0,
|
||||
ManifestSnippet),
|
||||
NewManifestSQN = State#state.manifest_sqn + 1,
|
||||
ok = simple_manifest_writer(Man1, NewManifestSQN, State#state.root_path),
|
||||
PendingRemovals = case PromptDeletion of
|
||||
true ->
|
||||
State#state.pending_removals ++
|
||||
{NewManifestSQN, DeletedFiles};
|
||||
[{NewManifestSQN, DeletedFiles}];
|
||||
_ ->
|
||||
State#state.pending_removals
|
||||
end,
|
||||
|
|
@ -283,14 +304,24 @@ handle_call({update_manifest,
|
|||
handle_call(print_manifest, _From, State) ->
|
||||
manifest_printer(State#state.manifest),
|
||||
{reply, ok, State};
|
||||
handle_call({compact_journal, Penciller, Timeout}, _From, State) ->
|
||||
handle_call({compact,
|
||||
Checker,
|
||||
InitiateFun,
|
||||
FilterFun,
|
||||
Timeout},
|
||||
_From, State) ->
|
||||
leveled_iclerk:clerk_compact(State#state.clerk,
|
||||
Checker,
|
||||
InitiateFun,
|
||||
FilterFun,
|
||||
self(),
|
||||
Penciller,
|
||||
Timeout),
|
||||
{reply, ok, State};
|
||||
handle_call(close, _From, State) ->
|
||||
{stop, normal, ok, State}.
|
||||
{stop, normal, ok, State};
|
||||
handle_call(Msg, _From, State) ->
|
||||
io:format("Unexpected message ~w~n", [Msg]),
|
||||
{reply, error, State}.
|
||||
|
||||
handle_cast(_Msg, State) ->
|
||||
{noreply, State}.
|
||||
|
|
@ -712,6 +743,15 @@ manifest_printer(Manifest) ->
|
|||
[SQN, FN]) end,
|
||||
Manifest).
|
||||
|
||||
|
||||
initiate_penciller_snapshot(Penciller) ->
|
||||
PclOpts = #penciller_options{start_snapshot = true,
|
||||
source_penciller = Penciller,
|
||||
requestor = self()},
|
||||
FilterServer = leveled_penciller:pcl_start(PclOpts),
|
||||
ok = leveled_penciller:pcl_loadsnapshot(FilterServer, []),
|
||||
FilterServer.
|
||||
|
||||
%%%============================================================================
|
||||
%%% Test
|
||||
%%%============================================================================
|
||||
|
|
@ -720,6 +760,7 @@ manifest_printer(Manifest) ->
|
|||
|
||||
build_dummy_journal() ->
|
||||
RootPath = "../test/journal",
|
||||
clean_testdir(RootPath),
|
||||
JournalFP = filepath(RootPath, journal_dir),
|
||||
ManifestFP = filepath(RootPath, manifest_dir),
|
||||
ok = filelib:ensure_dir(RootPath),
|
||||
|
|
@ -753,7 +794,13 @@ clean_testdir(RootPath) ->
|
|||
|
||||
clean_subdir(DirPath) ->
|
||||
{ok, Files} = file:list_dir(DirPath),
|
||||
lists:foreach(fun(FN) -> file:delete(filename:join(DirPath, FN)) end,
|
||||
lists:foreach(fun(FN) ->
|
||||
File = filename:join(DirPath, FN),
|
||||
case file:delete(File) of
|
||||
ok -> io:format("Success deleting ~s~n", [File]);
|
||||
_ -> io:format("Error deleting ~s~n", [File])
|
||||
end
|
||||
end,
|
||||
Files).
|
||||
|
||||
simple_buildmanifest_test() ->
|
||||
|
|
@ -872,8 +919,53 @@ rollafile_simplejournal_test() ->
|
|||
?assertMatch(R1, {{5, "KeyAA"}, {"TestValueAA", []}}),
|
||||
R2 = ink_get(Ink1, "KeyBB", 54),
|
||||
?assertMatch(R2, {{54, "KeyBB"}, {"TestValueBB", []}}),
|
||||
Man = ink_getmanifest(Ink1),
|
||||
FakeMan = [{3, "test", dummy}, {1, "other_test", dummy}],
|
||||
ok = ink_updatemanifest(Ink1, FakeMan, true, Man),
|
||||
?assertMatch(FakeMan, ink_getmanifest(Ink1)),
|
||||
ok = ink_updatemanifest(Ink1, Man, true, FakeMan),
|
||||
?assertMatch({{5, "KeyAA"}, {"TestValueAA", []}},
|
||||
ink_get(Ink1, "KeyAA", 5)),
|
||||
?assertMatch({{54, "KeyBB"}, {"TestValueBB", []}},
|
||||
ink_get(Ink1, "KeyBB", 54)),
|
||||
ink_close(Ink1),
|
||||
clean_testdir(RootPath).
|
||||
|
||||
compact_journal_test() ->
|
||||
RootPath = "../test/journal",
|
||||
build_dummy_journal(),
|
||||
CDBopts = #cdb_options{max_size=300000},
|
||||
{ok, Ink1} = ink_start(#inker_options{root_path=RootPath,
|
||||
cdb_options=CDBopts}),
|
||||
FunnyLoop = lists:seq(1, 48),
|
||||
{ok, NewSQN1, _ObjSize} = ink_put(Ink1, "KeyAA", "TestValueAA", []),
|
||||
?assertMatch(NewSQN1, 5),
|
||||
ok = ink_print_manifest(Ink1),
|
||||
R0 = ink_get(Ink1, "KeyAA", 5),
|
||||
?assertMatch(R0, {{5, "KeyAA"}, {"TestValueAA", []}}),
|
||||
Checker = lists:map(fun(X) ->
|
||||
PK = "KeyZ" ++ integer_to_list(X),
|
||||
{ok, SQN, _} = ink_put(Ink1,
|
||||
PK,
|
||||
crypto:rand_bytes(10000),
|
||||
[]),
|
||||
{SQN, PK}
|
||||
end,
|
||||
FunnyLoop),
|
||||
{ok, NewSQN2, _ObjSize} = ink_put(Ink1, "KeyBB", "TestValueBB", []),
|
||||
?assertMatch(NewSQN2, 54),
|
||||
ActualManifest = ink_getmanifest(Ink1),
|
||||
?assertMatch(2, length(ActualManifest)),
|
||||
ok = ink_compactjournal(Ink1,
|
||||
Checker,
|
||||
fun(X) -> X end,
|
||||
fun(L, K, SQN) -> lists:member({SQN, K}, L) end,
|
||||
5000),
|
||||
timer:sleep(1000),
|
||||
CompactedManifest = ink_getmanifest(Ink1),
|
||||
?assertMatch(1, length(CompactedManifest)),
|
||||
ink_updatemanifest(Ink1, ActualManifest, true, CompactedManifest),
|
||||
ink_close(Ink1),
|
||||
clean_testdir(RootPath).
|
||||
|
||||
-endif.
|
||||
Loading…
Add table
Add a link
Reference in a new issue