From 15f57a0b4aa1cd5228426e8c44e2b089228587a4 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 28 Sep 2016 18:26:52 +0100 Subject: [PATCH] Further Journal compaction tests Improved unit testing --- src/leveled_cdb.erl | 6 ++- src/leveled_iclerk.erl | 34 +++++++------ src/leveled_inker.erl | 112 +++++++++++++++++++++++++++++++++++++---- 3 files changed, 125 insertions(+), 27 deletions(-) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index b1eb11a..40bf5ef 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -336,8 +336,10 @@ handle_call(cdb_complete, _From, State) -> %% Rename file NewName = filename:rootname(State#state.filename, ".pnd") ++ ".cdb", - io:format("Renaming file from ~s to ~s~n", - [State#state.filename, NewName]), + io:format("Renaming file from ~s to ~s " ++ + "for which existence is ~w~n", + [State#state.filename, NewName, + filelib:is_file(NewName)]), ok = file:rename(State#state.filename, NewName), {stop, normal, {ok, NewName}, State}; false -> diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index f6d85d7..49a4f43 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -12,7 +12,7 @@ handle_info/2, terminate/2, clerk_new/1, - clerk_compact/4, + clerk_compact/6, clerk_remove/2, clerk_stop/1, code_change/3]). @@ -51,8 +51,14 @@ clerk_remove(Pid, Removals) -> gen_server:cast(Pid, {remove, Removals}), ok. -clerk_compact(Pid, Penciller, Inker, Timeout) -> - clerk_compact(Pid, Penciller, Inker, Timeout). +clerk_compact(Pid, Checker, InitiateFun, FilterFun, Inker, Timeout) -> + gen_server:cast(Pid, + {compact, + Checker, + InitiateFun, + FilterFun, + Inker, + Timeout}). clerk_stop(Pid) -> gen_server:cast(Pid, stop). @@ -76,25 +82,18 @@ init([IClerkOpts]) -> handle_call(_Msg, _From, State) -> {reply, not_supprted, State}. -handle_cast({compact, Penciller, Inker, _Timeout}, State) -> +handle_cast({compact, Checker, InitiateFun, FilterFun, Inker, _Timeout}, + State) -> % Need to fetch manifest at start rather than have it be passed in % Don't want to process a queued call waiting on an old manifest Manifest = leveled_inker:ink_getmanifest(Inker), MaxRunLength = State#state.max_run_length, - PclOpts = #penciller_options{start_snapshot = true, - source_penciller = Penciller, - requestor = self()}, - FilterFun = fun leveled_penciller:pcl_checksequencenumber/3, - FilterServer = leveled_penciller:pcl_start(PclOpts), - ok = leveled_penciller:pcl_loadsnapshot(FilterServer, []), - + FilterServer = InitiateFun(Checker), CDBopts = State#state.cdb_options, FP = CDBopts#cdb_options.file_path, ok = filelib:ensure_dir(FP), - Candidates = scan_all_files(Manifest, - FilterFun, - FilterServer), + Candidates = scan_all_files(Manifest, FilterFun, FilterServer), BestRun = assess_candidates(Candidates, MaxRunLength), case score_run(BestRun, MaxRunLength) of Score when Score > 0 -> @@ -278,6 +277,10 @@ compact_files(BestRun, CDBopts, FilterFun, FilterServer) -> [], true). + +compact_files([], _CDBopts, null, _FilterFun, _FilterServer, + ManSlice0, PromptDelete0) -> + {ManSlice0, PromptDelete0}; compact_files([], _CDBopts, ActiveJournal0, _FilterFun, _FilterServer, ManSlice0, PromptDelete0) -> ManSlice1 = ManSlice0 ++ generate_manifest_entry(ActiveJournal0), @@ -538,7 +541,8 @@ compact_single_file_test() -> ?assertMatch(missing, leveled_cdb:cdb_get(PidR, {7, "Key1"})), ?assertMatch(missing, leveled_cdb:cdb_get(PidR, {1, "Key1"})), {_RK1, RV1} = leveled_cdb:cdb_get(PidR, {2, "Key2"}), - ?assertMatch("Value2", binary_to_term(RV1)). + ?assertMatch("Value2", binary_to_term(RV1)), + ok = leveled_cdb:cdb_destroy(CDB). -endif. \ No newline at end of file diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 8142799..56c1664 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -164,7 +164,26 @@ ink_loadpcl(Pid, MinSQN, FilterFun, Penciller) -> gen_server:call(Pid, {load_pcl, MinSQN, FilterFun, Penciller}, infinity). ink_compactjournal(Pid, Penciller, Timeout) -> - gen_server:call(Pid, {compact_journal, Penciller, Timeout}, infinty). + CheckerInitiateFun = fun initiate_penciller_snapshot/1, + CheckerFilterFun = fun leveled_penciller:pcl_checksequencenumber/3, + gen_server:call(Pid, + {compact, + Penciller, + CheckerInitiateFun, + CheckerFilterFun, + Timeout}, + infiniy). + +%% Allows the Checker to be overriden in test, use something other than a +%% penciller +ink_compactjournal(Pid, Checker, InitiateFun, FilterFun, Timeout) -> + gen_server:call(Pid, + {compact, + Checker, + InitiateFun, + FilterFun, + Timeout}, + infinity). ink_getmanifest(Pid) -> gen_server:call(Pid, get_manifest, infinity). @@ -259,21 +278,23 @@ handle_call({update_manifest, Check = lists:member(ManEntry, DeletedFiles), if Check == false -> - lists:append(AccMan, ManEntry) + AccMan ++ [ManEntry]; + true -> + AccMan end end, [], State#state.manifest), Man1 = lists:foldl(fun(ManEntry, AccMan) -> - add_to_manifest(AccMan, ManEntry) end, - Man0, - ManifestSnippet), + add_to_manifest(AccMan, ManEntry) end, + Man0, + ManifestSnippet), NewManifestSQN = State#state.manifest_sqn + 1, ok = simple_manifest_writer(Man1, NewManifestSQN, State#state.root_path), PendingRemovals = case PromptDeletion of true -> State#state.pending_removals ++ - {NewManifestSQN, DeletedFiles}; + [{NewManifestSQN, DeletedFiles}]; _ -> State#state.pending_removals end, @@ -283,14 +304,24 @@ handle_call({update_manifest, handle_call(print_manifest, _From, State) -> manifest_printer(State#state.manifest), {reply, ok, State}; -handle_call({compact_journal, Penciller, Timeout}, _From, State) -> +handle_call({compact, + Checker, + InitiateFun, + FilterFun, + Timeout}, + _From, State) -> leveled_iclerk:clerk_compact(State#state.clerk, + Checker, + InitiateFun, + FilterFun, self(), - Penciller, Timeout), {reply, ok, State}; handle_call(close, _From, State) -> - {stop, normal, ok, State}. + {stop, normal, ok, State}; +handle_call(Msg, _From, State) -> + io:format("Unexpected message ~w~n", [Msg]), + {reply, error, State}. handle_cast(_Msg, State) -> {noreply, State}. @@ -712,6 +743,15 @@ manifest_printer(Manifest) -> [SQN, FN]) end, Manifest). + +initiate_penciller_snapshot(Penciller) -> + PclOpts = #penciller_options{start_snapshot = true, + source_penciller = Penciller, + requestor = self()}, + FilterServer = leveled_penciller:pcl_start(PclOpts), + ok = leveled_penciller:pcl_loadsnapshot(FilterServer, []), + FilterServer. + %%%============================================================================ %%% Test %%%============================================================================ @@ -720,6 +760,7 @@ manifest_printer(Manifest) -> build_dummy_journal() -> RootPath = "../test/journal", + clean_testdir(RootPath), JournalFP = filepath(RootPath, journal_dir), ManifestFP = filepath(RootPath, manifest_dir), ok = filelib:ensure_dir(RootPath), @@ -753,7 +794,13 @@ clean_testdir(RootPath) -> clean_subdir(DirPath) -> {ok, Files} = file:list_dir(DirPath), - lists:foreach(fun(FN) -> file:delete(filename:join(DirPath, FN)) end, + lists:foreach(fun(FN) -> + File = filename:join(DirPath, FN), + case file:delete(File) of + ok -> io:format("Success deleting ~s~n", [File]); + _ -> io:format("Error deleting ~s~n", [File]) + end + end, Files). simple_buildmanifest_test() -> @@ -872,8 +919,53 @@ rollafile_simplejournal_test() -> ?assertMatch(R1, {{5, "KeyAA"}, {"TestValueAA", []}}), R2 = ink_get(Ink1, "KeyBB", 54), ?assertMatch(R2, {{54, "KeyBB"}, {"TestValueBB", []}}), + Man = ink_getmanifest(Ink1), + FakeMan = [{3, "test", dummy}, {1, "other_test", dummy}], + ok = ink_updatemanifest(Ink1, FakeMan, true, Man), + ?assertMatch(FakeMan, ink_getmanifest(Ink1)), + ok = ink_updatemanifest(Ink1, Man, true, FakeMan), + ?assertMatch({{5, "KeyAA"}, {"TestValueAA", []}}, + ink_get(Ink1, "KeyAA", 5)), + ?assertMatch({{54, "KeyBB"}, {"TestValueBB", []}}, + ink_get(Ink1, "KeyBB", 54)), ink_close(Ink1), clean_testdir(RootPath). +compact_journal_test() -> + RootPath = "../test/journal", + build_dummy_journal(), + CDBopts = #cdb_options{max_size=300000}, + {ok, Ink1} = ink_start(#inker_options{root_path=RootPath, + cdb_options=CDBopts}), + FunnyLoop = lists:seq(1, 48), + {ok, NewSQN1, _ObjSize} = ink_put(Ink1, "KeyAA", "TestValueAA", []), + ?assertMatch(NewSQN1, 5), + ok = ink_print_manifest(Ink1), + R0 = ink_get(Ink1, "KeyAA", 5), + ?assertMatch(R0, {{5, "KeyAA"}, {"TestValueAA", []}}), + Checker = lists:map(fun(X) -> + PK = "KeyZ" ++ integer_to_list(X), + {ok, SQN, _} = ink_put(Ink1, + PK, + crypto:rand_bytes(10000), + []), + {SQN, PK} + end, + FunnyLoop), + {ok, NewSQN2, _ObjSize} = ink_put(Ink1, "KeyBB", "TestValueBB", []), + ?assertMatch(NewSQN2, 54), + ActualManifest = ink_getmanifest(Ink1), + ?assertMatch(2, length(ActualManifest)), + ok = ink_compactjournal(Ink1, + Checker, + fun(X) -> X end, + fun(L, K, SQN) -> lists:member({SQN, K}, L) end, + 5000), + timer:sleep(1000), + CompactedManifest = ink_getmanifest(Ink1), + ?assertMatch(1, length(CompactedManifest)), + ink_updatemanifest(Ink1, ActualManifest, true, CompactedManifest), + ink_close(Ink1), + clean_testdir(RootPath). -endif. \ No newline at end of file