umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Deferred Deletion of Journals

Browse source 
This allows for deleted journals to be retained for a period (the
waste_retnetion_period).  The idea being that a backup strategy can
ensure that all journals are backed up, even ones created and removed
from within a backup period - so that any restore pont is possible.

This is also a pre-cursor to removing some of the PromptDelete
complexity from the Inker Clerk - all compactions can prompt deletion as
deletion is now deferred.
This commit is contained in:
martinsumner 2016-11-14 11:17:14 +00:00
parent dbb840d75e
commit 44738f7c75
8 changed files with 147 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

35
test/end_to_end/basic_SUITE.erl
View file

@ -144,21 +144,9 @@ journal_compaction(_Config) ->
%% Now replace all the other objects
ObjList2 = testutil:generate_objects(40000, 10002),
testutil:riakload(Bookie1, ObjList2),
ok = leveled_bookie:book_compactjournal(Bookie1, 30000),
F = fun leveled_bookie:book_islastcompactionpending/1,
lists:foldl(fun(X, Pending) ->
case Pending of
false ->
false;
true ->
io:format("Loop ~w waiting for journal "
++ "compaction to complete~n", [X]),
timer:sleep(20000),
F(Bookie1)
end end,
true,
lists:seq(1, 15)),
ok = leveled_bookie:book_compactjournal(Bookie1, 30000),
testutil:wait_for_compaction(Bookie1),
ChkList3 = lists:sublist(lists:sort(ObjList2), 500),
testutil:check_forlist(Bookie1, ChkList3),
@ -168,6 +156,25 @@ journal_compaction(_Config) ->
testutil:check_forobject(Bookie2, TestObject),
testutil:check_forlist(Bookie2, ChkList3),
ok = leveled_bookie:book_close(Bookie2),
WasteFP = RootPath ++ "/journal/journal_files/waste",
{ok, ClearedJournals} = file:list_dir(WasteFP),
io:format("~w ClearedJournals found~n", [length(ClearedJournals)]),
true = length(ClearedJournals) > 0,
StartOpts2 = [{root_path, RootPath},
{max_journalsize, 10000000},
{max_run_length, 1},
{waste_retention_period, 1}],
{ok, Bookie3} = leveled_bookie:book_start(StartOpts2),
ok = leveled_bookie:book_compactjournal(Bookie3, 30000),
testutil:wait_for_compaction(Bookie3),
ok = leveled_bookie:book_close(Bookie3),
{ok, ClearedJournalsPC} = file:list_dir(WasteFP),
io:format("~w ClearedJournals found~n", [length(ClearedJournalsPC)]),
true = length(ClearedJournalsPC) == 0,
testutil:reset_filestructure(10000).