umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Change fold_heads to do basic Journal presence check

Browse source 
This at least checks the file is present, and the Key exists in the
index of that file.  If the value is corrupt it will be removed by
compation, and then this will fail (unless the file is never compacted).

TODO: resolve issus of files which are corrupt - but never compacted
- a job for backup?
This commit is contained in:
martinsumner 2017-04-21 15:55:03 +01:00
parent 4d12dfe0ab
commit fbb4879d81
2 changed files with 81 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

30
src/leveled_bookie.erl
View file

@ -1101,6 +1101,15 @@ accumulate_objects(FoldObjectsFun, InkerClone, Tag, DeferredFetch) ->
Now = leveled_codec:integer_now(),
AccFun =
fun(LK, V, Acc) ->
% The function takes the Ledger Key and the value from the
% ledger (with the value being the object metadata)
%
% Need to check if this is an active object (so TTL has not
% expired).
% If this is a deferred_fetch (i.e. the fold is a fold_heads not
% a fold_objects), then a metadata object needs to be built to be
% returned - but a quick check that Key is present in the Journal
% is made first
case leveled_codec:is_active(LK, V, Now) of
true ->
{SQN, _St, _MH, MD} =
@ -1115,14 +1124,29 @@ accumulate_objects(FoldObjectsFun, InkerClone, Tag, DeferredFetch) ->
JK = {leveled_codec:to_ledgerkey(B, K, Tag), SQN},
case DeferredFetch of
true ->
InJournal =
leveled_inker:ink_keycheck(InkerClone,
LK,
SQN),
case InJournal of
probably ->
Size = leveled_codec:get_size(LK, V),
MDBin =
leveled_codec:build_metadata_object(LK, MD),
leveled_codec:build_metadata_object(LK,
MD),
Value = {proxy_object,
MDBin,
Size,
{fun fetch_value/2, InkerClone, JK}},
FoldObjectsFun(B, K, term_to_binary(Value), Acc);
{fun fetch_value/2,
InkerClone,
JK}},
FoldObjectsFun(B,
K,
term_to_binary(Value),
Acc);
missing ->
Acc
end;
false ->
R = fetch_value(InkerClone, JK),
case R of

47
test/end_to_end/recovery_SUITE.erl
View file

@ -4,6 +4,7 @@
-export([all/0]).
-export([retain_strategy/1,
recovr_strategy/1,
aae_missingjournal/1,
aae_bustedjournal/1,
journal_compaction_bustedjournal/1
]).
@ -11,6 +12,7 @@
all() -> [
retain_strategy,
recovr_strategy,
aae_missingjournal,
aae_bustedjournal,
journal_compaction_bustedjournal
].
@ -94,6 +96,51 @@ recovr_strategy(_Config) ->
testutil:reset_filestructure().
aae_missingjournal(_Config) ->
RootPath = testutil:reset_filestructure(),
StartOpts = [{root_path, RootPath},
{max_journalsize, 20000000},
{sync_strategy, testutil:sync_strategy()}],
{ok, Bookie1} = leveled_bookie:book_start(StartOpts),
{TestObject, TestSpec} = testutil:generate_testobject(),
ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
testutil:check_forobject(Bookie1, TestObject),
GenList = [2],
_CLs = testutil:load_objects(20000, GenList, Bookie1, TestObject,
fun testutil:generate_objects/2),
FoldHeadsFun =
fun(B, K, _V, Acc) -> [{B, K}|Acc] end,
{async, AllHeadF1} =
leveled_bookie:book_returnfolder(Bookie1,
{foldheads_allkeys,
?RIAK_TAG,
FoldHeadsFun}),
HeadL1 = length(AllHeadF1()),
io:format("Fold head returned ~w objects~n", [HeadL1]),
ok = leveled_bookie:book_close(Bookie1),
CDBFiles = testutil:find_journals(RootPath),
[HeadF|_Rest] = CDBFiles,
io:format("Selected Journal for removal of ~s~n", [HeadF]),
ok = file:delete(RootPath ++ "/journal/journal_files/" ++ HeadF),
{ok, Bookie2} = leveled_bookie:book_start(StartOpts),
% Check that fold heads picks up on the missing file
{async, AllHeadF2} =
leveled_bookie:book_returnfolder(Bookie2,
{foldheads_allkeys,
?RIAK_TAG,
FoldHeadsFun}),
HeadL2 = length(AllHeadF2()),
io:format("Fold head returned ~w objects~n", [HeadL2]),
true = HeadL2 < HeadL1,
true = HeadL2 > 0,
ok = leveled_bookie:book_close(Bookie2),
testutil:reset_filestructure().
aae_bustedjournal(_Config) ->
RootPath = testutil:reset_filestructure(),
StartOpts = [{root_path, RootPath},