umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Manifest ordering

Browse source 
Be more explicit about manifest ordering to stop keys being laoded in
incorrect order
This commit is contained in:
martinsumner 2016-09-19 15:56:35 +01:00
parent 7c28ffbd96
commit a1c970a66a
2 changed files with 9 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/leveled_bookie.erl
View file

@ -457,7 +457,7 @@ generate_multiple_objects(0, _KeyNumber, ObjL) ->
generate_multiple_objects(Count, KeyNumber, ObjL) -> generate_multiple_objects(Count, KeyNumber, ObjL) ->
Obj = {"Bucket", Obj = {"Bucket",
"Key" ++ integer_to_list(KeyNumber), "Key" ++ integer_to_list(KeyNumber),
crypto:rand_bytes(128), crypto:rand_bytes(1024),
[], [],
[{"MDK", "MDV" ++ integer_to_list(KeyNumber)}, [{"MDK", "MDV" ++ integer_to_list(KeyNumber)},
{"MDK2", "MDV" ++ integer_to_list(KeyNumber)}]}, {"MDK2", "MDV" ++ integer_to_list(KeyNumber)}]},
@ -541,7 +541,6 @@ multi_key_test() ->
{ok, F2D} = book_riakget(Bookie2, B2, K2), {ok, F2D} = book_riakget(Bookie2, B2, K2),
?assertMatch(F2D, Obj2), ?assertMatch(F2D, Obj2),
ok = book_close(Bookie2), ok = book_close(Bookie2),
reset_filestructure(), reset_filestructure().
?assertMatch(true, false).
-endif. -endif.

19
src/leveled_inker.erl
View file

@ -189,7 +189,7 @@ init([InkerOpts]) ->
fun simple_manifest_reader/2, fun simple_manifest_reader/2,
RootPath, RootPath,
CDBopts), CDBopts),
{ok, #state{manifest = Manifest, {ok, #state{manifest = lists:reverse(lists:keysort(1, Manifest)),
manifest_sqn = ManifestSQN, manifest_sqn = ManifestSQN,
journal_sqn = JournalSQN, journal_sqn = JournalSQN,
active_journaldb = ActiveJournal, active_journaldb = ActiveJournal,
@ -224,7 +224,7 @@ handle_call({fetch, Key, SQN}, _From, State) ->
{reply, {ok, Value}, State}; {reply, {ok, Value}, State};
Other -> Other ->
io:format("Unexpected failure to fetch value for" ++ io:format("Unexpected failure to fetch value for" ++
"Key=~s SQN=~w with reason ~w", [Key, SQN, Other]), "Key=~w SQN=~w with reason ~w", [Key, SQN, Other]),
{reply, not_present, State} {reply, not_present, State}
end; end;
handle_call({get, Key, SQN}, _From, State) -> handle_call({get, Key, SQN}, _From, State) ->
@ -234,9 +234,10 @@ handle_call({get, Key, SQN}, _From, State) ->
State#state.active_journaldb, State#state.active_journaldb,
State#state.active_journaldb_sqn), State}; State#state.active_journaldb_sqn), State};
handle_call({load_pcl, StartSQN, FilterFun, Penciller}, _From, State) -> handle_call({load_pcl, StartSQN, FilterFun, Penciller}, _From, State) ->
Manifest = State#state.manifest ++ [{State#state.active_journaldb_sqn, Manifest = lists:reverse(State#state.manifest)
dummy, ++ [{State#state.active_journaldb_sqn,
State#state.active_journaldb}], dummy,
State#state.active_journaldb}],
Reply = load_from_sequence(StartSQN, FilterFun, Penciller, Manifest), Reply = load_from_sequence(StartSQN, FilterFun, Penciller, Manifest),
{reply, Reply, State}; {reply, Reply, State};
handle_call(snapshot, _From , State) -> handle_call(snapshot, _From , State) ->
@ -311,21 +312,15 @@ roll_active_file(OldActiveJournal, Manifest, ManifestSQN, RootPath) ->
SW = os:timestamp(), SW = os:timestamp(),
io:format("Rolling old journal ~w~n", [OldActiveJournal]), io:format("Rolling old journal ~w~n", [OldActiveJournal]),
{ok, NewFilename} = leveled_cdb:cdb_complete(OldActiveJournal), {ok, NewFilename} = leveled_cdb:cdb_complete(OldActiveJournal),
io:format("Rolling old journal S1 completed in ~w microseconds~n",
[timer:now_diff(os:timestamp(),SW)]),
{ok, PidR} = leveled_cdb:cdb_open_reader(NewFilename), {ok, PidR} = leveled_cdb:cdb_open_reader(NewFilename),
io:format("Rolling old journal S2 completed in ~w microseconds~n",
[timer:now_diff(os:timestamp(),SW)]),
JournalRegex2 = "nursery_(?<SQN>[0-9]+)\\." ++ ?JOURNAL_FILEX, JournalRegex2 = "nursery_(?<SQN>[0-9]+)\\." ++ ?JOURNAL_FILEX,
[JournalSQN] = sequencenumbers_fromfilenames([NewFilename], [JournalSQN] = sequencenumbers_fromfilenames([NewFilename],
JournalRegex2, JournalRegex2,
'SQN'), 'SQN'),
NewManifest = add_to_manifest(Manifest, {JournalSQN, NewFilename, PidR}), NewManifest = add_to_manifest(Manifest, {JournalSQN, NewFilename, PidR}),
io:format("Rolling old journal S3 completed in ~w microseconds~n",
[timer:now_diff(os:timestamp(),SW)]),
NewManifestSQN = ManifestSQN + 1, NewManifestSQN = ManifestSQN + 1,
ok = simple_manifest_writer(NewManifest, NewManifestSQN, RootPath), ok = simple_manifest_writer(NewManifest, NewManifestSQN, RootPath),
io:format("Rolling old journal S4 completed in ~w microseconds~n", io:format("Rolling old journal completed in ~w microseconds~n",
[timer:now_diff(os:timestamp(),SW)]), [timer:now_diff(os:timestamp(),SW)]),
{NewManifest, NewManifestSQN}. {NewManifest, NewManifestSQN}.