umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Reverted back out the additional bloom check

Browse source 
This is desirable to add back in going forward, but wasn't implemented
in a safe or clear way.

The way the bloom was or was not on the LoopState was clumsy, and it got
persisted in multiple places without a CRC check.

Intention to implement back in wherby it is requested on-demand by the
Penciller, and then the SFT worker lifts it off disk and CRC checks it.
So it is never on the SFT LoopState.  Also it will be easier to control
the logic over which levels have the bloom in the Penciller.
This commit is contained in:
martinsumner 2016-12-11 21:01:10 +00:00
parent 4b48ed14c6
commit 86bdfdeaf0
4 changed files with 21 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

1
include/leveled.hrl
View file

@ -41,7 +41,6 @@
{start_key :: tuple(), {start_key :: tuple(),
end_key :: tuple(), end_key :: tuple(),
owner :: pid(), owner :: pid(),
bloom,
filename :: string()}). filename :: string()}).
-record(cdb_options, -record(cdb_options,

2
src/leveled_pclerk.erl
View file

@ -320,7 +320,6 @@ do_merge(KL1, KL2, {SrcLevel, IsB}, {Filepath, MSN}, FileCounter, OutList) ->
KL1, KL1,
KL2, KL2,
LevelR), LevelR),
{ok, Bloom} = leveled_sft:sft_getbloom(Pid),
case Reply of case Reply of
{{[], []}, null, _} -> {{[], []}, null, _} ->
leveled_log:log("PC013", [FileName]), leveled_log:log("PC013", [FileName]),
@ -332,7 +331,6 @@ do_merge(KL1, KL2, {SrcLevel, IsB}, {Filepath, MSN}, FileCounter, OutList) ->
[#manifest_entry{start_key=SmallestKey, [#manifest_entry{start_key=SmallestKey,
end_key=HighestKey, end_key=HighestKey,
owner=Pid, owner=Pid,
bloom=Bloom,
filename=FileName}]), filename=FileName}]),
leveled_log:log_timer("PC015", [], TS1), leveled_log:log_timer("PC015", [], TS1),
do_merge(KL1Rem, KL2Rem, do_merge(KL1Rem, KL2Rem,

31
src/leveled_penciller.erl
View file

@ -175,7 +175,7 @@
pcl_checksequencenumber/4, pcl_checksequencenumber/4,
pcl_workforclerk/1, pcl_workforclerk/1,
pcl_promptmanifestchange/2, pcl_promptmanifestchange/2,
pcl_confirml0complete/5, pcl_confirml0complete/4,
pcl_confirmdelete/2, pcl_confirmdelete/2,
pcl_close/1, pcl_close/1,
pcl_doom/1, pcl_doom/1,
@ -286,8 +286,8 @@ pcl_workforclerk(Pid) ->
pcl_promptmanifestchange(Pid, WI) -> pcl_promptmanifestchange(Pid, WI) ->
gen_server:cast(Pid, {manifest_change, WI}). gen_server:cast(Pid, {manifest_change, WI}).
pcl_confirml0complete(Pid, FN, StartKey, EndKey, Bloom) -> pcl_confirml0complete(Pid, FN, StartKey, EndKey) ->
gen_server:cast(Pid, {levelzero_complete, FN, StartKey, EndKey, Bloom}). gen_server:cast(Pid, {levelzero_complete, FN, StartKey, EndKey}).
pcl_confirmdelete(Pid, FileName) -> pcl_confirmdelete(Pid, FileName) ->
gen_server:cast(Pid, {confirm_delete, FileName}). gen_server:cast(Pid, {confirm_delete, FileName}).
@ -458,11 +458,10 @@ handle_cast({confirm_delete, FileName}, State=#state{is_snapshot=Snap})
_ -> _ ->
{noreply, State} {noreply, State}
end; end;
handle_cast({levelzero_complete, FN, StartKey, EndKey, Bloom}, State) -> handle_cast({levelzero_complete, FN, StartKey, EndKey}, State) ->
leveled_log:log("P0029", []), leveled_log:log("P0029", []),
ManEntry = #manifest_entry{start_key=StartKey, ManEntry = #manifest_entry{start_key=StartKey,
end_key=EndKey, end_key=EndKey,
bloom=Bloom,
owner=State#state.levelzero_constructor, owner=State#state.levelzero_constructor,
filename=FN}, filename=FN},
UpdMan = lists:keystore(0, 1, State#state.manifest, {0, [ManEntry]}), UpdMan = lists:keystore(0, 1, State#state.manifest, {0, [ManEntry]}),
@ -737,7 +736,7 @@ fetch_mem(Key, Hash, Manifest, L0Cache, none) ->
L0Check = leveled_pmem:check_levelzero(Key, Hash, L0Cache), L0Check = leveled_pmem:check_levelzero(Key, Hash, L0Cache),
case L0Check of case L0Check of
{false, not_found} -> {false, not_found} ->
fetch(Key, Hash, Manifest, 0, fun leveled_sft:sft_get/2); fetch(Key, Manifest, 0, fun leveled_sft:sft_get/2);
{true, KV} -> {true, KV} ->
KV KV
end; end;
@ -746,38 +745,32 @@ fetch_mem(Key, Hash, Manifest, L0Cache, L0Index) ->
true -> true ->
fetch_mem(Key, Hash, Manifest, L0Cache, none); fetch_mem(Key, Hash, Manifest, L0Cache, none);
false -> false ->
fetch(Key, Hash, Manifest, 0, fun leveled_sft:sft_get/2) fetch(Key, Manifest, 0, fun leveled_sft:sft_get/2)
end. end.
fetch(_Key, _Hash, _Manifest, ?MAX_LEVELS + 1, _FetchFun) -> fetch(_Key, _Manifest, ?MAX_LEVELS + 1, _FetchFun) ->
not_present; not_present;
fetch(Key, Hash, Manifest, Level, FetchFun) -> fetch(Key, Manifest, Level, FetchFun) ->
LevelManifest = get_item(Level, Manifest, []), LevelManifest = get_item(Level, Manifest, []),
case lists:foldl(fun(File, Acc) -> case lists:foldl(fun(File, Acc) ->
case Acc of case Acc of
not_present when not_present when
Key >= File#manifest_entry.start_key, Key >= File#manifest_entry.start_key,
File#manifest_entry.end_key >= Key -> File#manifest_entry.end_key >= Key ->
{File#manifest_entry.owner, File#manifest_entry.owner;
File#manifest_entry.bloom};
FoundDetails -> FoundDetails ->
FoundDetails FoundDetails
end end, end end,
not_present, not_present,
LevelManifest) of LevelManifest) of
not_present -> not_present ->
fetch(Key, Hash, Manifest, Level + 1, FetchFun); fetch(Key, Manifest, Level + 1, FetchFun);
{FileToCheck, Bloom} -> FileToCheck ->
case leveled_tinybloom:check({hash, Hash}, Bloom) of
true ->
case FetchFun(FileToCheck, Key) of case FetchFun(FileToCheck, Key) of
not_present -> not_present ->
fetch(Key, Hash, Manifest, Level + 1, FetchFun); fetch(Key, Manifest, Level + 1, FetchFun);
ObjectFound -> ObjectFound ->
ObjectFound ObjectFound
end;
false ->
fetch(Key, Hash, Manifest, Level + 1, FetchFun)
end end
end. end.

23
src/leveled_sft.erl
View file

@ -161,7 +161,6 @@
sft_newfroml0cache/4, sft_newfroml0cache/4,
sft_open/1, sft_open/1,
sft_get/2, sft_get/2,
sft_getbloom/1,
sft_getkvrange/4, sft_getkvrange/4,
sft_close/1, sft_close/1,
sft_clear/1, sft_clear/1,
@ -213,8 +212,7 @@
handle :: file:fd(), handle :: file:fd(),
background_complete = false :: boolean(), background_complete = false :: boolean(),
oversized_file = false :: boolean(), oversized_file = false :: boolean(),
penciller :: pid(), penciller :: pid()}).
bloom}).
%%%============================================================================ %%%============================================================================
@ -271,9 +269,6 @@ sft_open(Filename) ->
sft_setfordelete(Pid, Penciller) -> sft_setfordelete(Pid, Penciller) ->
gen_fsm:sync_send_event(Pid, {set_for_delete, Penciller}, infinity). gen_fsm:sync_send_event(Pid, {set_for_delete, Penciller}, infinity).
sft_getbloom(Pid) ->
gen_fsm:sync_send_event(Pid, get_bloom, infinity).
sft_get(Pid, Key) -> sft_get(Pid, Key) ->
gen_fsm:sync_send_event(Pid, {get_kv, Key}, infinity). gen_fsm:sync_send_event(Pid, {get_kv, Key}, infinity).
@ -348,9 +343,8 @@ starting({sft_newfroml0cache, Filename, Slots, FetchFun, PCL}, _State) ->
leveled_penciller:pcl_confirml0complete(PCL, leveled_penciller:pcl_confirml0complete(PCL,
State#state.filename, State#state.filename,
State#state.smallest_key, State#state.smallest_key,
State#state.highest_key, State#state.highest_key),
State#state.bloom), {next_state, reader, State}
{next_state, reader, State#state{bloom=none}}
end. end.
@ -385,12 +379,6 @@ reader(background_complete, _From, State) ->
reader, reader,
State} State}
end; end;
reader(get_bloom, _From, State) ->
Bloom = State#state.bloom,
if
Bloom /= none ->
{reply, {ok, Bloom}, reader, State#state{bloom=none}}
end;
reader(close, _From, State) -> reader(close, _From, State) ->
ok = file:close(State#state.handle), ok = file:close(State#state.handle),
{stop, normal, ok, State}. {stop, normal, ok, State}.
@ -523,7 +511,7 @@ open_file(FileMD) ->
Slen:32/integer>> = HeaderLengths, Slen:32/integer>> = HeaderLengths,
{ok, SummaryBin} = file:pread(Handle, {ok, SummaryBin} = file:pread(Handle,
?HEADER_LEN + Blen + Ilen + Flen, Slen), ?HEADER_LEN + Blen + Ilen + Flen, Slen),
{{LowSQN, HighSQN}, {LowKey, HighKey}, Bloom} = binary_to_term(SummaryBin), {{LowSQN, HighSQN}, {LowKey, HighKey}, _Bloom} = binary_to_term(SummaryBin),
{ok, SlotIndexBin} = file:pread(Handle, ?HEADER_LEN + Blen, Ilen), {ok, SlotIndexBin} = file:pread(Handle, ?HEADER_LEN + Blen, Ilen),
SlotIndex = binary_to_term(SlotIndexBin), SlotIndex = binary_to_term(SlotIndexBin),
{Handle, FileMD#state{slot_index=SlotIndex, {Handle, FileMD#state{slot_index=SlotIndex,
@ -536,8 +524,7 @@ open_file(FileMD) ->
filter_pointer=?HEADER_LEN + Blen + Ilen, filter_pointer=?HEADER_LEN + Blen + Ilen,
summ_pointer=?HEADER_LEN + Blen + Ilen + Flen, summ_pointer=?HEADER_LEN + Blen + Ilen + Flen,
summ_length=Slen, summ_length=Slen,
handle=Handle, handle=Handle}}.
bloom=Bloom}}.
%% Take a file handle with a previously created header and complete it based on %% Take a file handle with a previously created header and complete it based on
%% the two key lists KL1 and KL2 %% the two key lists KL1 and KL2