From 86bdfdeaf034ca51ccbc2920e5793e55203c8b10 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sun, 11 Dec 2016 21:01:10 +0000 Subject: [PATCH] Reverted back out the additional bloom check This is desirable to add back in going forward, but wasn't implemented in a safe or clear way. The way the bloom was or was not on the LoopState was clumsy, and it got persisted in multiple places without a CRC check. Intention to implement back in wherby it is requested on-demand by the Penciller, and then the SFT worker lifts it off disk and CRC checks it. So it is never on the SFT LoopState. Also it will be easier to control the logic over which levels have the bloom in the Penciller. --- include/leveled.hrl | 1 - src/leveled_pclerk.erl | 2 -- src/leveled_penciller.erl | 39 ++++++++++++++++----------------------- src/leveled_sft.erl | 23 +++++------------------ 4 files changed, 21 insertions(+), 44 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index f57ffd4..25216f6 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -41,7 +41,6 @@ {start_key :: tuple(), end_key :: tuple(), owner :: pid(), - bloom, filename :: string()}). -record(cdb_options, diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 63884bd..b5f8e3f 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -320,7 +320,6 @@ do_merge(KL1, KL2, {SrcLevel, IsB}, {Filepath, MSN}, FileCounter, OutList) -> KL1, KL2, LevelR), - {ok, Bloom} = leveled_sft:sft_getbloom(Pid), case Reply of {{[], []}, null, _} -> leveled_log:log("PC013", [FileName]), @@ -332,7 +331,6 @@ do_merge(KL1, KL2, {SrcLevel, IsB}, {Filepath, MSN}, FileCounter, OutList) -> [#manifest_entry{start_key=SmallestKey, end_key=HighestKey, owner=Pid, - bloom=Bloom, filename=FileName}]), leveled_log:log_timer("PC015", [], TS1), do_merge(KL1Rem, KL2Rem, diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index ce83252..d5b70d1 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -175,7 +175,7 @@ pcl_checksequencenumber/4, pcl_workforclerk/1, pcl_promptmanifestchange/2, - pcl_confirml0complete/5, + pcl_confirml0complete/4, pcl_confirmdelete/2, pcl_close/1, pcl_doom/1, @@ -286,8 +286,8 @@ pcl_workforclerk(Pid) -> pcl_promptmanifestchange(Pid, WI) -> gen_server:cast(Pid, {manifest_change, WI}). -pcl_confirml0complete(Pid, FN, StartKey, EndKey, Bloom) -> - gen_server:cast(Pid, {levelzero_complete, FN, StartKey, EndKey, Bloom}). +pcl_confirml0complete(Pid, FN, StartKey, EndKey) -> + gen_server:cast(Pid, {levelzero_complete, FN, StartKey, EndKey}). pcl_confirmdelete(Pid, FileName) -> gen_server:cast(Pid, {confirm_delete, FileName}). @@ -458,11 +458,10 @@ handle_cast({confirm_delete, FileName}, State=#state{is_snapshot=Snap}) _ -> {noreply, State} end; -handle_cast({levelzero_complete, FN, StartKey, EndKey, Bloom}, State) -> +handle_cast({levelzero_complete, FN, StartKey, EndKey}, State) -> leveled_log:log("P0029", []), ManEntry = #manifest_entry{start_key=StartKey, end_key=EndKey, - bloom=Bloom, owner=State#state.levelzero_constructor, filename=FN}, UpdMan = lists:keystore(0, 1, State#state.manifest, {0, [ManEntry]}), @@ -737,7 +736,7 @@ fetch_mem(Key, Hash, Manifest, L0Cache, none) -> L0Check = leveled_pmem:check_levelzero(Key, Hash, L0Cache), case L0Check of {false, not_found} -> - fetch(Key, Hash, Manifest, 0, fun leveled_sft:sft_get/2); + fetch(Key, Manifest, 0, fun leveled_sft:sft_get/2); {true, KV} -> KV end; @@ -746,38 +745,32 @@ fetch_mem(Key, Hash, Manifest, L0Cache, L0Index) -> true -> fetch_mem(Key, Hash, Manifest, L0Cache, none); false -> - fetch(Key, Hash, Manifest, 0, fun leveled_sft:sft_get/2) + fetch(Key, Manifest, 0, fun leveled_sft:sft_get/2) end. -fetch(_Key, _Hash, _Manifest, ?MAX_LEVELS + 1, _FetchFun) -> +fetch(_Key, _Manifest, ?MAX_LEVELS + 1, _FetchFun) -> not_present; -fetch(Key, Hash, Manifest, Level, FetchFun) -> +fetch(Key, Manifest, Level, FetchFun) -> LevelManifest = get_item(Level, Manifest, []), case lists:foldl(fun(File, Acc) -> case Acc of not_present when Key >= File#manifest_entry.start_key, File#manifest_entry.end_key >= Key -> - {File#manifest_entry.owner, - File#manifest_entry.bloom}; + File#manifest_entry.owner; FoundDetails -> FoundDetails end end, not_present, LevelManifest) of not_present -> - fetch(Key, Hash, Manifest, Level + 1, FetchFun); - {FileToCheck, Bloom} -> - case leveled_tinybloom:check({hash, Hash}, Bloom) of - true -> - case FetchFun(FileToCheck, Key) of - not_present -> - fetch(Key, Hash, Manifest, Level + 1, FetchFun); - ObjectFound -> - ObjectFound - end; - false -> - fetch(Key, Hash, Manifest, Level + 1, FetchFun) + fetch(Key, Manifest, Level + 1, FetchFun); + FileToCheck -> + case FetchFun(FileToCheck, Key) of + not_present -> + fetch(Key, Manifest, Level + 1, FetchFun); + ObjectFound -> + ObjectFound end end. diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 206f0bd..5b4f24e 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -161,7 +161,6 @@ sft_newfroml0cache/4, sft_open/1, sft_get/2, - sft_getbloom/1, sft_getkvrange/4, sft_close/1, sft_clear/1, @@ -213,8 +212,7 @@ handle :: file:fd(), background_complete = false :: boolean(), oversized_file = false :: boolean(), - penciller :: pid(), - bloom}). + penciller :: pid()}). %%%============================================================================ @@ -271,9 +269,6 @@ sft_open(Filename) -> sft_setfordelete(Pid, Penciller) -> gen_fsm:sync_send_event(Pid, {set_for_delete, Penciller}, infinity). -sft_getbloom(Pid) -> - gen_fsm:sync_send_event(Pid, get_bloom, infinity). - sft_get(Pid, Key) -> gen_fsm:sync_send_event(Pid, {get_kv, Key}, infinity). @@ -348,9 +343,8 @@ starting({sft_newfroml0cache, Filename, Slots, FetchFun, PCL}, _State) -> leveled_penciller:pcl_confirml0complete(PCL, State#state.filename, State#state.smallest_key, - State#state.highest_key, - State#state.bloom), - {next_state, reader, State#state{bloom=none}} + State#state.highest_key), + {next_state, reader, State} end. @@ -385,12 +379,6 @@ reader(background_complete, _From, State) -> reader, State} end; -reader(get_bloom, _From, State) -> - Bloom = State#state.bloom, - if - Bloom /= none -> - {reply, {ok, Bloom}, reader, State#state{bloom=none}} - end; reader(close, _From, State) -> ok = file:close(State#state.handle), {stop, normal, ok, State}. @@ -523,7 +511,7 @@ open_file(FileMD) -> Slen:32/integer>> = HeaderLengths, {ok, SummaryBin} = file:pread(Handle, ?HEADER_LEN + Blen + Ilen + Flen, Slen), - {{LowSQN, HighSQN}, {LowKey, HighKey}, Bloom} = binary_to_term(SummaryBin), + {{LowSQN, HighSQN}, {LowKey, HighKey}, _Bloom} = binary_to_term(SummaryBin), {ok, SlotIndexBin} = file:pread(Handle, ?HEADER_LEN + Blen, Ilen), SlotIndex = binary_to_term(SlotIndexBin), {Handle, FileMD#state{slot_index=SlotIndex, @@ -536,8 +524,7 @@ open_file(FileMD) -> filter_pointer=?HEADER_LEN + Blen + Ilen, summ_pointer=?HEADER_LEN + Blen + Ilen + Flen, summ_length=Slen, - handle=Handle, - bloom=Bloom}}. + handle=Handle}}. %% Take a file handle with a previously created header and complete it based on %% the two key lists KL1 and KL2