umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Timeout and close race

Browse source 
Race condition presvented in test - but still not handled nicely.
Perhaps need to consider making it a FSM and handling close differently
when L0 pending - i.e. don't close immediately, but set a timeout to
close on if we don't get the last fetch_levelzero
This commit is contained in:
martinsumner 2016-11-03 19:02:50 +00:00
parent f41c788bff
commit 2716d912ea
2 changed files with 15 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

19
src/leveled_penciller.erl
View file

@ -290,7 +290,12 @@ pcl_pushmem(Pid, DumpList) ->
gen_server:call(Pid, {push_mem, DumpList}, infinity). gen_server:call(Pid, {push_mem, DumpList}, infinity).
pcl_fetchlevelzero(Pid, Slot) -> pcl_fetchlevelzero(Pid, Slot) ->
gen_server:call(Pid, {fetch_levelzero, Slot}, infinity). %% Timeout to cause crash of L0 file when it can't get the close signal
%% as it is deadlocked making this call.
%%
%% If the timeout gets hit outside of close scenario the Penciller will
%% be stuck in L0 pending
gen_server:call(Pid, {fetch_levelzero, Slot}, 10000).
pcl_fetch(Pid, Key) -> pcl_fetch(Pid, Key) ->
gen_server:call(Pid, {fetch, Key}, infinity). gen_server:call(Pid, {fetch, Key}, infinity).
@ -1661,12 +1666,16 @@ coverage_test() ->
clean_testdir(RootPath), clean_testdir(RootPath),
{ok, PCL} = pcl_start(#penciller_options{root_path=RootPath, {ok, PCL} = pcl_start(#penciller_options{root_path=RootPath,
max_inmemory_tablesize=1000}), max_inmemory_tablesize=1000}),
Key1 = {{o,"Bucket0001", "Key0001", null}, {1, {active, infinity}, null}}, Key1 = {{o,"Bucket0001", "Key0001", null}, {1001, {active, infinity}, null}},
KL1 = leveled_sft:generate_randomkeys({1000, 2}), KL1 = leveled_sft:generate_randomkeys({1000, 1}),
ok = maybe_pause_push(PCL, [Key1]),
ok = maybe_pause_push(PCL, KL1 ++ [Key1]),
%% Added together, as split apart there will be a race between the close
%% call to the penciller and the second fetch of the cache entry
?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})),
ok = maybe_pause_push(PCL, KL1),
ok = pcl_close(PCL), ok = pcl_close(PCL),
ManifestFP = filepath(RootPath, manifest), ManifestFP = filepath(RootPath, manifest),
file:write_file(ManifestFP ++ "/yeszero_123.man", term_to_binary("hello")), file:write_file(ManifestFP ++ "/yeszero_123.man", term_to_binary("hello")),
{ok, PCLr} = pcl_start(#penciller_options{root_path=RootPath, {ok, PCLr} = pcl_start(#penciller_options{root_path=RootPath,

2
src/leveled_sft.erl
View file

@ -270,7 +270,7 @@ sft_clear(Pid) ->
gen_server:call(Pid, clear, infinity). gen_server:call(Pid, clear, infinity).
sft_close(Pid) -> sft_close(Pid) ->
gen_server:call(Pid, close, infinity). gen_server:call(Pid, close, 1000).
sft_deleteconfirmed(Pid) -> sft_deleteconfirmed(Pid) ->
gen_server:cast(Pid, close). gen_server:cast(Pid, close).