From eba21f49fa75cc52dc16b12b8c21ad658fdf651c Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 15 Sep 2017 15:10:04 +0100 Subject: [PATCH] Make tests compatible with OTP 16 this required a switch to change the sync strategy based on rebar parameter. However tests could be slow on macbook with OTP16 and sync - so timeouts added in unit tests, and ct tests sync_startegy changed to not sync for OTP16. --- .rebar/erlcinfo | Bin 0 -> 913 bytes rebar.config | 3 ++- src/leveled_bookie.erl | 20 ++++++++++++++++---- src/leveled_cdb.erl | 31 ++++++++++++++++++++++++++----- src/leveled_iclerk.erl | 5 ++++- src/leveled_penciller.erl | 1 + src/leveled_rand.erl | 11 +++++++++++ test/end_to_end/testutil.erl | 5 +++-- 8 files changed, 63 insertions(+), 13 deletions(-) create mode 100644 .rebar/erlcinfo diff --git a/.rebar/erlcinfo b/.rebar/erlcinfo new file mode 100644 index 0000000000000000000000000000000000000000..dc2cd62c4e84fff2719fa64c98ce1aeb5c3a5bcf GIT binary patch literal 913 zcmV;C18)3-PyhfI9e7@~SjleNFc4+zG;#JlZI)(h(iN8C7)egO6zHKS&_mmxM`0u} z5v|S85+Hxu{z)mtPDEOwXuv$l3(1-H=FLJmO&hgvOn*5E8GQ3lpv3%VHtpYy%< zp_lqtk4bRnVIv4>a5J|}h@H?dDNe_*BXW-;gUTtF6i+O7EE8)p3|+$E9tL8DigrY; zX?V+*GNN!l0Mg?9PGQ~sNf zsv{oBqoB&5v|xdfyZ~Le&PZmIV$q!8RLIrX&D@Yjb0O{5V`nN2)yjiOcltuQM#?~h zt&ryILm3t>ENo%(At1MP(P0W(-^XI-7z0Ggz@PELpV4d3?4pRdZxx zEDkJcn%dRbr_%5SS5uz4k4mFwE|Mr5GO=3V)l$rIl@OMA!^Oc&rQs<~ox|}?hOOPI z1|qZh6*o;kkFGMiUvRshM<|)yGjvhO-9jbL?@BZ}I^)VXzgl?>k=eb*K_A?dR6;*- zk?Ahr4n(&p6RpxqCL8>Qhn-euNlglXcSZc^r z3jctsCC@J!WB}|{Wu-dNtO(kVI;*oB(81e48IC^4^ox42O$#Otb-3Tu_D;d84>r-Z n#Mz<_!`qTFGkULA+_O(>i4?uJWSM*|$k~a~*z5fVHQZjJ5ai0S literal 0 HcmV?d00001 diff --git a/rebar.config b/rebar.config index bee896c..c849ed0 100644 --- a/rebar.config +++ b/rebar.config @@ -1,7 +1,8 @@ {erl_opts, [warnings_as_errors, {platform_define, "18", old_rand}, {platform_define, "17", old_rand}, - {platform_define, "^R", old_rand}]}. + {platform_define, "^R", old_rand}, + {platform_define, "^R", no_sync}]}. {profiles, [{eqc, [{deps, [meck, fqc]}, diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 877bbee..b240b16 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -1671,7 +1671,10 @@ ttl_test() -> ok = book_close(Bookie2), reset_filestructure(). -hashlist_query_test() -> +hashlist_query_test_() -> + {timeout, 60, fun hashlist_query_testto/0}. + +hashlist_query_testto() -> RootPath = reset_filestructure(), {ok, Bookie1} = book_start([{root_path, RootPath}, {max_journalsize, 1000000}, @@ -1719,7 +1722,11 @@ hashlist_query_test() -> ok = book_close(Bookie2), reset_filestructure(). -hashlist_query_withjournalcheck_test() -> + +hashlist_query_withjournalcheck_test_() -> + {timeout, 60, fun hashlist_query_withjournalcheck_testto/0}. + +hashlist_query_withjournalcheck_testto() -> RootPath = reset_filestructure(), {ok, Bookie1} = book_start([{root_path, RootPath}, {max_journalsize, 1000000}, @@ -1745,7 +1752,10 @@ hashlist_query_withjournalcheck_test() -> ok = book_close(Bookie1), reset_filestructure(). -foldobjects_vs_hashtree_test() -> +foldobjects_vs_hashtree_test_() -> + {timeout, 60, fun foldobjects_vs_hashtree_testto/0}. + +foldobjects_vs_hashtree_testto() -> RootPath = reset_filestructure(), {ok, Bookie1} = book_start([{root_path, RootPath}, {max_journalsize, 1000000}, @@ -1815,8 +1825,10 @@ foldobjects_vs_hashtree_test() -> ok = book_close(Bookie1), reset_filestructure(). +foldobjects_vs_foldheads_bybucket_test_() -> + {timeout, 60, fun foldobjects_vs_foldheads_bybucket_testto/0}. -foldobjects_vs_foldheads_bybucket_test() -> +foldobjects_vs_foldheads_bybucket_testto() -> RootPath = reset_filestructure(), {ok, Bookie1} = book_start([{root_path, RootPath}, {max_journalsize, 1000000}, diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index c18277f..73959b1 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -370,10 +370,11 @@ init([Opts]) -> starting({open_writer, Filename}, _From, State) -> leveled_log:log("CDB01", [Filename]), {LastPosition, HashTree, LastKey} = open_active_file(Filename), - WriteOps = set_writeops(State#state.sync_strategy), + {WriteOps, UpdStrategy} = set_writeops(State#state.sync_strategy), leveled_log:log("CDB13", [WriteOps]), {ok, Handle} = file:open(Filename, WriteOps), {reply, ok, writer, State#state{handle=Handle, + sync_strategy = UpdStrategy, last_position=LastPosition, last_key=LastKey, filename=Filename, @@ -714,6 +715,8 @@ code_change(_OldVsn, StateName, State, _Extra) -> %%% Internal functions %%%============================================================================ + +-spec set_writeops(sync|riak_sync|none) -> {list(), sync|riak_sync|none}. %% Assumption is that sync should be used - it is a transaction log. %% %% However this flag is not supported in OTP 16. Bitcask appears to pass an @@ -721,16 +724,31 @@ code_change(_OldVsn, StateName, State, _Extra) -> %% bitcask nif fileops). %% %% To get round this will try and datasync on each PUT with riak_sync +-ifdef(no_sync). + set_writeops(SyncStrategy) -> case SyncStrategy of sync -> - [sync | ?WRITE_OPS]; + {?WRITE_OPS, riak_sync}; riak_sync -> - ?WRITE_OPS; + {?WRITE_OPS, riak_sync}; none -> - ?WRITE_OPS + {?WRITE_OPS, none} end. +-else. + +set_writeops(SyncStrategy) -> + case SyncStrategy of + sync -> + {[sync | ?WRITE_OPS], sync}; + riak_sync -> + {?WRITE_OPS, riak_sync}; + none -> + {?WRITE_OPS, none} + end. + +-endif. %% from_dict(FileName,ListOfKeyValueTuples) %% Given a filename and a dictionary, create a cdb @@ -1956,7 +1974,10 @@ generate_sequentialkeys(Count, KVList) -> KV = {"Key" ++ integer_to_list(Count), "Value" ++ integer_to_list(Count)}, generate_sequentialkeys(Count - 1, KVList ++ [KV]). -get_keys_byposition_manykeys_test() -> +get_keys_byposition_manykeys_test_() -> + {timeout, 60, fun get_keys_byposition_manykeys_test_to/0}. + +get_keys_byposition_manykeys_test_to() -> KeyCount = 1024, {ok, P1} = cdb_open_writer("../test/poskeymany.pnd", #cdb_options{binary_mode=false}), diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index 0a4e4c0..f19746a 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -923,7 +923,10 @@ compare_candidate_test() -> ?assertMatch([Candidate1, Candidate2, Candidate3, Candidate4], sort_run([Candidate3, Candidate2, Candidate4, Candidate1])). -compact_singlefile_totwosmallfiles_test() -> +compact_singlefile_totwosmallfiles_test_() -> + {timeout, 60, fun compact_singlefile_totwosmallfiles_testto/0}. + +compact_singlefile_totwosmallfiles_testto() -> RP = "../test/journal", CP = "../test/journal/journal_file/post_compact/", ok = filelib:ensure_dir(CP), diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 268feca..8330235 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -1335,6 +1335,7 @@ add_missing_hash({K, {SQN, ST, MD}}) -> clean_dir_test() -> % Pointless gesture to test coverage RootPath = "../test/ledger", + ok = filelib:ensure_dir(RootPath), ?assertMatch(ok, file:write_file(RootPath ++ "/test.bob", "hello")), ok = clean_subdir(RootPath ++ "/test.bob"), ok = file:delete(RootPath ++ "/test.bob"). diff --git a/src/leveled_rand.erl b/src/leveled_rand.erl index 0849250..4aabe5e 100644 --- a/src/leveled_rand.erl +++ b/src/leveled_rand.erl @@ -11,6 +11,9 @@ rand_bytes/1 ]). + +-include_lib("eunit/include/eunit.hrl"). + %%%=================================================================== %%% New (r19+) rand style functions %%%=================================================================== @@ -45,3 +48,11 @@ rand_bytes(Size) -> crypto:rand_bytes(Size). -endif. + + +-ifdef(TEST). + +rand_test() -> + ?assertMatch(true, uniform() < 1). + +-endif. diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index ef0318f..a804e55 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -137,11 +137,12 @@ sync_strategy() -> sync; "19" -> sync; - "16" -> + _ -> + % running the sync strategy with OTP16 on macbook is + % super slow. So revert to no sync none end. - book_riakput(Pid, RiakObject, IndexSpecs) -> leveled_bookie:book_put(Pid, RiakObject#r_object.bucket,