From ba628c2f40fc261ab6faed542d19b8aa5cbbbcb1 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 4 Nov 2016 12:22:15 +0000 Subject: [PATCH] Test Rolling of CDB to two files trhough compaction This exposed a potential issue with not opening readers in binary_mode - so now defaults to binary mode. Will add test using object filder to confirm values remain readable in rolled journals after shutdown/startup. --- src/leveled_cdb.erl | 42 +++++++++++++++++++++++++++--------------- src/leveled_iclerk.erl | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 15 deletions(-) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 4cef684..2d9746f 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -116,7 +116,7 @@ cdb_open_writer(Filename) -> %% No options passed - cdb_open_writer(Filename, #cdb_options{}). + cdb_open_writer(Filename, #cdb_options{binary_mode=true}). cdb_open_writer(Filename, Opts) -> {ok, Pid} = gen_fsm:start(?MODULE, [Opts], []), @@ -124,7 +124,10 @@ cdb_open_writer(Filename, Opts) -> {ok, Pid}. cdb_open_reader(Filename) -> - {ok, Pid} = gen_fsm:start(?MODULE, [#cdb_options{}], []), + cdb_open_reader(Filename, #cdb_options{binary_mode=true}). + +cdb_open_reader(Filename, Opts) -> + {ok, Pid} = gen_fsm:start(?MODULE, [Opts], []), ok = gen_fsm:sync_send_event(Pid, {open_reader, Filename}, infinity), {ok, Pid}. @@ -1637,7 +1640,8 @@ emptyvalue_fromdict_test() -> ok = file:delete("../test/from_dict_test_ev.cdb"). find_lastkey_test() -> - {ok, P1} = cdb_open_writer("../test/lastkey.pnd"), + {ok, P1} = cdb_open_writer("../test/lastkey.pnd", + #cdb_options{binary_mode=false}), ok = cdb_put(P1, "Key1", "Value1"), ok = cdb_put(P1, "Key3", "Value3"), ok = cdb_put(P1, "Key2", "Value2"), @@ -1645,7 +1649,8 @@ find_lastkey_test() -> ?assertMatch("Key1", cdb_firstkey(P1)), probably = cdb_keycheck(P1, "Key2"), ok = cdb_close(P1), - {ok, P2} = cdb_open_writer("../test/lastkey.pnd"), + {ok, P2} = cdb_open_writer("../test/lastkey.pnd", + #cdb_options{binary_mode=false}), ?assertMatch("Key2", cdb_lastkey(P2)), probably = cdb_keycheck(P2, "Key2"), {ok, F2} = cdb_complete(P2), @@ -1658,13 +1663,14 @@ find_lastkey_test() -> ok = file:delete("../test/lastkey.cdb"). get_keys_byposition_simple_test() -> - {ok, P1} = cdb_open_writer("../test/poskey.pnd"), + {ok, P1} = cdb_open_writer("../test/poskey.pnd", + #cdb_options{binary_mode=false}), ok = cdb_put(P1, "Key1", "Value1"), ok = cdb_put(P1, "Key3", "Value3"), ok = cdb_put(P1, "Key2", "Value2"), KeyList = ["Key1", "Key2", "Key3"], {ok, F2} = cdb_complete(P1), - {ok, P2} = cdb_open_reader(F2), + {ok, P2} = cdb_open_reader(F2, #cdb_options{binary_mode=false}), PositionList = cdb_getpositions(P2, all), io:format("Position list of ~w~n", [PositionList]), ?assertMatch(3, length(PositionList)), @@ -1699,7 +1705,8 @@ generate_sequentialkeys(Count, KVList) -> get_keys_byposition_manykeys_test() -> KeyCount = 1024, - {ok, P1} = cdb_open_writer("../test/poskeymany.pnd"), + {ok, P1} = cdb_open_writer("../test/poskeymany.pnd", + #cdb_options{binary_mode=false}), KVList = generate_sequentialkeys(KeyCount, []), lists:foreach(fun({K, V}) -> cdb_put(P1, K, V) end, KVList), SW1 = os:timestamp(), @@ -1707,7 +1714,7 @@ get_keys_byposition_manykeys_test() -> SW2 = os:timestamp(), io:format("CDB completed in ~w microseconds~n", [timer:now_diff(SW2, SW1)]), - {ok, P2} = cdb_open_reader(F2), + {ok, P2} = cdb_open_reader(F2, #cdb_options{binary_mode=false}), SW3 = os:timestamp(), io:format("CDB opened for read in ~w microseconds~n", [timer:now_diff(SW3, SW2)]), @@ -1729,9 +1736,10 @@ get_keys_byposition_manykeys_test() -> nokeys_test() -> - {ok, P1} = cdb_open_writer("../test/nohash_emptyfile.pnd"), + {ok, P1} = cdb_open_writer("../test/nohash_emptyfile.pnd", + #cdb_options{binary_mode=false}), {ok, F2} = cdb_complete(P1), - {ok, P2} = cdb_open_reader(F2), + {ok, P2} = cdb_open_reader(F2, #cdb_options{binary_mode=false}), io:format("FirstKey is ~s~n", [cdb_firstkey(P2)]), io:format("LastKey is ~s~n", [cdb_lastkey(P2)]), ?assertMatch(empty, cdb_firstkey(P2)), @@ -1741,7 +1749,8 @@ nokeys_test() -> mput_test() -> KeyCount = 1024, - {ok, P1} = cdb_open_writer("../test/nohash_keysinfile.pnd"), + {ok, P1} = cdb_open_writer("../test/nohash_keysinfile.pnd", + #cdb_options{binary_mode=false}), KVList = generate_sequentialkeys(KeyCount, []), ok = cdb_mput(P1, KVList), ?assertMatch({"Key1", "Value1"}, cdb_get(P1, "Key1")), @@ -1749,7 +1758,7 @@ mput_test() -> ?assertMatch(missing, cdb_get(P1, "Key1025")), ?assertMatch(missing, cdb_get(P1, "Key1026")), {ok, F2} = cdb_complete(P1), - {ok, P2} = cdb_open_reader(F2), + {ok, P2} = cdb_open_reader(F2, #cdb_options{binary_mode=false}), ?assertMatch("Key1", cdb_firstkey(P2)), ?assertMatch("Key1024", cdb_lastkey(P2)), ?assertMatch({"Key1", "Value1"}, cdb_get(P2, "Key1")), @@ -1760,7 +1769,8 @@ mput_test() -> ok = file:delete(F2). state_test() -> - {ok, P1} = cdb_open_writer("../test/state_test.pnd"), + {ok, P1} = cdb_open_writer("../test/state_test.pnd", + #cdb_options{binary_mode=false}), KVList = generate_sequentialkeys(1000, []), ok = cdb_mput(P1, KVList), ?assertMatch(probably, cdb_keycheck(P1, "Key1")), @@ -1778,7 +1788,8 @@ state_test() -> corruptfile_test() -> file:delete("../test/corrupt_test.pnd"), - {ok, P1} = cdb_open_writer("../test/corrupt_test.pnd"), + {ok, P1} = cdb_open_writer("../test/corrupt_test.pnd", + #cdb_options{binary_mode=false}), KVList = generate_sequentialkeys(100, []), ok = cdb_mput(P1, []), % Not relevant to this test, but needs testing lists:foreach(fun({K, V}) -> cdb_put(P1, K, V) end, KVList), @@ -1796,7 +1807,8 @@ corrupt_testfile_at_offset(Offset) -> file:position(F1, EofPos - Offset), ok = file:truncate(F1), ok = file:close(F1), - {ok, P2} = cdb_open_writer("../test/corrupt_test.pnd"), + {ok, P2} = cdb_open_writer("../test/corrupt_test.pnd", + #cdb_options{binary_mode=false}), ?assertMatch(probably, cdb_keycheck(P2, "Key1")), ?assertMatch({"Key1", "Value1"}, cdb_get(P2, "Key1")), ?assertMatch(missing, cdb_get(P2, "Key100")), diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index ecd05ab..f955ccb 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -780,4 +780,45 @@ compare_candidate_test() -> ?assertMatch([Candidate1, Candidate2, Candidate3, Candidate4], sort_run([Candidate3, Candidate2, Candidate4, Candidate1])). +compact_singlefile_totwosmallfiles_test() -> + RP = "../test/journal", + CP = "../test/journal/journal_file/post_compact/", + ok = filelib:ensure_dir(CP), + FN1 = leveled_inker:filepath(RP, 1, new_journal), + CDBoptsLarge = #cdb_options{binary_mode=true, max_size=30000000}, + {ok, CDB1} = leveled_cdb:cdb_open_writer(FN1, CDBoptsLarge), + lists:foreach(fun(X) -> + LK = test_ledgerkey("Key" ++ integer_to_list(X)), + Value = term_to_binary({crypto:rand_bytes(1024), []}), + ok = leveled_cdb:cdb_put(CDB1, + {X, ?INKT_STND, LK}, + Value) + end, + lists:seq(1, 1000)), + {ok, NewName} = leveled_cdb:cdb_complete(CDB1), + {ok, CDBr} = leveled_cdb:cdb_open_reader(NewName), + CDBoptsSmall = #cdb_options{binary_mode=true, max_size=400000, file_path=CP}, + BestRun1 = [#candidate{low_sqn=1, + filename=leveled_cdb:cdb_filename(CDBr), + journal=CDBr, + compaction_perc=50.0}], + FakeFilterFun = fun(_FS, _LK, SQN) -> SQN rem 2 == 0 end, + + {ManifestSlice, PromptDelete} = compact_files(BestRun1, + CDBoptsSmall, + FakeFilterFun, + null, + 900, + [{?STD_TAG, recovr}]), + ?assertMatch(2, length(ManifestSlice)), + ?assertMatch(true, PromptDelete), + lists:foreach(fun({_SQN, _FN, CDB}) -> + ok = leveled_cdb:cdb_deletepending(CDB), + ok = leveled_cdb:cdb_destroy(CDB) + end, + ManifestSlice), + ok = leveled_cdb:cdb_deletepending(CDBr), + ok = leveled_cdb:cdb_destroy(CDBr). + + -endif. \ No newline at end of file