diff --git a/include/leveled.hrl b/include/leveled.hrl index 3a85aa7..028eb95 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -62,15 +62,6 @@ start_snapshot = false :: boolean(), source_penciller :: pid()}). --record(bookie_options, - {root_path :: string(), - cache_size :: integer(), - max_journalsize :: integer(), - max_pencillercachesize :: integer(), - snapshot_bookie :: pid(), - reload_strategy = [] :: list(), - max_run_length :: integer()}). - -record(iclerk_options, {inker :: pid(), max_run_length :: integer(), diff --git a/src/eleveleddb.app.src b/src/eleveleddb.app.src index 6d0069e..37b004d 100644 --- a/src/eleveleddb.app.src +++ b/src/eleveleddb.app.src @@ -1,6 +1,6 @@ {application, eleveleddb, [ - {description, ""}, + {description, "Key Value store based on LSM-Tree and designed for larger values"}, {vsn, "1"}, {registered, []}, {applications, [ @@ -8,5 +8,5 @@ stdlib ]}, {mod, { eleveleddb_app, []}}, - {env, []} + {env, [{root_path, "test"}]} ]}. diff --git a/src/eleveleddb_sup.erl b/src/eleveleddb_sup.erl index 37ef58b..391aca9 100644 --- a/src/eleveleddb_sup.erl +++ b/src/eleveleddb_sup.erl @@ -16,7 +16,7 @@ %% =================================================================== start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). + supervisor:start_link({local, leveled_bookie}, ?MODULE, []). %% =================================================================== %% Supervisor callbacks diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 8ab54cf..4ece04b 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -151,6 +151,9 @@ book_islastcompactionpending/1, book_close/1]). +-export([get_opt/2, + get_opt/3]). + -include_lib("eunit/include/eunit.hrl"). -define(CACHE_SIZE, 2000). @@ -175,9 +178,9 @@ %%%============================================================================ book_start(RootPath, LedgerCacheSize, JournalSize) -> - book_start(#bookie_options{root_path=RootPath, - cache_size=LedgerCacheSize, - max_journalsize=JournalSize}). + book_start([{root_path, RootPath}, + {cache_size, LedgerCacheSize}, + {max_journalsize, JournalSize}]). book_start(Opts) -> gen_server:start(?MODULE, [Opts], []). @@ -247,17 +250,12 @@ book_close(Pid) -> %%%============================================================================ init([Opts]) -> - case Opts#bookie_options.snapshot_bookie of + case get_opt(snapshot_bookie, Opts) of undefined -> % Start from file not snapshot {InkerOpts, PencillerOpts} = set_options(Opts), {Inker, Penciller} = startup(InkerOpts, PencillerOpts), - CacheSize = if - Opts#bookie_options.cache_size == undefined -> - ?CACHE_SIZE; - true -> - Opts#bookie_options.cache_size - end, + CacheSize = get_opt(cache_size, Opts, ?CACHE_SIZE), io:format("Bookie starting with Pcl ~w Ink ~w~n", [Penciller, Inker]), {ok, #state{inker=Inker, @@ -551,21 +549,21 @@ shutdown_wait([TopPause|Rest], Inker) -> set_options(Opts) -> - MaxJournalSize = case Opts#bookie_options.max_journalsize of - undefined -> - 30000; - MS -> - MS - end, + MaxJournalSize = get_opt(max_journalsize, Opts, 10000000000), - AltStrategy = Opts#bookie_options.reload_strategy, + AltStrategy = get_opt(reload_strategy, Opts, []), ReloadStrategy = leveled_codec:inker_reload_strategy(AltStrategy), - PCLL0CacheSize = Opts#bookie_options.max_pencillercachesize, - JournalFP = Opts#bookie_options.root_path ++ "/" ++ ?JOURNAL_FP, - LedgerFP = Opts#bookie_options.root_path ++ "/" ++ ?LEDGER_FP, + + PCLL0CacheSize = get_opt(max_pencillercachesize, Opts), + RootPath = get_opt(root_path, Opts), + JournalFP = RootPath ++ "/" ++ ?JOURNAL_FP, + LedgerFP = RootPath ++ "/" ++ ?LEDGER_FP, + ok =filelib:ensure_dir(JournalFP), + ok =filelib:ensure_dir(LedgerFP), + {#inker_options{root_path = JournalFP, reload_strategy = ReloadStrategy, - max_run_length = Opts#bookie_options.max_run_length, + max_run_length = get_opt(max_run_length, Opts), cdb_options = #cdb_options{max_size=MaxJournalSize, binary_mode=true}}, #penciller_options{root_path = LedgerFP, @@ -781,6 +779,23 @@ load_fun(KeyInLedger, ValueInLedger, _Position, Acc0, ExtractFun) -> end. +get_opt(Key, Opts) -> + get_opt(Key, Opts, undefined). + +get_opt(Key, Opts, Default) -> + case proplists:get_value(Key, Opts) of + undefined -> + case application:get_env(?MODULE, Key) of + {ok, Value} -> + Value; + undefined -> + Default + end; + Value -> + Value + end. + + %%%============================================================================ %%% Test %%%============================================================================ @@ -828,7 +843,7 @@ generate_multiple_robjects(Count, KeyNumber, ObjL) -> single_key_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath}), + {ok, Bookie1} = book_start([{root_path, RootPath}]), {B1, K1, V1, Spec1, MD} = {"Bucket1", "Key1", "Value1", @@ -840,7 +855,7 @@ single_key_test() -> {ok, F1} = book_riakget(Bookie1, B1, K1), ?assertMatch(F1, Object), ok = book_close(Bookie1), - {ok, Bookie2} = book_start(#bookie_options{root_path=RootPath}), + {ok, Bookie2} = book_start([{root_path, RootPath}]), {ok, F2} = book_riakget(Bookie2, B1, K1), ?assertMatch(F2, Object), ok = book_close(Bookie2), @@ -848,7 +863,7 @@ single_key_test() -> multi_key_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath}), + {ok, Bookie1} = book_start([{root_path, RootPath}]), {B1, K1, V1, Spec1, MD1} = {"Bucket", "Key1", "Value1", @@ -885,7 +900,7 @@ multi_key_test() -> ?assertMatch(F2B, Obj2), ok = book_close(Bookie1), % Now reopen the file, and confirm that a fetch is still possible - {ok, Bookie2} = book_start(#bookie_options{root_path=RootPath}), + {ok, Bookie2} = book_start([{root_path, RootPath}]), {ok, F1C} = book_riakget(Bookie2, B1, K1), ?assertMatch(F1C, Obj1), {ok, F2C} = book_riakget(Bookie2, B2, K2), @@ -904,7 +919,7 @@ multi_key_test() -> ttl_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath}), + {ok, Bookie1} = book_start([{root_path, RootPath}]), ObjL1 = generate_multiple_objects(100, 1), % Put in all the objects with a TTL in the future Future = leveled_codec:integer_now() + 300, @@ -962,7 +977,7 @@ ttl_test() -> ?assertMatch(10, length(TermKeyList)), ok = book_close(Bookie1), - {ok, Bookie2} = book_start(#bookie_options{root_path=RootPath}), + {ok, Bookie2} = book_start([{root_path, RootPath}]), {async, IndexFolderTR2} = book_returnfolder(Bookie2, @@ -987,9 +1002,9 @@ ttl_test() -> hashtree_query_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath, - max_journalsize=1000000, - cache_size=500}), + {ok, Bookie1} = book_start([{root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500}]), ObjL1 = generate_multiple_objects(1200, 1), % Put in all the objects with a TTL in the future Future = leveled_codec:integer_now() + 300, @@ -1019,9 +1034,9 @@ hashtree_query_test() -> KeyHashList), ?assertMatch(1200, length(KeyHashList)), ok = book_close(Bookie1), - {ok, Bookie2} = book_start(#bookie_options{root_path=RootPath, - max_journalsize=200000, - cache_size=500}), + {ok, Bookie2} = book_start([{root_path, RootPath}, + {max_journalsize, 200000}, + {cache_size, 500}]), {async, HTFolder2} = book_returnfolder(Bookie2, {hashtree_query, ?STD_TAG, @@ -1032,9 +1047,9 @@ hashtree_query_test() -> hashtree_query_withjournalcheck_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath, - max_journalsize=1000000, - cache_size=500}), + {ok, Bookie1} = book_start([{root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500}]), ObjL1 = generate_multiple_objects(800, 1), % Put in all the objects with a TTL in the future Future = leveled_codec:integer_now() + 300, diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 612a448..bf21318 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -24,15 +24,15 @@ all() -> [ simple_put_fetch_head_delete(_Config) -> RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath}, + StartOpts1 = [{root_path, RootPath}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), testutil:check_forobject(Bookie1, TestObject), testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"), ok = leveled_bookie:book_close(Bookie1), - StartOpts2 = #bookie_options{root_path=RootPath, - max_journalsize=3000000}, + StartOpts2 = [{root_path, RootPath}, + {max_journalsize, 3000000}], {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), testutil:check_forobject(Bookie2, TestObject), ObjList1 = testutil:generate_objects(5000, 2), @@ -69,16 +69,15 @@ simple_put_fetch_head_delete(_Config) -> many_put_fetch_head(_Config) -> RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath, - max_pencillercachesize=16000}, + StartOpts1 = [{root_path, RootPath}, {max_pencillercachesize, 16000}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), testutil:check_forobject(Bookie1, TestObject), ok = leveled_bookie:book_close(Bookie1), - StartOpts2 = #bookie_options{root_path=RootPath, - max_journalsize=1000000000, - max_pencillercachesize=32000}, + StartOpts2 = [{root_path, RootPath}, + {max_journalsize, 1000000000}, + {max_pencillercachesize, 32000}], {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), testutil:check_forobject(Bookie2, TestObject), GenList = [2, 20002, 40002, 60002, 80002, @@ -108,9 +107,9 @@ many_put_fetch_head(_Config) -> journal_compaction(_Config) -> RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath, - max_journalsize=10000000, - max_run_length=1}, + StartOpts1 = [{root_path, RootPath}, + {max_journalsize, 10000000}, + {max_run_length, 1}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), ok = leveled_bookie:book_compactjournal(Bookie1, 30000), {TestObject, TestSpec} = testutil:generate_testobject(), @@ -172,7 +171,7 @@ journal_compaction(_Config) -> fetchput_snapshot(_Config) -> RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=30000000}, + StartOpts1 = [{root_path, RootPath}, {max_journalsize, 30000000}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), @@ -180,7 +179,7 @@ fetchput_snapshot(_Config) -> lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, ObjList1), - SnapOpts1 = #bookie_options{snapshot_bookie=Bookie1}, + SnapOpts1 = [{snapshot_bookie, Bookie1}], {ok, SnapBookie1} = leveled_bookie:book_start(SnapOpts1), ChkList1 = lists:sublist(lists:sort(ObjList1), 100), testutil:check_forlist(Bookie1, ChkList1), @@ -191,7 +190,7 @@ fetchput_snapshot(_Config) -> io:format("Closed initial bookies~n"), {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), - SnapOpts2 = #bookie_options{snapshot_bookie=Bookie2}, + SnapOpts2 = [{snapshot_bookie, Bookie2}], {ok, SnapBookie2} = leveled_bookie:book_start(SnapOpts2), io:format("Bookies restarted~n"), @@ -279,7 +278,7 @@ load_and_count(_Config) -> % Use artificially small files, and the load keys, counting they're all % present RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=50000000}, + StartOpts1 = [{root_path, RootPath}, {max_journalsize, 50000000}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), @@ -362,7 +361,7 @@ load_and_count(_Config) -> load_and_count_withdelete(_Config) -> RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=50000000}, + StartOpts1 = [{root_path, RootPath}, {max_journalsize, 50000000}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), @@ -418,7 +417,7 @@ load_and_count_withdelete(_Config) -> space_clear_ondelete(_Config) -> RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=20000000}, + StartOpts1 = [{root_path, RootPath}, {max_journalsize, 20000000}], {ok, Book1} = leveled_bookie:book_start(StartOpts1), G2 = fun testutil:generate_compressibleobjects/2, testutil:load_objects(20000, diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index 5d34786..1353aa8 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -18,8 +18,8 @@ all() -> [ simple_load_with2i(_Config) -> RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath, - max_journalsize=50000000}, + StartOpts1 = [{root_path, RootPath}, + {max_journalsize, 50000000}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index 723dbd7..b4fbfa2 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -15,12 +15,15 @@ all() -> [ retain_strategy(_Config) -> RootPath = testutil:reset_filestructure(), - BookOpts = #bookie_options{root_path=RootPath, - cache_size=1000, - max_journalsize=5000000, - reload_strategy=[{?RIAK_TAG, retain}]}, - BookOptsAlt = BookOpts#bookie_options{max_run_length=8, - max_journalsize=100000}, + BookOpts = [{root_path, RootPath}, + {cache_size, 1000}, + {max_journalsize, 5000000}, + {reload_strategy, [{?RIAK_TAG, retain}]}], + BookOptsAlt = [{root_path, RootPath}, + {cache_size, 1000}, + {max_journalsize, 100000}, + {reload_strategy, [{?RIAK_TAG, retain}]}, + {max_run_length, 8}], {ok, Spcl3, LastV3} = rotating_object_check(BookOpts, "Bucket3", 800), ok = restart_from_blankledger(BookOpts, [{"Bucket3", Spcl3, LastV3}]), {ok, Spcl4, LastV4} = rotating_object_check(BookOpts, "Bucket4", 1600), @@ -40,8 +43,8 @@ retain_strategy(_Config) -> aae_bustedjournal(_Config) -> RootPath = testutil:reset_filestructure(), - StartOpts = #bookie_options{root_path=RootPath, - max_journalsize=20000000}, + StartOpts = [{root_path, RootPath}, + {max_journalsize, 20000000}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), @@ -107,9 +110,9 @@ aae_bustedjournal(_Config) -> journal_compaction_bustedjournal(_Config) -> % Simply confirms that none of this causes a crash RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath, - max_journalsize=10000000, - max_run_length=10}, + StartOpts1 = [{root_path, RootPath}, + {max_journalsize, 10000000}, + {max_run_length, 10}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), @@ -146,30 +149,6 @@ journal_compaction_bustedjournal(_Config) -> true, lists:seq(1, 15)), - ObjList3 = testutil:generate_objects(15000, 50002), - ObjList4 = testutil:generate_objects(15000, 50002), - lists:foreach(fun({_RN, Obj, Spc}) -> - leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, - ObjList3), - %% Now replace all the objects - lists:foreach(fun({_RN, Obj, Spc}) -> - leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, - ObjList4), - - ok = leveled_bookie:book_compactjournal(Bookie2, 30000), - lists:foldl(fun(X, Pending) -> - case Pending of - false -> - false; - true -> - io:format("Loop ~w waiting for journal " - ++ "compaction to complete~n", [X]), - timer:sleep(20000), - F(Bookie2) - end end, - true, - lists:seq(1, 15)), - ok = leveled_bookie:book_close(Bookie2), testutil:reset_filestructure(10000). @@ -237,7 +216,7 @@ rotating_object_check(BookOpts, B, NumberOfObjects) -> restart_from_blankledger(BookOpts, B_SpcL) -> - leveled_penciller:clean_testdir(BookOpts#bookie_options.root_path ++ + leveled_penciller:clean_testdir(proplists:get_value(root_path, BookOpts) ++ "/ledger"), {ok, Book1} = leveled_bookie:book_start(BookOpts), io:format("Checking index following restart~n"), diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index 3a7585c..1c2536f 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -365,9 +365,9 @@ put_altered_indexed_objects(Book, Bucket, KSpecL, RemoveOld2i) -> {RplKSpecL, V}. rotating_object_check(RootPath, B, NumberOfObjects) -> - BookOpts = #bookie_options{root_path=RootPath, - cache_size=1000, - max_journalsize=5000000}, + BookOpts = [{root_path, RootPath}, + {cache_size, 1000}, + {max_journalsize, 5000000}], {ok, Book1} = leveled_bookie:book_start(BookOpts), {KSpcL1, V1} = testutil:put_indexed_objects(Book1, B, NumberOfObjects), ok = testutil:check_indexed_objects(Book1, B, KSpcL1, V1),