Startup work - baffled

Changes the stratup otpions to a prolist to make it easier to get
environment variables as default.

Tried application:start - and completely baffled as to how to get this
to work.

include/leveled.hrl

@@ -62,15 +62,6 @@
                         start_snapshot = false :: boolean(),
                         source_penciller :: pid()}).
 
--record(bookie_options,
-                       {root_path :: string(),
-                        cache_size :: integer(),
-                        max_journalsize :: integer(),
-                        max_pencillercachesize :: integer(),
-                        snapshot_bookie :: pid(),
-                        reload_strategy = [] :: list(),
-                        max_run_length :: integer()}).
-
 -record(iclerk_options,
                         {inker :: pid(),
                         max_run_length :: integer(),

src/eleveleddb.app.src

@@ -1,6 +1,6 @@
 {application, eleveleddb,
  [
-  {description, ""},
+  {description, "Key Value store based on LSM-Tree and designed for larger values"},
   {vsn, "1"},
   {registered, []},
   {applications, [
@@ -8,5 +8,5 @@
                   stdlib
                  ]},
   {mod, { eleveleddb_app, []}},
-  {env, []}
+  {env, [{root_path, "test"}]}
  ]}.

src/eleveleddb_sup.erl

@@ -16,7 +16,7 @@
 %% ===================================================================
 
 start_link() ->
-    supervisor:start_link({local, ?MODULE}, ?MODULE, []).
+    supervisor:start_link({local, leveled_bookie}, ?MODULE, []).
 
 %% ===================================================================
 %% Supervisor callbacks

src/leveled_bookie.erl

@@ -151,6 +151,9 @@
         book_islastcompactionpending/1,
         book_close/1]).
 
+-export([get_opt/2,
+            get_opt/3]).  
+
 -include_lib("eunit/include/eunit.hrl").
 
 -define(CACHE_SIZE, 2000).
@@ -175,9 +178,9 @@
 %%%============================================================================
 
 book_start(RootPath, LedgerCacheSize, JournalSize) ->
-    book_start(#bookie_options{root_path=RootPath,
-                                cache_size=LedgerCacheSize,
-                                max_journalsize=JournalSize}).
+    book_start([{root_path, RootPath},
+                    {cache_size, LedgerCacheSize},
+                    {max_journalsize, JournalSize}]).
 
 book_start(Opts) ->
     gen_server:start(?MODULE, [Opts], []).
@@ -247,17 +250,12 @@ book_close(Pid) ->
 %%%============================================================================
 
 init([Opts]) ->
-    case Opts#bookie_options.snapshot_bookie of
+    case get_opt(snapshot_bookie, Opts) of
         undefined ->
             % Start from file not snapshot
             {InkerOpts, PencillerOpts} = set_options(Opts),
             {Inker, Penciller} = startup(InkerOpts, PencillerOpts),
-            CacheSize = if
-                            Opts#bookie_options.cache_size == undefined ->
-                                ?CACHE_SIZE;
-                            true ->
-                                Opts#bookie_options.cache_size
-                        end,
+            CacheSize = get_opt(cache_size, Opts, ?CACHE_SIZE),
             io:format("Bookie starting with Pcl ~w Ink ~w~n",
                                                 [Penciller, Inker]),
             {ok, #state{inker=Inker,
@@ -551,21 +549,21 @@ shutdown_wait([TopPause|Rest], Inker) ->
     
 
 set_options(Opts) ->
-    MaxJournalSize = case Opts#bookie_options.max_journalsize of
-                            undefined ->
-                                30000;
-                            MS ->
-                                MS
-                        end,
+    MaxJournalSize = get_opt(max_journalsize, Opts, 10000000000),
     
-    AltStrategy = Opts#bookie_options.reload_strategy,
+    AltStrategy = get_opt(reload_strategy, Opts, []),
     ReloadStrategy = leveled_codec:inker_reload_strategy(AltStrategy),
-    PCLL0CacheSize = Opts#bookie_options.max_pencillercachesize,
-    JournalFP = Opts#bookie_options.root_path ++ "/" ++ ?JOURNAL_FP,
-    LedgerFP = Opts#bookie_options.root_path ++ "/" ++ ?LEDGER_FP,
+    
+    PCLL0CacheSize = get_opt(max_pencillercachesize, Opts),
+    RootPath = get_opt(root_path, Opts),
+    JournalFP = RootPath ++ "/" ++ ?JOURNAL_FP,
+    LedgerFP = RootPath ++ "/" ++ ?LEDGER_FP,
+    ok =filelib:ensure_dir(JournalFP),
+    ok =filelib:ensure_dir(LedgerFP),
+    
     {#inker_options{root_path = JournalFP,
                         reload_strategy = ReloadStrategy,
-                        max_run_length = Opts#bookie_options.max_run_length,
+                        max_run_length = get_opt(max_run_length, Opts),
                         cdb_options = #cdb_options{max_size=MaxJournalSize,
                                                     binary_mode=true}},
         #penciller_options{root_path = LedgerFP,
@@ -781,6 +779,23 @@ load_fun(KeyInLedger, ValueInLedger, _Position, Acc0, ExtractFun) ->
     end.
 
 
+get_opt(Key, Opts) ->
+    get_opt(Key, Opts, undefined).
+
+get_opt(Key, Opts, Default) ->
+    case proplists:get_value(Key, Opts) of
+        undefined ->
+            case application:get_env(?MODULE, Key) of
+                {ok, Value} ->
+                    Value;
+                undefined ->
+                    Default
+            end;
+        Value ->
+            Value
+    end.
+
+
 %%%============================================================================
 %%% Test
 %%%============================================================================
@@ -828,7 +843,7 @@ generate_multiple_robjects(Count, KeyNumber, ObjL) ->
 
 single_key_test() ->
     RootPath = reset_filestructure(),
-    {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath}),
+    {ok, Bookie1} = book_start([{root_path, RootPath}]),
     {B1, K1, V1, Spec1, MD} = {"Bucket1",
                                 "Key1",
                                 "Value1",
@@ -840,7 +855,7 @@ single_key_test() ->
     {ok, F1} = book_riakget(Bookie1, B1, K1),
     ?assertMatch(F1, Object),
     ok = book_close(Bookie1),
-    {ok, Bookie2} = book_start(#bookie_options{root_path=RootPath}),
+    {ok, Bookie2} = book_start([{root_path, RootPath}]),
     {ok, F2} = book_riakget(Bookie2, B1, K1),
     ?assertMatch(F2, Object),
     ok = book_close(Bookie2),
@@ -848,7 +863,7 @@ single_key_test() ->
 
 multi_key_test() ->
     RootPath = reset_filestructure(),
-    {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath}),
+    {ok, Bookie1} = book_start([{root_path, RootPath}]),
     {B1, K1, V1, Spec1, MD1} = {"Bucket",
                                 "Key1",
                                 "Value1",
@@ -885,7 +900,7 @@ multi_key_test() ->
     ?assertMatch(F2B, Obj2),
     ok = book_close(Bookie1),
     % Now reopen the file, and confirm that a fetch is still possible
-    {ok, Bookie2} = book_start(#bookie_options{root_path=RootPath}),
+    {ok, Bookie2} = book_start([{root_path, RootPath}]),
     {ok, F1C} = book_riakget(Bookie2, B1, K1),
     ?assertMatch(F1C, Obj1),
     {ok, F2C} = book_riakget(Bookie2, B2, K2),
@@ -904,7 +919,7 @@ multi_key_test() ->
     
 ttl_test() ->
     RootPath = reset_filestructure(),
-    {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath}),
+    {ok, Bookie1} = book_start([{root_path, RootPath}]),
     ObjL1 = generate_multiple_objects(100, 1),
     % Put in all the objects with a TTL in the future
     Future = leveled_codec:integer_now() + 300,
@@ -962,7 +977,7 @@ ttl_test() ->
     ?assertMatch(10, length(TermKeyList)),
     
     ok = book_close(Bookie1),
-    {ok, Bookie2} = book_start(#bookie_options{root_path=RootPath}),
+    {ok, Bookie2} = book_start([{root_path, RootPath}]),
     
     {async,
         IndexFolderTR2} = book_returnfolder(Bookie2,
@@ -987,9 +1002,9 @@ ttl_test() ->
 
 hashtree_query_test() ->
     RootPath = reset_filestructure(),
-    {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath,
-                                                max_journalsize=1000000,
-                                                cache_size=500}),
+    {ok, Bookie1} = book_start([{root_path, RootPath},
+                                {max_journalsize, 1000000},
+                                {cache_size, 500}]),
     ObjL1 = generate_multiple_objects(1200, 1),
     % Put in all the objects with a TTL in the future
     Future = leveled_codec:integer_now() + 300,
@@ -1019,9 +1034,9 @@ hashtree_query_test() ->
                         KeyHashList),
     ?assertMatch(1200, length(KeyHashList)),
     ok = book_close(Bookie1),
-    {ok, Bookie2} = book_start(#bookie_options{root_path=RootPath,
-                                                max_journalsize=200000,
-                                                cache_size=500}),
+    {ok, Bookie2} = book_start([{root_path, RootPath},
+                                    {max_journalsize, 200000},
+                                    {cache_size, 500}]),
     {async, HTFolder2} = book_returnfolder(Bookie2,
                                                 {hashtree_query,
                                                     ?STD_TAG,
@@ -1032,9 +1047,9 @@ hashtree_query_test() ->
 
 hashtree_query_withjournalcheck_test() ->
     RootPath = reset_filestructure(),
-    {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath,
-                                                max_journalsize=1000000,
-                                                cache_size=500}),
+    {ok, Bookie1} = book_start([{root_path, RootPath},
+                                    {max_journalsize, 1000000},
+                                    {cache_size, 500}]),
     ObjL1 = generate_multiple_objects(800, 1),
     % Put in all the objects with a TTL in the future
     Future = leveled_codec:integer_now() + 300,

test/end_to_end/basic_SUITE.erl

@@ -24,15 +24,15 @@ all() -> [
 
 simple_put_fetch_head_delete(_Config) ->
     RootPath = testutil:reset_filestructure(),
-    StartOpts1 = #bookie_options{root_path=RootPath},
+    StartOpts1 = [{root_path, RootPath}],
     {ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
     {TestObject, TestSpec} = testutil:generate_testobject(),
     ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec),
     testutil:check_forobject(Bookie1, TestObject),
     testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"),
     ok = leveled_bookie:book_close(Bookie1),
-    StartOpts2 = #bookie_options{root_path=RootPath,
-                                 max_journalsize=3000000},
+    StartOpts2 = [{root_path, RootPath},
+                    {max_journalsize, 3000000}],
     {ok, Bookie2} = leveled_bookie:book_start(StartOpts2),
     testutil:check_forobject(Bookie2, TestObject),
     ObjList1 = testutil:generate_objects(5000, 2),
@@ -69,16 +69,15 @@ simple_put_fetch_head_delete(_Config) ->
 
 many_put_fetch_head(_Config) ->
     RootPath = testutil:reset_filestructure(),
-    StartOpts1 = #bookie_options{root_path=RootPath,
-                                    max_pencillercachesize=16000},
+    StartOpts1 = [{root_path, RootPath}, {max_pencillercachesize, 16000}],
     {ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
     {TestObject, TestSpec} = testutil:generate_testobject(),
     ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec),
     testutil:check_forobject(Bookie1, TestObject),
     ok = leveled_bookie:book_close(Bookie1),
-    StartOpts2 = #bookie_options{root_path=RootPath,
-                                 max_journalsize=1000000000,
-                                 max_pencillercachesize=32000},
+    StartOpts2 = [{root_path, RootPath},
+                    {max_journalsize, 1000000000},
+                    {max_pencillercachesize, 32000}],
     {ok, Bookie2} = leveled_bookie:book_start(StartOpts2),
     testutil:check_forobject(Bookie2, TestObject),
     GenList = [2, 20002, 40002, 60002, 80002,
@@ -108,9 +107,9 @@ many_put_fetch_head(_Config) ->
 
 journal_compaction(_Config) ->
     RootPath = testutil:reset_filestructure(),
-    StartOpts1 = #bookie_options{root_path=RootPath,
-                                 max_journalsize=10000000,
-                                 max_run_length=1},
+    StartOpts1 = [{root_path, RootPath},
+                    {max_journalsize, 10000000},
+                    {max_run_length, 1}],
     {ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
     ok = leveled_bookie:book_compactjournal(Bookie1, 30000),
     {TestObject, TestSpec} = testutil:generate_testobject(),
@@ -172,7 +171,7 @@ journal_compaction(_Config) ->
 
 fetchput_snapshot(_Config) ->
     RootPath = testutil:reset_filestructure(),
-    StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=30000000},
+    StartOpts1 = [{root_path, RootPath}, {max_journalsize, 30000000}],
     {ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
     {TestObject, TestSpec} = testutil:generate_testobject(),
     ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec),
@@ -180,7 +179,7 @@ fetchput_snapshot(_Config) ->
     lists:foreach(fun({_RN, Obj, Spc}) ->
                         leveled_bookie:book_riakput(Bookie1, Obj, Spc) end,
                     ObjList1),
-    SnapOpts1 = #bookie_options{snapshot_bookie=Bookie1},
+    SnapOpts1 = [{snapshot_bookie, Bookie1}],
     {ok, SnapBookie1} = leveled_bookie:book_start(SnapOpts1),
     ChkList1 = lists:sublist(lists:sort(ObjList1), 100),
     testutil:check_forlist(Bookie1, ChkList1),
@@ -191,7 +190,7 @@ fetchput_snapshot(_Config) ->
     io:format("Closed initial bookies~n"),
     
     {ok, Bookie2} = leveled_bookie:book_start(StartOpts1),
-    SnapOpts2 = #bookie_options{snapshot_bookie=Bookie2},
+    SnapOpts2 = [{snapshot_bookie, Bookie2}],
     {ok, SnapBookie2} = leveled_bookie:book_start(SnapOpts2),
     io:format("Bookies restarted~n"),
     
@@ -279,7 +278,7 @@ load_and_count(_Config) ->
     % Use artificially small files, and the load keys, counting they're all
     % present
     RootPath = testutil:reset_filestructure(),
-    StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=50000000},
+    StartOpts1 = [{root_path, RootPath}, {max_journalsize, 50000000}],
     {ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
     {TestObject, TestSpec} = testutil:generate_testobject(),
     ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec),
@@ -362,7 +361,7 @@ load_and_count(_Config) ->
 
 load_and_count_withdelete(_Config) ->
     RootPath = testutil:reset_filestructure(),
-    StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=50000000},
+    StartOpts1 = [{root_path, RootPath}, {max_journalsize, 50000000}],
     {ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
     {TestObject, TestSpec} = testutil:generate_testobject(),
     ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec),
@@ -418,7 +417,7 @@ load_and_count_withdelete(_Config) ->
 
 space_clear_ondelete(_Config) ->
     RootPath = testutil:reset_filestructure(),
-    StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=20000000},
+    StartOpts1 = [{root_path, RootPath}, {max_journalsize, 20000000}],
     {ok, Book1} = leveled_bookie:book_start(StartOpts1),
     G2 = fun testutil:generate_compressibleobjects/2,
     testutil:load_objects(20000,

test/end_to_end/iterator_SUITE.erl

@@ -18,8 +18,8 @@ all() -> [
 
 simple_load_with2i(_Config) ->
     RootPath = testutil:reset_filestructure(),
-    StartOpts1 = #bookie_options{root_path=RootPath,
-                                    max_journalsize=50000000},
+    StartOpts1 = [{root_path, RootPath},
+                    {max_journalsize, 50000000}],
     {ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
     {TestObject, TestSpec} = testutil:generate_testobject(),
     ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec),

test/end_to_end/recovery_SUITE.erl

@@ -15,12 +15,15 @@ all() -> [
 
 retain_strategy(_Config) ->
     RootPath = testutil:reset_filestructure(),
-    BookOpts = #bookie_options{root_path=RootPath,
-                                cache_size=1000,
-                                max_journalsize=5000000,
-                                reload_strategy=[{?RIAK_TAG, retain}]},
-    BookOptsAlt = BookOpts#bookie_options{max_run_length=8,
-                                            max_journalsize=100000},
+    BookOpts = [{root_path, RootPath},
+                    {cache_size, 1000},
+                    {max_journalsize, 5000000},
+                    {reload_strategy, [{?RIAK_TAG, retain}]}],
+    BookOptsAlt = [{root_path, RootPath},
+                    {cache_size, 1000},
+                    {max_journalsize, 100000},
+                    {reload_strategy, [{?RIAK_TAG, retain}]},
+                    {max_run_length, 8}],
     {ok, Spcl3, LastV3} = rotating_object_check(BookOpts, "Bucket3", 800),
     ok = restart_from_blankledger(BookOpts, [{"Bucket3", Spcl3, LastV3}]),
     {ok, Spcl4, LastV4} = rotating_object_check(BookOpts, "Bucket4", 1600),
@@ -40,8 +43,8 @@ retain_strategy(_Config) ->
 
 aae_bustedjournal(_Config) ->
     RootPath = testutil:reset_filestructure(),
-    StartOpts = #bookie_options{root_path=RootPath,
-                                 max_journalsize=20000000},
+    StartOpts = [{root_path, RootPath},
+                    {max_journalsize, 20000000}],
     {ok, Bookie1} = leveled_bookie:book_start(StartOpts),
     {TestObject, TestSpec} = testutil:generate_testobject(),
     ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec),
@@ -107,9 +110,9 @@ aae_bustedjournal(_Config) ->
 journal_compaction_bustedjournal(_Config) ->
     % Simply confirms that none of this causes a crash
     RootPath = testutil:reset_filestructure(),
-    StartOpts1 = #bookie_options{root_path=RootPath,
-                                 max_journalsize=10000000,
-                                 max_run_length=10},
+    StartOpts1 = [{root_path, RootPath},
+                    {max_journalsize, 10000000},
+                    {max_run_length, 10}],
     {ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
     {TestObject, TestSpec} = testutil:generate_testobject(),
     ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec),
@@ -146,30 +149,6 @@ journal_compaction_bustedjournal(_Config) ->
                     true,
                     lists:seq(1, 15)),
     
-    ObjList3 = testutil:generate_objects(15000, 50002),
-    ObjList4 = testutil:generate_objects(15000, 50002),
-    lists:foreach(fun({_RN, Obj, Spc}) ->
-                        leveled_bookie:book_riakput(Bookie2, Obj, Spc) end,
-                    ObjList3),
-    %% Now replace all the objects
-    lists:foreach(fun({_RN, Obj, Spc}) ->
-                        leveled_bookie:book_riakput(Bookie2, Obj, Spc) end,
-                    ObjList4),
-    
-    ok = leveled_bookie:book_compactjournal(Bookie2, 30000),
-    lists:foldl(fun(X, Pending) ->
-                        case Pending of
-                            false ->
-                                false;
-                            true ->
-                                io:format("Loop ~w waiting for journal "
-                                    ++ "compaction to complete~n", [X]),
-                                timer:sleep(20000),
-                                F(Bookie2)
-                        end end,
-                    true,
-                    lists:seq(1, 15)),
-    
     ok = leveled_bookie:book_close(Bookie2),
     testutil:reset_filestructure(10000).
 
@@ -237,7 +216,7 @@ rotating_object_check(BookOpts, B, NumberOfObjects) ->
     
     
 restart_from_blankledger(BookOpts, B_SpcL) ->
-    leveled_penciller:clean_testdir(BookOpts#bookie_options.root_path ++
+    leveled_penciller:clean_testdir(proplists:get_value(root_path, BookOpts) ++
                                     "/ledger"),
     {ok, Book1} = leveled_bookie:book_start(BookOpts),
     io:format("Checking index following restart~n"),

test/end_to_end/testutil.erl

@@ -365,9 +365,9 @@ put_altered_indexed_objects(Book, Bucket, KSpecL, RemoveOld2i) ->
     {RplKSpecL, V}.
 
 rotating_object_check(RootPath, B, NumberOfObjects) ->
-    BookOpts = #bookie_options{root_path=RootPath,
-                                cache_size=1000,
-                                max_journalsize=5000000},
+    BookOpts = [{root_path, RootPath},
+                    {cache_size, 1000},
+                    {max_journalsize, 5000000}],
     {ok, Book1} = leveled_bookie:book_start(BookOpts),
     {KSpcL1, V1} = testutil:put_indexed_objects(Book1, B, NumberOfObjects),
     ok = testutil:check_indexed_objects(Book1, B, KSpcL1, V1),