2016-10-31 20:58:19 +00:00
|
|
|
-module(recovery_SUITE).
|
2016-10-27 00:57:19 +01:00
|
|
|
-include_lib("common_test/include/ct.hrl").
|
|
|
|
|
-include("include/leveled.hrl").
|
|
|
|
|
-export([all/0]).
|
2016-10-31 20:58:19 +00:00
|
|
|
-export([retain_strategy/1,
|
2016-11-09 16:27:47 +00:00
|
|
|
recovr_strategy/1,
|
2016-11-01 00:46:14 +00:00
|
|
|
aae_bustedjournal/1,
|
|
|
|
|
journal_compaction_bustedjournal/1
|
2016-10-27 00:57:19 +01:00
|
|
|
]).
|
|
|
|
|
|
|
|
|
|
all() -> [
|
2016-11-09 22:06:02 +00:00
|
|
|
retain_strategy,
|
|
|
|
|
recovr_strategy,
|
|
|
|
|
aae_bustedjournal,
|
|
|
|
|
journal_compaction_bustedjournal
|
2016-10-27 00:57:19 +01:00
|
|
|
].
|
|
|
|
|
|
|
|
|
|
retain_strategy(_Config) ->
|
|
|
|
|
RootPath = testutil:reset_filestructure(),
|
2016-11-02 12:58:27 +00:00
|
|
|
BookOpts = [{root_path, RootPath},
|
|
|
|
|
{cache_size, 1000},
|
|
|
|
|
{max_journalsize, 5000000},
|
|
|
|
|
{reload_strategy, [{?RIAK_TAG, retain}]}],
|
|
|
|
|
BookOptsAlt = [{root_path, RootPath},
|
|
|
|
|
{cache_size, 1000},
|
|
|
|
|
{max_journalsize, 100000},
|
|
|
|
|
{reload_strategy, [{?RIAK_TAG, retain}]},
|
|
|
|
|
{max_run_length, 8}],
|
2016-10-27 00:57:19 +01:00
|
|
|
{ok, Spcl3, LastV3} = rotating_object_check(BookOpts, "Bucket3", 800),
|
|
|
|
|
ok = restart_from_blankledger(BookOpts, [{"Bucket3", Spcl3, LastV3}]),
|
|
|
|
|
{ok, Spcl4, LastV4} = rotating_object_check(BookOpts, "Bucket4", 1600),
|
|
|
|
|
ok = restart_from_blankledger(BookOpts, [{"Bucket3", Spcl3, LastV3},
|
|
|
|
|
{"Bucket4", Spcl4, LastV4}]),
|
|
|
|
|
{ok, Spcl5, LastV5} = rotating_object_check(BookOpts, "Bucket5", 3200),
|
2016-10-29 00:52:49 +01:00
|
|
|
ok = restart_from_blankledger(BookOptsAlt, [{"Bucket3", Spcl3, LastV3},
|
2016-10-27 00:57:19 +01:00
|
|
|
{"Bucket5", Spcl5, LastV5}]),
|
|
|
|
|
{ok, Spcl6, LastV6} = rotating_object_check(BookOpts, "Bucket6", 6400),
|
|
|
|
|
ok = restart_from_blankledger(BookOpts, [{"Bucket3", Spcl3, LastV3},
|
|
|
|
|
{"Bucket4", Spcl4, LastV4},
|
|
|
|
|
{"Bucket5", Spcl5, LastV5},
|
|
|
|
|
{"Bucket6", Spcl6, LastV6}]),
|
|
|
|
|
testutil:reset_filestructure().
|
|
|
|
|
|
|
|
|
|
|
2016-11-09 16:27:47 +00:00
|
|
|
recovr_strategy(_Config) ->
|
|
|
|
|
RootPath = testutil:reset_filestructure(),
|
|
|
|
|
BookOpts = [{root_path, RootPath},
|
|
|
|
|
{cache_size, 1000},
|
|
|
|
|
{max_journalsize, 5000000},
|
|
|
|
|
{reload_strategy, [{?RIAK_TAG, recovr}]}],
|
|
|
|
|
|
|
|
|
|
R6 = rotating_object_check(BookOpts, "Bucket6", 6400),
|
|
|
|
|
{ok, AllSpcL, V4} = R6,
|
|
|
|
|
leveled_penciller:clean_testdir(proplists:get_value(root_path, BookOpts) ++
|
|
|
|
|
"/ledger"),
|
|
|
|
|
{ok, Book1} = leveled_bookie:book_start(BookOpts),
|
|
|
|
|
|
2016-11-09 20:03:44 +00:00
|
|
|
{TestObject, TestSpec} = testutil:generate_testobject(),
|
|
|
|
|
ok = testutil:book_riakput(Book1, TestObject, TestSpec),
|
|
|
|
|
ok = testutil:book_riakdelete(Book1,
|
|
|
|
|
TestObject#r_object.bucket,
|
|
|
|
|
TestObject#r_object.key,
|
|
|
|
|
[]),
|
|
|
|
|
|
2016-11-09 16:27:47 +00:00
|
|
|
lists:foreach(fun({K, _SpcL}) ->
|
|
|
|
|
{ok, OH} = testutil:book_riakhead(Book1, "Bucket6", K),
|
|
|
|
|
K = OH#r_object.key,
|
|
|
|
|
{ok, OG} = testutil:book_riakget(Book1, "Bucket6", K),
|
|
|
|
|
V = testutil:get_value(OG),
|
|
|
|
|
true = V == V4
|
|
|
|
|
end,
|
|
|
|
|
lists:nthtail(6400, AllSpcL)),
|
2016-11-18 11:53:14 +00:00
|
|
|
Q = fun(RT) -> {index_query,
|
|
|
|
|
"Bucket6",
|
|
|
|
|
{fun testutil:foldkeysfun/3, []},
|
|
|
|
|
{"idx1_bin", "#", "~"},
|
|
|
|
|
{RT, undefined}}
|
|
|
|
|
end,
|
|
|
|
|
{async, TFolder} = leveled_bookie:book_returnfolder(Book1, Q(true)),
|
2016-11-09 16:27:47 +00:00
|
|
|
KeyTermList = TFolder(),
|
2016-11-18 11:53:14 +00:00
|
|
|
{async, KFolder} = leveled_bookie:book_returnfolder(Book1, Q(false)),
|
2016-11-09 16:27:47 +00:00
|
|
|
KeyList = lists:usort(KFolder()),
|
|
|
|
|
io:format("KeyList ~w KeyTermList ~w~n",
|
|
|
|
|
[length(KeyList), length(KeyTermList)]),
|
|
|
|
|
true = length(KeyList) == 6400,
|
|
|
|
|
true = length(KeyList) < length(KeyTermList),
|
|
|
|
|
true = length(KeyTermList) < 25600.
|
|
|
|
|
|
2016-10-27 00:57:19 +01:00
|
|
|
|
2016-10-31 20:58:19 +00:00
|
|
|
aae_bustedjournal(_Config) ->
|
|
|
|
|
RootPath = testutil:reset_filestructure(),
|
2016-11-02 12:58:27 +00:00
|
|
|
StartOpts = [{root_path, RootPath},
|
|
|
|
|
{max_journalsize, 20000000}],
|
2016-10-31 20:58:19 +00:00
|
|
|
{ok, Bookie1} = leveled_bookie:book_start(StartOpts),
|
|
|
|
|
{TestObject, TestSpec} = testutil:generate_testobject(),
|
2016-11-07 10:11:57 +00:00
|
|
|
ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
|
2016-10-31 20:58:19 +00:00
|
|
|
testutil:check_forobject(Bookie1, TestObject),
|
|
|
|
|
GenList = [2],
|
|
|
|
|
_CLs = testutil:load_objects(20000, GenList, Bookie1, TestObject,
|
|
|
|
|
fun testutil:generate_objects/2),
|
|
|
|
|
ok = leveled_bookie:book_close(Bookie1),
|
2016-11-01 00:46:14 +00:00
|
|
|
CDBFiles = testutil:find_journals(RootPath),
|
2016-10-31 20:58:19 +00:00
|
|
|
[HeadF|_Rest] = CDBFiles,
|
|
|
|
|
io:format("Selected Journal for corruption of ~s~n", [HeadF]),
|
2016-11-03 12:11:50 +00:00
|
|
|
testutil:corrupt_journal(RootPath, HeadF, 1000, 2048, 1000),
|
2016-10-31 20:58:19 +00:00
|
|
|
{ok, Bookie2} = leveled_bookie:book_start(StartOpts),
|
|
|
|
|
|
2016-11-18 17:58:43 +00:00
|
|
|
FoldKeysFun = fun(B, K, Acc) -> Acc ++ [{B, K}] end,
|
|
|
|
|
AllKeyQuery = {keylist, o_rkv, {FoldKeysFun, []}},
|
2016-11-18 15:53:22 +00:00
|
|
|
{async, KeyF} = leveled_bookie:book_returnfolder(Bookie2, AllKeyQuery),
|
2016-10-31 20:58:19 +00:00
|
|
|
KeyList = KeyF(),
|
|
|
|
|
20001 = length(KeyList),
|
|
|
|
|
HeadCount = lists:foldl(fun({B, K}, Acc) ->
|
2016-11-07 10:11:57 +00:00
|
|
|
case testutil:book_riakhead(Bookie2,
|
2016-10-31 20:58:19 +00:00
|
|
|
B,
|
|
|
|
|
K) of
|
|
|
|
|
{ok, _} -> Acc + 1;
|
|
|
|
|
not_found -> Acc
|
|
|
|
|
end
|
|
|
|
|
end,
|
|
|
|
|
0,
|
|
|
|
|
KeyList),
|
|
|
|
|
20001 = HeadCount,
|
|
|
|
|
GetCount = lists:foldl(fun({B, K}, Acc) ->
|
2016-11-07 10:11:57 +00:00
|
|
|
case testutil:book_riakget(Bookie2,
|
2016-10-31 20:58:19 +00:00
|
|
|
B,
|
|
|
|
|
K) of
|
|
|
|
|
{ok, _} -> Acc + 1;
|
|
|
|
|
not_found -> Acc
|
|
|
|
|
end
|
|
|
|
|
end,
|
|
|
|
|
0,
|
|
|
|
|
KeyList),
|
|
|
|
|
true = GetCount > 19000,
|
|
|
|
|
true = GetCount < HeadCount,
|
|
|
|
|
|
2016-10-31 22:17:29 +00:00
|
|
|
{async, HashTreeF1} = leveled_bookie:book_returnfolder(Bookie2,
|
|
|
|
|
{hashtree_query,
|
|
|
|
|
?RIAK_TAG,
|
|
|
|
|
false}),
|
|
|
|
|
KeyHashList1 = HashTreeF1(),
|
|
|
|
|
20001 = length(KeyHashList1),
|
|
|
|
|
{async, HashTreeF2} = leveled_bookie:book_returnfolder(Bookie2,
|
|
|
|
|
{hashtree_query,
|
|
|
|
|
?RIAK_TAG,
|
|
|
|
|
check_presence}),
|
|
|
|
|
KeyHashList2 = HashTreeF2(),
|
|
|
|
|
% The file is still there, and the hashtree is not corrupted
|
|
|
|
|
KeyHashList2 = KeyHashList1,
|
|
|
|
|
% Will need to remove the file or corrupt the hashtree to get presence to
|
|
|
|
|
% fail
|
|
|
|
|
|
2016-11-04 11:01:37 +00:00
|
|
|
FoldObjectsFun = fun(B, K, V, Acc) -> [{B, K, testutil:riak_hash(V)}|Acc]
|
|
|
|
|
end,
|
2016-11-02 15:38:51 +00:00
|
|
|
SW = os:timestamp(),
|
|
|
|
|
{async, HashTreeF3} = leveled_bookie:book_returnfolder(Bookie2,
|
|
|
|
|
{foldobjects_allkeys,
|
|
|
|
|
?RIAK_TAG,
|
|
|
|
|
FoldObjectsFun}),
|
|
|
|
|
KeyHashList3 = HashTreeF3(),
|
|
|
|
|
|
|
|
|
|
true = length(KeyHashList3) > 19000,
|
|
|
|
|
true = length(KeyHashList3) < HeadCount,
|
|
|
|
|
Delta = length(lists:subtract(KeyHashList1, KeyHashList3)),
|
|
|
|
|
true = Delta < 1001,
|
|
|
|
|
io:format("Fetch of hashtree using fold objects took ~w microseconds" ++
|
|
|
|
|
" and found a Delta of ~w and an objects count of ~w~n",
|
|
|
|
|
[timer:now_diff(os:timestamp(), SW),
|
|
|
|
|
Delta,
|
|
|
|
|
length(KeyHashList3)]),
|
|
|
|
|
|
2016-10-31 20:58:19 +00:00
|
|
|
ok = leveled_bookie:book_close(Bookie2),
|
2016-11-03 12:11:50 +00:00
|
|
|
{ok, BytesCopied} = testutil:restore_file(RootPath, HeadF),
|
|
|
|
|
io:format("File restored is of size ~w~n", [BytesCopied]),
|
|
|
|
|
{ok, Bookie3} = leveled_bookie:book_start(StartOpts),
|
|
|
|
|
|
|
|
|
|
SW4 = os:timestamp(),
|
|
|
|
|
{async, HashTreeF4} = leveled_bookie:book_returnfolder(Bookie3,
|
|
|
|
|
{foldobjects_allkeys,
|
|
|
|
|
?RIAK_TAG,
|
|
|
|
|
FoldObjectsFun}),
|
|
|
|
|
KeyHashList4 = HashTreeF4(),
|
|
|
|
|
|
|
|
|
|
true = length(KeyHashList4) == 20001,
|
|
|
|
|
io:format("Fetch of hashtree using fold objects took ~w microseconds" ++
|
|
|
|
|
" and found an object count of ~w~n",
|
|
|
|
|
[timer:now_diff(os:timestamp(), SW4), length(KeyHashList4)]),
|
|
|
|
|
|
|
|
|
|
ok = leveled_bookie:book_close(Bookie3),
|
|
|
|
|
testutil:corrupt_journal(RootPath, HeadF, 500, BytesCopied - 8000, 14),
|
|
|
|
|
|
|
|
|
|
{ok, Bookie4} = leveled_bookie:book_start(StartOpts),
|
|
|
|
|
|
|
|
|
|
SW5 = os:timestamp(),
|
|
|
|
|
{async, HashTreeF5} = leveled_bookie:book_returnfolder(Bookie4,
|
|
|
|
|
{foldobjects_allkeys,
|
|
|
|
|
?RIAK_TAG,
|
|
|
|
|
FoldObjectsFun}),
|
|
|
|
|
KeyHashList5 = HashTreeF5(),
|
|
|
|
|
|
|
|
|
|
true = length(KeyHashList5) > 19000,
|
|
|
|
|
true = length(KeyHashList5) < HeadCount,
|
|
|
|
|
Delta5 = length(lists:subtract(KeyHashList1, KeyHashList5)),
|
|
|
|
|
true = Delta5 < 1001,
|
|
|
|
|
io:format("Fetch of hashtree using fold objects took ~w microseconds" ++
|
|
|
|
|
" and found a Delta of ~w and an objects count of ~w~n",
|
|
|
|
|
[timer:now_diff(os:timestamp(), SW5),
|
|
|
|
|
Delta5,
|
|
|
|
|
length(KeyHashList5)]),
|
|
|
|
|
|
|
|
|
|
{async, HashTreeF6} = leveled_bookie:book_returnfolder(Bookie4,
|
|
|
|
|
{hashtree_query,
|
|
|
|
|
?RIAK_TAG,
|
|
|
|
|
check_presence}),
|
|
|
|
|
KeyHashList6 = HashTreeF6(),
|
|
|
|
|
true = length(KeyHashList6) > 19000,
|
|
|
|
|
true = length(KeyHashList6) < HeadCount,
|
|
|
|
|
|
|
|
|
|
ok = leveled_bookie:book_close(Bookie4),
|
|
|
|
|
|
|
|
|
|
testutil:restore_topending(RootPath, HeadF),
|
|
|
|
|
|
|
|
|
|
{ok, Bookie5} = leveled_bookie:book_start(StartOpts),
|
|
|
|
|
|
|
|
|
|
SW6 = os:timestamp(),
|
|
|
|
|
{async, HashTreeF7} = leveled_bookie:book_returnfolder(Bookie5,
|
|
|
|
|
{foldobjects_allkeys,
|
|
|
|
|
?RIAK_TAG,
|
|
|
|
|
FoldObjectsFun}),
|
|
|
|
|
KeyHashList7 = HashTreeF7(),
|
|
|
|
|
|
|
|
|
|
true = length(KeyHashList7) == 20001,
|
|
|
|
|
io:format("Fetch of hashtree using fold objects took ~w microseconds" ++
|
|
|
|
|
" and found an object count of ~w~n",
|
|
|
|
|
[timer:now_diff(os:timestamp(), SW6), length(KeyHashList7)]),
|
|
|
|
|
|
|
|
|
|
ok = leveled_bookie:book_close(Bookie5),
|
2016-10-31 20:58:19 +00:00
|
|
|
testutil:reset_filestructure().
|
|
|
|
|
|
2016-11-03 12:11:50 +00:00
|
|
|
|
2016-11-01 00:46:14 +00:00
|
|
|
journal_compaction_bustedjournal(_Config) ->
|
|
|
|
|
% Simply confirms that none of this causes a crash
|
|
|
|
|
RootPath = testutil:reset_filestructure(),
|
2016-11-02 12:58:27 +00:00
|
|
|
StartOpts1 = [{root_path, RootPath},
|
|
|
|
|
{max_journalsize, 10000000},
|
|
|
|
|
{max_run_length, 10}],
|
2016-11-01 00:46:14 +00:00
|
|
|
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
|
|
|
|
|
{TestObject, TestSpec} = testutil:generate_testobject(),
|
2016-11-07 10:11:57 +00:00
|
|
|
ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
|
2016-11-01 00:46:14 +00:00
|
|
|
testutil:check_forobject(Bookie1, TestObject),
|
|
|
|
|
ObjList1 = testutil:generate_objects(50000, 2),
|
|
|
|
|
lists:foreach(fun({_RN, Obj, Spc}) ->
|
2016-11-07 10:11:57 +00:00
|
|
|
testutil:book_riakput(Bookie1, Obj, Spc) end,
|
2016-11-01 00:46:14 +00:00
|
|
|
ObjList1),
|
|
|
|
|
%% Now replace all the objects
|
|
|
|
|
ObjList2 = testutil:generate_objects(50000, 2),
|
|
|
|
|
lists:foreach(fun({_RN, Obj, Spc}) ->
|
2016-11-07 10:11:57 +00:00
|
|
|
testutil:book_riakput(Bookie1, Obj, Spc) end,
|
2016-11-01 00:46:14 +00:00
|
|
|
ObjList2),
|
|
|
|
|
ok = leveled_bookie:book_close(Bookie1),
|
|
|
|
|
|
|
|
|
|
CDBFiles = testutil:find_journals(RootPath),
|
2016-11-03 13:34:06 +00:00
|
|
|
lists:foreach(fun(FN) ->
|
|
|
|
|
testutil:corrupt_journal(RootPath, FN, 100, 2048, 1000)
|
|
|
|
|
end,
|
2016-11-01 00:46:14 +00:00
|
|
|
CDBFiles),
|
|
|
|
|
|
|
|
|
|
{ok, Bookie2} = leveled_bookie:book_start(StartOpts1),
|
|
|
|
|
|
|
|
|
|
ok = leveled_bookie:book_compactjournal(Bookie2, 30000),
|
|
|
|
|
F = fun leveled_bookie:book_islastcompactionpending/1,
|
|
|
|
|
lists:foldl(fun(X, Pending) ->
|
|
|
|
|
case Pending of
|
|
|
|
|
false ->
|
|
|
|
|
false;
|
|
|
|
|
true ->
|
|
|
|
|
io:format("Loop ~w waiting for journal "
|
|
|
|
|
++ "compaction to complete~n", [X]),
|
|
|
|
|
timer:sleep(20000),
|
|
|
|
|
F(Bookie2)
|
|
|
|
|
end end,
|
|
|
|
|
true,
|
|
|
|
|
lists:seq(1, 15)),
|
|
|
|
|
|
|
|
|
|
ok = leveled_bookie:book_close(Bookie2),
|
|
|
|
|
testutil:reset_filestructure(10000).
|
|
|
|
|
|
|
|
|
|
|
2016-10-27 00:57:19 +01:00
|
|
|
rotating_object_check(BookOpts, B, NumberOfObjects) ->
|
|
|
|
|
{ok, Book1} = leveled_bookie:book_start(BookOpts),
|
|
|
|
|
{KSpcL1, V1} = testutil:put_indexed_objects(Book1, B, NumberOfObjects),
|
|
|
|
|
ok = testutil:check_indexed_objects(Book1,
|
|
|
|
|
B,
|
|
|
|
|
KSpcL1,
|
|
|
|
|
V1),
|
|
|
|
|
{KSpcL2, V2} = testutil:put_altered_indexed_objects(Book1,
|
|
|
|
|
B,
|
|
|
|
|
KSpcL1,
|
|
|
|
|
false),
|
|
|
|
|
ok = testutil:check_indexed_objects(Book1,
|
|
|
|
|
B,
|
|
|
|
|
KSpcL1 ++ KSpcL2,
|
|
|
|
|
V2),
|
|
|
|
|
{KSpcL3, V3} = testutil:put_altered_indexed_objects(Book1,
|
|
|
|
|
B,
|
|
|
|
|
KSpcL2,
|
|
|
|
|
false),
|
|
|
|
|
ok = leveled_bookie:book_close(Book1),
|
|
|
|
|
{ok, Book2} = leveled_bookie:book_start(BookOpts),
|
|
|
|
|
ok = testutil:check_indexed_objects(Book2,
|
|
|
|
|
B,
|
|
|
|
|
KSpcL1 ++ KSpcL2 ++ KSpcL3,
|
|
|
|
|
V3),
|
|
|
|
|
{KSpcL4, V4} = testutil:put_altered_indexed_objects(Book2,
|
|
|
|
|
B,
|
|
|
|
|
KSpcL3,
|
|
|
|
|
false),
|
|
|
|
|
io:format("Bucket complete - checking index before compaction~n"),
|
|
|
|
|
ok = testutil:check_indexed_objects(Book2,
|
|
|
|
|
B,
|
|
|
|
|
KSpcL1 ++ KSpcL2 ++ KSpcL3 ++ KSpcL4,
|
|
|
|
|
V4),
|
|
|
|
|
|
|
|
|
|
ok = leveled_bookie:book_compactjournal(Book2, 30000),
|
|
|
|
|
F = fun leveled_bookie:book_islastcompactionpending/1,
|
|
|
|
|
lists:foldl(fun(X, Pending) ->
|
|
|
|
|
case Pending of
|
|
|
|
|
false ->
|
|
|
|
|
false;
|
|
|
|
|
true ->
|
|
|
|
|
io:format("Loop ~w waiting for journal "
|
|
|
|
|
++ "compaction to complete~n", [X]),
|
|
|
|
|
timer:sleep(20000),
|
|
|
|
|
F(Book2)
|
|
|
|
|
end end,
|
|
|
|
|
true,
|
|
|
|
|
lists:seq(1, 15)),
|
|
|
|
|
io:format("Waiting for journal deletes~n"),
|
|
|
|
|
timer:sleep(20000),
|
|
|
|
|
|
|
|
|
|
io:format("Checking index following compaction~n"),
|
|
|
|
|
ok = testutil:check_indexed_objects(Book2,
|
|
|
|
|
B,
|
|
|
|
|
KSpcL1 ++ KSpcL2 ++ KSpcL3 ++ KSpcL4,
|
|
|
|
|
V4),
|
|
|
|
|
|
|
|
|
|
ok = leveled_bookie:book_close(Book2),
|
|
|
|
|
{ok, KSpcL1 ++ KSpcL2 ++ KSpcL3 ++ KSpcL4, V4}.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
restart_from_blankledger(BookOpts, B_SpcL) ->
|
2016-11-02 12:58:27 +00:00
|
|
|
leveled_penciller:clean_testdir(proplists:get_value(root_path, BookOpts) ++
|
2016-10-27 00:57:19 +01:00
|
|
|
"/ledger"),
|
|
|
|
|
{ok, Book1} = leveled_bookie:book_start(BookOpts),
|
|
|
|
|
io:format("Checking index following restart~n"),
|
|
|
|
|
lists:foreach(fun({B, SpcL, V}) ->
|
|
|
|
|
ok = testutil:check_indexed_objects(Book1, B, SpcL, V)
|
|
|
|
|
end,
|
|
|
|
|
B_SpcL),
|
|
|
|
|
ok = leveled_bookie:book_close(Book1),
|
|
|
|
|
ok.
|
