diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index 50ab518..cf6e9ef 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -3,6 +3,7 @@ -include("include/leveled.hrl"). -export([all/0]). -export([hot_backup_simple/1, + hot_backup_changes/1, hot_backup_double/1, retain_strategy/1, recovr_strategy/1, @@ -13,6 +14,7 @@ all() -> [ hot_backup_simple, + hot_backup_changes, hot_backup_double, retain_strategy, recovr_strategy, @@ -78,6 +80,57 @@ hot_backup_double(_Config) -> ok = leveled_bookie:book_close(BookBackup), BackupPath = testutil:reset_filestructure("backup0"). +hot_backup_changes(_Config) -> + RootPath = testutil:reset_filestructure(), + BackupPath = testutil:reset_filestructure("backup0"), + BookOpts = [{root_path, RootPath}, + {cache_size, 1000}, + {max_journalsize, 10000000}, + {sync_strategy, testutil:sync_strategy()}], + B = "Bucket0", + + {ok, Book1} = leveled_bookie:book_start(BookOpts), + {KSpcL1, _V1} = testutil:put_indexed_objects(Book1, B, 20000), + + {async, BackupFun1} = leveled_bookie:book_hotbackup(Book1), + ok = BackupFun1(BackupPath), + {ok, FileList1} = + file:list_dir(filename:join(BackupPath, "journal/journal_files/")), + + {KSpcL2, V2} = testutil:put_altered_indexed_objects(Book1, B, KSpcL1), + + {async, BackupFun2} = leveled_bookie:book_hotbackup(Book1), + ok = BackupFun2(BackupPath), + {ok, FileList2} = + file:list_dir(filename:join(BackupPath, "journal/journal_files/")), + + ok = testutil:check_indexed_objects(Book1, B, KSpcL2, V2), + compact_and_wait(Book1), + + {async, BackupFun3} = leveled_bookie:book_hotbackup(Book1), + ok = BackupFun3(BackupPath), + {ok, FileList3} = + file:list_dir(filename:join(BackupPath, "journal/journal_files/")), + + true = length(FileList2) > length(FileList1), + true = length(FileList2) > length(FileList3), + + ok = leveled_bookie:book_close(Book1), + + RootPath = testutil:reset_filestructure(), + BookOptsBackup = [{root_path, BackupPath}, + {cache_size, 2000}, + {max_journalsize, 20000000}, + {sync_strategy, testutil:sync_strategy()}], + {ok, BookBackup} = leveled_bookie:book_start(BookOptsBackup), + + ok = testutil:check_indexed_objects(BookBackup, B, KSpcL2, V2), + + testutil:reset_filestructure("backup0"), + testutil:reset_filestructure(). + + + retain_strategy(_Config) -> RootPath = testutil:reset_filestructure(), BookOpts = [{root_path, RootPath}, @@ -480,22 +533,7 @@ rotating_object_check(BookOpts, B, NumberOfObjects) -> KSpcL1 ++ KSpcL2 ++ KSpcL3 ++ KSpcL4, V4), - ok = leveled_bookie:book_compactjournal(Book2, 30000), - F = fun leveled_bookie:book_islastcompactionpending/1, - lists:foldl(fun(X, Pending) -> - case Pending of - false -> - false; - true -> - io:format("Loop ~w waiting for journal " - ++ "compaction to complete~n", [X]), - timer:sleep(20000), - F(Book2) - end end, - true, - lists:seq(1, 15)), - io:format("Waiting for journal deletes~n"), - timer:sleep(20000), + compact_and_wait(Book2), io:format("Checking index following compaction~n"), ok = testutil:check_indexed_objects(Book2, @@ -505,8 +543,25 @@ rotating_object_check(BookOpts, B, NumberOfObjects) -> ok = leveled_bookie:book_close(Book2), {ok, KSpcL1 ++ KSpcL2 ++ KSpcL3 ++ KSpcL4, V4}. - - + +compact_and_wait(Book) -> + ok = leveled_bookie:book_compactjournal(Book, 30000), + F = fun leveled_bookie:book_islastcompactionpending/1, + lists:foldl(fun(X, Pending) -> + case Pending of + false -> + false; + true -> + io:format("Loop ~w waiting for journal " + ++ "compaction to complete~n", [X]), + timer:sleep(20000), + F(Book) + end end, + true, + lists:seq(1, 15)), + io:format("Waiting for journal deletes~n"), + timer:sleep(20000). + restart_from_blankledger(BookOpts, B_SpcL) -> leveled_penciller:clean_testdir(proplists:get_value(root_path, BookOpts) ++ "/ledger"),