Merge pull request #295 from martinsumner/mas-i294-syncintest

Test improvements

src/leveled_cdb.erl

@@ -63,7 +63,6 @@
 -endif.
 
 -ifdef(slow_test).
--define(KEYCOUNT, 2048).
 -define(SPECIAL_DELFUN, fun(_F) -> ok end).
     % There are problems with the pendingdelete_test/0 in riak make test
     % The deletion of the file causes the process to crash and the test to
@@ -71,7 +70,6 @@
     % Workaround this problem by not performing the delete when running unit
     % tests in R16
 -else.
--define(KEYCOUNT, 16384).
 -define(SPECIAL_DELFUN, fun(F) -> file:delete(F) end).
 -endif.
 
@@ -2341,18 +2339,19 @@ get_keys_byposition_simple_test() ->
     ok = file:delete(F2).
 
 generate_sequentialkeys(0, KVList) ->
-    lists:reverse(KVList);
+    KVList;
 generate_sequentialkeys(Count, KVList) ->
     KV = {"Key" ++ integer_to_list(Count), "Value" ++ integer_to_list(Count)},
-    generate_sequentialkeys(Count - 1, KVList ++ [KV]).
+    generate_sequentialkeys(Count - 1, [KV|KVList]).
 
 get_keys_byposition_manykeys_test_() ->
     {timeout, 600, fun get_keys_byposition_manykeys_test_to/0}.
 
 get_keys_byposition_manykeys_test_to() ->
-    KeyCount = ?KEYCOUNT,
+    KeyCount = 16384,
     {ok, P1} = cdb_open_writer("test/test_area/poskeymany.pnd",
-                                #cdb_options{binary_mode=false}),
+                                #cdb_options{binary_mode=false,
+                                                sync_strategy=none}),
     KVList = generate_sequentialkeys(KeyCount, []),
     lists:foreach(fun({K, V}) -> cdb_put(P1, K, V) end, KVList),
     ok = cdb_roll(P1),

test/end_to_end/recovery_SUITE.erl

@@ -603,6 +603,7 @@ busted_journal_test(MaxJournalSize, PressMethod, PressPoint, Bust) ->
 
 allkeydelta_journal_multicompact(_Config) ->
     RootPath = testutil:reset_filestructure(),
+    CompPath = filename:join(RootPath, "journal/journal_files/post_compact"),
     B = <<"test_bucket">>,
     StartOptsFun = 
         fun(JOC) ->
@@ -621,14 +622,10 @@ allkeydelta_journal_multicompact(_Config) ->
                                                         false),
     compact_and_wait(Bookie1, 0),
     compact_and_wait(Bookie1, 0),
-    {ok, FileList1} = 
-        file:list_dir(
-            filename:join(RootPath, "journal/journal_files/post_compact")),
+    {ok, FileList1} =  file:list_dir(CompPath),
     io:format("Number of files after compaction ~w~n", [length(FileList1)]),
     compact_and_wait(Bookie1, 0),
-    {ok, FileList2} = 
-        file:list_dir(
-            filename:join(RootPath, "journal/journal_files/post_compact")),
+    {ok, FileList2} =  file:list_dir(CompPath),
     io:format("Number of files after compaction ~w~n", [length(FileList2)]),
     true = FileList1 == FileList2,
 
@@ -652,9 +649,7 @@ allkeydelta_journal_multicompact(_Config) ->
                                                         KSpcL2,
                                                         false),
     compact_and_wait(Bookie2, 0),
-    {ok, FileList3} = 
-        file:list_dir(
-            filename:join(RootPath, "journal/journal_files/post_compact")),
+    {ok, FileList3} = file:list_dir(CompPath),
     io:format("Number of files after compaction ~w~n", [length(FileList3)]),
 
     ok = leveled_bookie:book_close(Bookie2),
@@ -673,13 +668,25 @@ allkeydelta_journal_multicompact(_Config) ->
                                         B,
                                         KSpcL1 ++ KSpcL2 ++ KSpcL3 ++ KSpcL4,
                                         V4),
-    {ok, FileList4} = 
-        file:list_dir(
-            filename:join(RootPath, "journal/journal_files/post_compact")),
+    {ok, FileList4} = file:list_dir(CompPath),
     io:format("Number of files after compaction ~w~n", [length(FileList4)]),
-    true = length(FileList4) >= length(FileList3) + 3,
 
     ok = leveled_bookie:book_close(Bookie3),
+
+    NewlyCompactedFiles = lists:subtract(FileList4, FileList3),
+    true = length(NewlyCompactedFiles) >= 3,
+    CDBFilterFun = fun(_K, _V, _P, Acc, _EF) -> {loop, Acc + 1} end,
+    CheckLengthFun =
+        fun(FN) ->
+            {ok, CF} =
+                leveled_cdb:cdb_open_reader(filename:join(CompPath, FN)),
+            {_LP, TK} =
+                leveled_cdb:cdb_scan(CF, CDBFilterFun, 0, undefined),
+            io:format("File ~s has ~w keys~n", [FN, TK]),
+            true = TK =< 7000
+        end,
+    lists:foreach(CheckLengthFun, NewlyCompactedFiles),
+
     testutil:reset_filestructure(10000).
 
 recompact_keydeltas(_Config) ->