Extend perf_SUITE (#434)

* Extend perf_SUITE

This is v6 of the perf_SUITE tests.  The test adds a complex index entry to every object, and then adds a new test phase to test regex queries.

There are three profiles added so the full, mini and profiling versions of perf_SUITE can be run without having to edit the file itself:

e.g. ./rebar3 as perf_mini do ct --suite=test/end_to_end/perf_SUITE

When testing as `perf_prof` summarised versions of the eprof results are now printed to screen.

The volume of keys within the full test suite has been dropped ... just to make life easier so that test run times are not excessively increase by the new features.

* Load chunk in spawned processes

Assume to make the job of gs easier - name makes a massive difference to load time in OTP 24.

* Correctly account for pause

alos try and improve test stability by increasing pause

* Add microstate accounting to profile

* Add memory tracking during test phases

Identify and log out memory usage by test phase

* Use macros instead (#437)

* Don't print memory to screen in standard ct test

---------

Co-authored-by: Thomas Arts <thomas.arts@quviq.com>

rebar.config

@@ -22,7 +22,10 @@
     {plugins, [rebar_eqc]}
    ]},
   {test, [{extra_src_dirs, ["test/end_to_end", "test/property"]}
-   ]}
+   ]},
+  {perf_full, [{erl_opts, [{d, performance, riak_fullperf}]}]},
+  {perf_mini, [{erl_opts, [{d, performance, riak_miniperf}]}]},
+  {perf_prof, [{erl_opts, [{d, performance, riak_profileperf}]}]}
  ]}.
 
 {deps, [

test/end_to_end/iterator_SUITE.erl

@@ -213,11 +213,12 @@ breaking_folds(_Config) ->
     % Find all keys index, and then same again but stop at a midpoint using a
     % throw
     {async, IdxFolder} =
-        leveled_bookie:book_indexfold(Bookie1,
-                                        list_to_binary("Bucket"), 
-                                        {fun testutil:foldkeysfun/3, []}, 
-                                        {"idx1_bin", "#", "|"},
-                                        {true, undefined}),
+        leveled_bookie:book_indexfold(
+            Bookie1,
+            list_to_binary("Bucket"), 
+            {fun testutil:foldkeysfun/3, []}, 
+            {<<"idx1_bin">>, <<"#">>, <<"|">>},
+            {true, undefined}),
     KeyList1 = lists:reverse(IdxFolder()),
     io:format("Index fold with result size ~w~n", [length(KeyList1)]),
     true = KeyCount == length(KeyList1),
@@ -235,11 +236,12 @@ breaking_folds(_Config) ->
             end
         end,
     {async, IdxFolderToMidK} =
-        leveled_bookie:book_indexfold(Bookie1,
-                                        list_to_binary("Bucket"), 
-                                        {FoldKeyThrowFun, []}, 
-                                        {"idx1_bin", "#", "|"},
-                                        {true, undefined}),
+        leveled_bookie:book_indexfold(
+            Bookie1,
+            list_to_binary("Bucket"), 
+            {FoldKeyThrowFun, []}, 
+            {<<"idx1_bin">>, <<"#">>, <<"|">>},
+            {true, undefined}),
     CatchingFold =
         fun(AsyncFolder) ->
             try
@@ -261,10 +263,8 @@ breaking_folds(_Config) ->
             [{K, Size}|Acc]
         end,
     {async, HeadFolder} = 
-        leveled_bookie:book_headfold(Bookie1,
-                                        ?RIAK_TAG, 
-                                        {HeadFoldFun, []}, 
-                                        true, true, false),
+        leveled_bookie:book_headfold(
+            Bookie1, ?RIAK_TAG,  {HeadFoldFun, []}, true, true, false),
     KeySizeList1 = lists:reverse(HeadFolder()),
     io:format("Head fold with result size ~w~n", [length(KeySizeList1)]),
     true = KeyCount == length(KeySizeList1),
@@ -472,11 +472,9 @@ small_load_with2i(_Config) ->
     testutil:check_forobject(Bookie1, TestObject),
     ObjectGen = testutil:get_compressiblevalue_andinteger(),
     IndexGen = testutil:get_randomindexes_generator(8),
-    ObjL1 = testutil:generate_objects(10000,
-                                        uuid,
-                                        [],
-                                        ObjectGen,
-                                        IndexGen),
+    ObjL1 =
+        testutil:generate_objects(
+            10000, uuid, [], ObjectGen, IndexGen),
     testutil:riakload(Bookie1, ObjL1),
     ChkList1 = lists:sublist(lists:sort(ObjL1), 100),
     testutil:check_forlist(Bookie1, ChkList1),
@@ -486,7 +484,7 @@ small_load_with2i(_Config) ->
     IdxQ1 = {index_query,
                 "Bucket",
                 {fun testutil:foldkeysfun/3, []},
-                {"idx1_bin", "#", "|"},
+                {<<"idx1_bin">>, <<"#">>, <<"|">>},
                 {true, undefined}},
     {async, IdxFolder} = leveled_bookie:book_returnfolder(Bookie1, IdxQ1),
     KeyList1 = lists:usort(IdxFolder()),
@@ -495,7 +493,7 @@ small_load_with2i(_Config) ->
     IdxQ2 = {index_query,
                 {"Bucket", LastKey},
                 {fun testutil:foldkeysfun/3, []},
-                {"idx1_bin", LastTerm, "|"},
+                {<<"idx1_bin">>, LastTerm, <<"|">>},
                 {false, undefined}},
     {async, IdxFolderLK} = leveled_bookie:book_returnfolder(Bookie1, IdxQ2),
     KeyList2 = lists:usort(IdxFolderLK()),
@@ -530,12 +528,14 @@ small_load_with2i(_Config) ->
                                                       {FoldObjectsFun, []},
                                                       false),
     KeyHashList2 = HTreeF2(),
-    {async, HTreeF3} = leveled_bookie:book_objectfold(Bookie1,
-                                                      ?RIAK_TAG,
-                                                      "Bucket",
-                                                      {"idx1_bin", "#", "|"},
-                                                      {FoldObjectsFun, []},
-                                                      false),
+    {async, HTreeF3} =
+        leveled_bookie:book_objectfold(
+            Bookie1,
+            ?RIAK_TAG,
+            "Bucket",
+            {<<"idx1_bin">>, <<"#">>, <<"|">>},
+            {FoldObjectsFun, []},
+            false),
     KeyHashList3 = HTreeF3(),
     true = 9901 == length(KeyHashList1), % also includes the test object
     true = 9900 == length(KeyHashList2),
@@ -585,96 +585,86 @@ small_load_with2i(_Config) ->
 
 query_count(_Config) ->
     RootPath = testutil:reset_filestructure(),
-    {ok, Book1} = leveled_bookie:book_start(RootPath,
-                                            2000,
-                                            50000000,
-                                            testutil:sync_strategy()),
+    {ok, Book1} =
+        leveled_bookie:book_start(
+            RootPath, 2000, 50000000, testutil:sync_strategy()),
     BucketBin = list_to_binary("Bucket"),
-    {TestObject, TestSpec} = testutil:generate_testobject(BucketBin,
-                                                            term_to_binary("Key1"),
-                                                            "Value1",
-                                                            [],
-                                                            [{"MDK1", "MDV1"}]),
+    {TestObject, TestSpec} =
+        testutil:generate_testobject(
+            BucketBin, term_to_binary("Key1"), "Value1", [], [{"MDK1", "MDV1"}]),
     ok = testutil:book_riakput(Book1, TestObject, TestSpec),
     testutil:check_forobject(Book1, TestObject),
     testutil:check_formissingobject(Book1, "Bucket1", "Key2"),
     testutil:check_forobject(Book1, TestObject),
-    lists:foreach(fun(_X) ->
-                        V = testutil:get_compressiblevalue(),
-                        Indexes = testutil:get_randomindexes_generator(8),
-                        SW = os:timestamp(),
-                        ObjL1 = testutil:generate_objects(10000,
-                                                            binary_uuid,
-                                                            [],
-                                                            V,
-                                                            Indexes),
-                        testutil:riakload(Book1, ObjL1),
-                        io:format("Put of 10000 objects with 8 index entries "
-                                        ++
-                                        "each completed in ~w microseconds~n",
-                                    [timer:now_diff(os:timestamp(), SW)])
-                        end,
-                        lists:seq(1, 8)),
+    lists:foreach(
+        fun(_X) ->
+            V = testutil:get_compressiblevalue(),
+            Indexes = testutil:get_randomindexes_generator(8),
+            SW = os:timestamp(),
+            ObjL1 = testutil:generate_objects(10000,
+                                                binary_uuid,
+                                                [],
+                                                V,
+                                                Indexes),
+            testutil:riakload(Book1, ObjL1),
+            io:format(
+                "Put of 10000 objects with 8 index entries "
+                "each completed in ~w microseconds~n",
+                [timer:now_diff(os:timestamp(), SW)])
+        end,
+        lists:seq(1, 8)),
     testutil:check_forobject(Book1, TestObject),
-    Total = lists:foldl(fun(X, Acc) ->
-                                IdxF = "idx" ++ integer_to_list(X) ++ "_bin",
-                                T = count_termsonindex(BucketBin,
-                                                        IdxF,
-                                                        Book1,
-                                                        ?KEY_ONLY),
-                                io:format("~w terms found on index ~s~n",
-                                            [T, IdxF]),
-                                Acc + T
-                                end,
-                            0,
-                            lists:seq(1, 8)),
-    ok = case Total of
-                640000 ->
-                    ok
+    Total =
+        lists:foldl(
+            fun(X, Acc) ->
+                IdxF = "idx" ++ integer_to_list(X) ++ "_bin",
+                T =
+                    count_termsonindex(
+                        BucketBin, list_to_binary(IdxF), Book1, ?KEY_ONLY),
+                io:format("~w terms found on index ~s~n", [T, IdxF]),
+                Acc + T
             end,
-    Index1Count = count_termsonindex(BucketBin,
-                                        "idx1_bin",
-                                        Book1,
-                                        ?KEY_ONLY),
+            0,
+            lists:seq(1, 8)),
+    true = Total == 640000,
+    Index1Count =
+        count_termsonindex(
+            BucketBin, <<"idx1_bin">>, Book1, ?KEY_ONLY),
     ok = leveled_bookie:book_close(Book1),
-    {ok, Book2} = leveled_bookie:book_start(RootPath,
-                                            1000,
-                                            50000000,
-                                            testutil:sync_strategy()),
-    Index1Count = count_termsonindex(BucketBin,
-                                        "idx1_bin",
-                                        Book2,
-                                        ?KEY_ONLY),
+    {ok, Book2} =
+        leveled_bookie:book_start(
+            RootPath, 1000, 50000000, testutil:sync_strategy()),
+    Index1Count =
+        count_termsonindex(
+            BucketBin, <<"idx1_bin">>, Book2, ?KEY_ONLY),
     NameList = testutil:name_list(),
-    TotalNameByName = lists:foldl(fun({_X, Name}, Acc) ->
-                                        {ok, Regex} = re:compile("[0-9]+" ++
-                                                                    Name),
-                                        SW = os:timestamp(),
-                                        T = count_termsonindex(BucketBin,
-                                                                "idx1_bin",
-                                                                Book2,
-                                                                {false,
-                                                                    Regex}),
-                                        TD = timer:now_diff(os:timestamp(),
-                                                                SW),
-                                        io:format("~w terms found on " ++
-                                                    "index idx1 with a " ++
-                                                    "regex in ~w " ++
-                                                    "microseconds~n",
-                                                    [T, TD]),
-                                        Acc + T
-                                        end,
-                                    0,
-                                    NameList),
-    ok = case TotalNameByName of
-                Index1Count ->
-                    ok
+    TotalNameByName =
+        lists:foldl(
+            fun({_X, Name}, Acc) ->
+                {ok, Regex} =
+                    re:compile("[0-9]+" ++ Name),
+                SW = os:timestamp(),
+                T =
+                    count_termsonindex(
+                        BucketBin,
+                        list_to_binary("idx1_bin"),
+                        Book2,
+                        {false, Regex}),
+                TD = timer:now_diff(os:timestamp(), SW),
+                io:format(
+                    "~w terms found on  index idx1 with a "
+                    "regex in ~w  microseconds~n",
+                    [T, TD]),
+                Acc + T
             end,
+            0,
+            NameList),
+    true = TotalNameByName == Index1Count,
     {ok, RegMia} = re:compile("[0-9]+Mia"),
     Query1 = {index_query,
                 BucketBin,
                 {fun testutil:foldkeysfun/3, []},
-                {"idx2_bin", "2000", "2000|"},
+                {<<"idx2_bin">>, <<"2000">>, <<"2000|">>},
                 {false, RegMia}},
     {async,
         Mia2KFolder1} = leveled_bookie:book_returnfolder(Book2, Query1),
@@ -682,7 +672,7 @@ query_count(_Config) ->
     Query2 = {index_query,
                 BucketBin,
                 {fun testutil:foldkeysfun/3, []},
-                {"idx2_bin", "2000", "2001"},
+                {<<"idx2_bin">>, <<"2000">>, <<"2001">>},
                 {true, undefined}},
     {async,
         Mia2KFolder2} = leveled_bookie:book_returnfolder(Book2, Query2),
@@ -705,7 +695,7 @@ query_count(_Config) ->
     Query3 = {index_query,
                 BucketBin,
                 {fun testutil:foldkeysfun/3, []},
-                {"idx2_bin", "1980", "2100"},
+                {<<"idx2_bin">>, <<"1980">>, <<"2100">>},
                 {false, RxMia2K}},
     {async,
         Mia2KFolder3} = leveled_bookie:book_returnfolder(Book2, Query3),
@@ -713,26 +703,26 @@ query_count(_Config) ->
     
     V9 = testutil:get_compressiblevalue(),
     Indexes9 = testutil:get_randomindexes_generator(8),
-    [{_RN, Obj9, Spc9}] = testutil:generate_objects(1,
-                                                    binary_uuid,
-                                                    [],
-                                                    V9,
-                                                    Indexes9),
+    [{_RN, Obj9, Spc9}] =
+        testutil:generate_objects(
+            1, binary_uuid, [], V9, Indexes9),
     ok = testutil:book_riakput(Book2, Obj9, Spc9),
-    R9 = lists:map(fun({add, IdxF, IdxT}) ->
-                        Q = {index_query,
-                                BucketBin,
-                                {fun testutil:foldkeysfun/3, []},
-                                {IdxF, IdxT, IdxT},
-                                ?KEY_ONLY},
-                        R = leveled_bookie:book_returnfolder(Book2, Q),
-                        {async, Fldr} = R,
-                        case length(Fldr()) of
-                            X when X > 0 ->
-                                {IdxF, IdxT, X}
-                        end
-                        end,
-                    Spc9),
+    R9 =
+        lists:map(
+            fun({add, IdxF, IdxT}) ->
+                Q = {index_query,
+                        BucketBin,
+                        {fun testutil:foldkeysfun/3, []},
+                        {IdxF, IdxT, IdxT},
+                        ?KEY_ONLY},
+                R = leveled_bookie:book_returnfolder(Book2, Q),
+                {async, Fldr} = R,
+                case length(Fldr()) of
+                    X when X > 0 ->
+                        {IdxF, IdxT, X}
+                end
+            end,
+            Spc9),
     Spc9Del = lists:map(fun({add, IdxF, IdxT}) -> {remove, IdxF, IdxT} end,
                         Spc9),
     ok = testutil:book_riakput(Book2, Obj9, Spc9Del),
@@ -751,44 +741,44 @@ query_count(_Config) ->
                         end,
                     R9),
     ok = leveled_bookie:book_close(Book2),
-    {ok, Book3} = leveled_bookie:book_start(RootPath,
-                                            2000,
-                                            50000000,
-                                            testutil:sync_strategy()),
-    lists:foreach(fun({IdxF, IdxT, X}) ->
-                        Q = {index_query,
-                                BucketBin,
-                                {fun testutil:foldkeysfun/3, []},
-                                {IdxF, IdxT, IdxT},
-                                ?KEY_ONLY},
-                        R = leveled_bookie:book_returnfolder(Book3, Q),
-                        {async, Fldr} = R,
-                        case length(Fldr()) of
-                            Y ->
-                                Y = X - 1
-                        end
-                        end,
-                    R9),
+    {ok, Book3} =
+        leveled_bookie:book_start(
+            RootPath, 2000, 50000000, testutil:sync_strategy()),
+    lists:foreach(
+        fun({IdxF, IdxT, X}) ->
+            Q = {index_query,
+                    BucketBin,
+                    {fun testutil:foldkeysfun/3, []},
+                    {IdxF, IdxT, IdxT},
+                    ?KEY_ONLY},
+            R = leveled_bookie:book_returnfolder(Book3, Q),
+            {async, Fldr} = R,
+            case length(Fldr()) of
+                Y ->
+                    Y = X - 1
+            end
+        end,
+        R9),
     ok = testutil:book_riakput(Book3, Obj9, Spc9),
     ok = leveled_bookie:book_close(Book3),
-    {ok, Book4} = leveled_bookie:book_start(RootPath,
-                                            2000,
-                                            50000000,
-                                            testutil:sync_strategy()),
-    lists:foreach(fun({IdxF, IdxT, X}) ->
-                        Q = {index_query,
-                                BucketBin,
-                                {fun testutil:foldkeysfun/3, []},
-                                {IdxF, IdxT, IdxT},
-                                ?KEY_ONLY},
-                        R = leveled_bookie:book_returnfolder(Book4, Q),
-                        {async, Fldr} = R,
-                        case length(Fldr()) of
-                            X ->
-                                ok
-                        end
-                        end,
-                    R9),
+    {ok, Book4} =
+        leveled_bookie:book_start(
+            RootPath, 2000, 50000000, testutil:sync_strategy()),
+    lists:foreach(
+        fun({IdxF, IdxT, X}) ->
+            Q = {index_query,
+                    BucketBin,
+                    {fun testutil:foldkeysfun/3, []},
+                    {IdxF, IdxT, IdxT},
+                    ?KEY_ONLY},
+            R = leveled_bookie:book_returnfolder(Book4, Q),
+            {async, Fldr} = R,
+            case length(Fldr()) of
+                X ->
+                    ok
+            end
+        end,
+        R9),
     testutil:check_forobject(Book4, TestObject),
     
     FoldBucketsFun = fun(B, Acc) -> sets:add_element(B, Acc) end,
@@ -803,24 +793,15 @@ query_count(_Config) ->
     
     true = sets:size(BucketSet1) == 1, 
     
-    ObjList10A = testutil:generate_objects(5000,
-                                            binary_uuid,
-                                            [],
-                                            V9,
-                                            Indexes9,
-                                            "BucketA"),
-    ObjList10B = testutil:generate_objects(5000,
-                                            binary_uuid,
-                                            [],
-                                            V9,
-                                            Indexes9,
-                                            "BucketB"),
-    ObjList10C = testutil:generate_objects(5000,
-                                            binary_uuid,
-                                            [],
-                                            V9,
-                                            Indexes9,
-                                            "BucketC"),
+    ObjList10A =
+        testutil:generate_objects(
+            5000, binary_uuid, [], V9, Indexes9, "BucketA"),
+    ObjList10B =
+        testutil:generate_objects(
+            5000, binary_uuid, [], V9, Indexes9, "BucketB"),
+    ObjList10C =
+        testutil:generate_objects(
+            5000, binary_uuid, [], V9, Indexes9, "BucketC"),
     testutil:riakload(Book4, ObjList10A),
     testutil:riakload(Book4, ObjList10B),
     testutil:riakload(Book4, ObjList10C),
@@ -849,29 +830,28 @@ query_count(_Config) ->
     testutil:reset_filestructure().
 
 
-
 count_termsonindex(Bucket, IdxField, Book, QType) ->
-    lists:foldl(fun(X, Acc) ->
-                        SW = os:timestamp(),
-                        ST = integer_to_list(X),
-                        ET = ST ++ "|",
-                        Q = {index_query,
-                                Bucket,
-                                {fun testutil:foldkeysfun/3, []},
-                                {IdxField, ST, ET},
-                                QType},
-                        R = leveled_bookie:book_returnfolder(Book, Q),
-                        {async, Folder} = R,
-                        Items = length(Folder()),
-                        io:format("2i query from term ~s on index ~s took " ++
-                                        "~w microseconds~n",
-                                    [ST,
-                                        IdxField,
-                                        timer:now_diff(os:timestamp(), SW)]),
-                        Acc + Items
-                        end,
-                    0,
-                    lists:seq(190, 221)).
+    lists:foldl(
+        fun(X, Acc) ->
+            SW = os:timestamp(),
+            ST = list_to_binary(integer_to_list(X)),
+            Pipe = <<"|">>,
+            ET = <<ST/binary, Pipe/binary>>,
+            Q = {index_query,
+                    Bucket,
+                    {fun testutil:foldkeysfun/3, []},
+                    {IdxField, ST, ET},
+                    QType},
+            R = leveled_bookie:book_returnfolder(Book, Q),
+            {async, Folder} = R,
+            Items = length(Folder()),
+            io:format(
+                "2i query from term ~s on index ~s took ~w microseconds~n",
+                [ST, IdxField, timer:now_diff(os:timestamp(), SW)]),
+            Acc + Items
+        end,
+        0,
+        lists:seq(190, 221)).
 
 multibucket_fold(_Config) ->
     RootPath = testutil:reset_filestructure(),

test/end_to_end/perf_SUITE.erl

@@ -3,10 +3,25 @@
 -define(INFO, info).
 -export([all/0, suite/0]).
 -export([
-    riak_ctperf/1, riak_fullperf/1, riak_profileperf/1
+    riak_ctperf/1, riak_fullperf/1, riak_profileperf/1, riak_miniperf/1
 ]).
 
-all() -> [riak_ctperf].
+-define(PEOPLE_INDEX, <<"people_bin">>).
+-define(MINI_QUERY_DIVISOR, 8).
+-define(RGEX_QUERY_DIVISOR, 32).
+
+-ifndef(performance).
+  -define(performance, riak_ctperf).
+-endif.
+all() -> [?performance].
+
+-if(?performance == riak_profileperf andalso ?OTP_RELEASE >= 24).
+   % Requires map functions from OTP 24
+   -define(ACCOUNTING, true).
+-else.
+   -define(ACCOUNTING, false).
+-endif.
+
 suite() -> [{timetrap, {hours, 16}}].
 
 riak_fullperf(_Config) ->
@@ -24,16 +39,16 @@ riak_fullperf(ObjSize, PM, LC) ->
     output_result(R5A),
     R5B = riak_load_tester(Bucket, 5000000, ObjSize, [], PM, LC),
     output_result(R5B),
-    R10 = riak_load_tester(Bucket, 10000000, ObjSize, [], PM, LC),
+    R10 = riak_load_tester(Bucket, 8000000, ObjSize, [], PM, LC),
     output_result(R10)
     .
 
 riak_profileperf(_Config) ->
     riak_load_tester(
         {<<"SensibleBucketTypeName">>, <<"SensibleBucketName0">>},
-        2000000,
+        1200000,
         2048,
-        [load, head, get, query, mini_query, full, guess, estimate, update],
+        [load, head, get, query, mini_query, regex_query, full, guess, estimate, update],
         zstd,
         as_store
     ).
@@ -42,12 +57,18 @@ riak_profileperf(_Config) ->
 riak_ctperf(_Config) ->
     riak_load_tester(<<"B0">>, 400000, 1024, [], native, as_store).
 
+riak_miniperf(_Config) ->
+    Bucket = {<<"SensibleBucketTypeName">>, <<"SensibleBucketName0">>},
+    R2A = riak_load_tester(Bucket, 2000000, 2048, [], zstd, as_store),
+    output_result(R2A).
+
 riak_load_tester(Bucket, KeyCount, ObjSize, ProfileList, PM, LC) ->
     ct:log(
         ?INFO,
         "Basic riak test with KeyCount ~w ObjSize ~w PressMethod ~w Ledger ~w",
         [KeyCount, ObjSize, PM, LC]
     ),
+
     IndexCount = 100000,
 
     GetFetches = KeyCount div 4,
@@ -74,6 +95,7 @@ riak_load_tester(Bucket, KeyCount, ObjSize, ProfileList, PM, LC) ->
                 IntIndex = "integer" ++ integer_to_list(ListID) ++ "_int",
                 BinIndex = "binary" ++ integer_to_list(ListID) ++ "_bin",
                 [{add, list_to_binary(IntIndex), RandInt},
+                {add, ?PEOPLE_INDEX, list_to_binary(random_people_index())},
                 {add, list_to_binary(IntIndex), RandInt + 1},
                 {add, list_to_binary(BinIndex), <<RandInt:32/integer>>},
                 {add, list_to_binary(BinIndex), <<(RandInt + 1):32/integer>>}]
@@ -82,6 +104,8 @@ riak_load_tester(Bucket, KeyCount, ObjSize, ProfileList, PM, LC) ->
 
     CountPerList = KeyCount div 10,
 
+    LoadMemoryTracker = memory_tracking(load, 1000),
+    LoadAccountant = accounting(load, 10000, ProfileList),
     TC4 = load_chunk(Bookie1, CountPerList, ObjSize, IndexGenFun, Bucket, 4),
     TC1 = load_chunk(Bookie1, CountPerList, ObjSize, IndexGenFun, Bucket, 1),
     TC9 = load_chunk(Bookie1, CountPerList, ObjSize, IndexGenFun, Bucket, 9),
@@ -92,6 +116,8 @@ riak_load_tester(Bucket, KeyCount, ObjSize, ProfileList, PM, LC) ->
     TC3 = load_chunk(Bookie1, CountPerList, ObjSize, IndexGenFun, Bucket, 3),
     TC7 = load_chunk(Bookie1, CountPerList, ObjSize, IndexGenFun, Bucket, 7),
     TC10 = load_chunk(Bookie1, CountPerList, ObjSize, IndexGenFun, Bucket, 10),
+    ok = stop_accounting(LoadAccountant),
+    {MT0, MP0, MB0} = stop_tracker(LoadMemoryTracker),
 
     ct:log(
         ?INFO,
@@ -104,20 +130,23 @@ riak_load_tester(Bucket, KeyCount, ObjSize, ProfileList, PM, LC) ->
         (TC1 + TC2 + TC3 + TC4 + TC5 + TC6 + TC7 + TC8 + TC9 + TC10) div 1000,
     ct:log(?INFO, "Total load time ~w ms", [TotalLoadTime]),
 
-    {MT0, MP0, MB0} = memory_usage(),
-
+    HeadMemoryTracker = memory_tracking(head, 1000),
+    HeadAccountant = accounting(head, 2000, ProfileList),
     TotalHeadTime = 
         random_fetches(head, Bookie1, Bucket, KeyCount, HeadFetches),
+    ok = stop_accounting(HeadAccountant),
+    {MT1, MP1, MB1} = stop_tracker(HeadMemoryTracker),
 
-    {MT1, MP1, MB1} = memory_usage(),
-
+    GetMemoryTracker = memory_tracking(get, 1000),
+    GetAccountant = accounting(get, 3000, ProfileList),
     TotalGetTime = 
         random_fetches(get, Bookie1, Bucket, KeyCount, GetFetches),
+    ok = stop_accounting(GetAccountant),
+    {MT2, MP2, MB2} = stop_tracker(GetMemoryTracker),
 
-    {MT2, MP2, MB2} = memory_usage(),
-
+    QueryMemoryTracker = memory_tracking(query, 1000),
+    QueryAccountant = accounting(query, 1000, ProfileList),
     QuerySize = max(10, IndexCount div 1000),
-    MiniQuerySize = max(1, IndexCount div 50000),
     TotalQueryTime =
         random_queries(
             Bookie1,
@@ -126,6 +155,12 @@ riak_load_tester(Bucket, KeyCount, ObjSize, ProfileList, PM, LC) ->
             IndexCount,
             QuerySize,
             IndexesReturned),
+    ok = stop_accounting(QueryAccountant),
+    {MT3a, MP3a, MB3a} = stop_tracker(QueryMemoryTracker),
+
+    MiniQueryMemoryTracker = memory_tracking(mini_query, 1000),
+    MiniQueryAccountant = accounting(mini_query, 1000, ProfileList),
+    MiniQuerySize = max(1, IndexCount div 50000),
     TotalMiniQueryTime =
         random_queries(
             Bookie1,
@@ -133,18 +168,76 @@ riak_load_tester(Bucket, KeyCount, ObjSize, ProfileList, PM, LC) ->
             10,
             IndexCount,
             MiniQuerySize,
-            IndexesReturned div 8),
+            IndexesReturned div ?MINI_QUERY_DIVISOR),
+    ok = stop_accounting(MiniQueryAccountant),
+    {MT3b, MP3b, MB3b} = stop_tracker(MiniQueryMemoryTracker),
 
-    {MT3, MP3, MB3} = memory_usage(),
+    RegexQueryMemoryTracker = memory_tracking(regex_query, 1000),
+    RegexQueryAccountant = accounting(regex_query, 2000, ProfileList),
+    RegexQueryTime =
+        random_people_queries(
+            Bookie1,
+            Bucket,
+            IndexesReturned div ?RGEX_QUERY_DIVISOR),
+    ok = stop_accounting(RegexQueryAccountant),
+    {MT3c, MP3c, MB3c} = stop_tracker(RegexQueryMemoryTracker),
 
-    {FullFoldTime, SegFoldTime} = size_estimate_summary(Bookie1),
+    GuessMemoryTracker = memory_tracking(guess, 1000),
+    GuessAccountant = accounting(guess, 1000, ProfileList),
+    {GuessTime, GuessCount} =
+        lists:foldl(
+            fun(_I, {TSAcc, CountAcc}) ->
+                {TS, Count} = counter(Bookie1, guess),
+                {TSAcc + TS, CountAcc + Count}
+            end,
+            {0, 0},
+            lists:seq(1, 60)
+        ),
+    ok = stop_accounting(GuessAccountant),
+    {MT4a, MP4a, MB4a} = stop_tracker(GuessMemoryTracker),
 
-    {MT4, MP4, MB4} = memory_usage(),
+    EstimateMemoryTracker = memory_tracking(estimate, 1000),
+    EstimateAccountant = accounting(estimate, 1000, ProfileList),
+    {EstimateTime, EstimateCount} =
+        lists:foldl(
+            fun(_I, {TSAcc, CountAcc}) ->
+                {TS, Count} = counter(Bookie1, estimate),
+                {TSAcc + TS, CountAcc + Count}
+            end,
+            {0, 0},
+            lists:seq(1, 40)
+        ),
+    ok = stop_accounting(EstimateAccountant),
+    {MT4b, MP4b, MB4b} = stop_tracker(EstimateMemoryTracker),
 
+    SegFoldTime = (GuessTime + EstimateTime) div 1000,
+    
+    FullFoldMemoryTracker = memory_tracking(full, 1000),
+    FullFoldAccountant = accounting(full, 2000, ProfileList),
+    {FullFoldTime, FullFoldCount} =
+        lists:foldl(
+            fun(_I, {TSAcc, CountAcc}) ->
+                {TS, Count} = counter(Bookie1, full),
+                {TSAcc + TS, CountAcc + Count}
+            end,
+            {0, 0},
+            lists:seq(1, 5)
+        ),
+    ok = stop_accounting(FullFoldAccountant),
+    {MT5, MP5, MB5} = stop_tracker(FullFoldMemoryTracker),
+
+    ct:log(
+        info,
+        "Guess size ~w Estimate size ~w Actual size ~w",
+        [GuessCount div 60, EstimateCount div 40, FullFoldCount div 10]
+    ),
+
+    UpdateMemoryTracker = memory_tracking(update, 1000),
+    UpdateAccountant = accounting(update, 1000, ProfileList),
     TotalUpdateTime =
         rotate_chunk(Bookie1, <<"UpdBucket">>, KeyCount div 50, ObjSize),
-
-    {MT5, MP5, MB5} = memory_usage(),
+    ok = stop_accounting(UpdateAccountant),
+    {MT6, MP6, MB6} = stop_tracker(UpdateMemoryTracker),
 
     DiskSpace = lists:nth(1, string:tokens(os:cmd("du -sh riakLoad"), "\t")),
     ct:log(?INFO, "Disk space taken by test ~s", [DiskSpace]),
@@ -172,8 +265,9 @@ riak_load_tester(Bucket, KeyCount, ObjSize, ProfileList, PM, LC) ->
                     P ->
                         P
                 end,
+            io:format(user, "~nProfile ~p:~n", [P]),
             ProFun = profile_fun(P0, ProfileData),
-            profile_test(Bookie1, ProFun)
+            profile_test(Bookie1, ProFun, P)
         end,
         ProfileList),
 
@@ -183,21 +277,26 @@ riak_load_tester(Bucket, KeyCount, ObjSize, ProfileList, PM, LC) ->
     {KeyCount, ObjSize, {PM, LC},
         TotalLoadTime,
         TotalHeadTime, TotalGetTime,
-        TotalQueryTime, TotalMiniQueryTime, FullFoldTime, SegFoldTime,
+        TotalQueryTime, TotalMiniQueryTime, RegexQueryTime,
+        FullFoldTime div 1000, SegFoldTime,
         TotalUpdateTime,
         DiskSpace,
-        {(MT0 + MT1 + MT2 + MT3 + MT4 + MT5) div 6000000,
-            (MP0 + MP1 + MP2 + MP3 + MP4 + MP5) div 6000000,
-            (MB0 + MB1 + MB2 + MB3 + MB4 + MB5) div 6000000},
+        {(MT0 + MT1 + MT2 + MT3a + MT3b + MT3c + MT4a + MT4b + MT5 + MT6)
+                div 9,
+            (MP0 + MP1 + MP2 + MP3a + MP3b + MP3c + MP4a + MP4b + MP5 + MP6)
+                div 9,
+            (MB0 + MB1 + MB2 + MB3a + MB3b + MB3c + MB4a + MB4b + MB5 + MB6)
+                div 9},
         SSTPids, CDBPids}.
 
-
-profile_test(Bookie, ProfileFun) ->
+profile_test(Bookie, ProfileFun, P) ->
     {Inker, Pcl, SSTPids, PClerk, CDBPids, IClerk} = get_pids(Bookie),
     TestPid = self(),
     profile_app(
         [TestPid, Bookie, Inker, IClerk, Pcl, PClerk] ++ SSTPids ++ CDBPids,
-        ProfileFun).
+        ProfileFun,
+        P
+    ).
 
 get_pids(Bookie) ->
     {ok, Inker, Pcl} = leveled_bookie:book_returnactors(Bookie),
@@ -211,7 +310,8 @@ output_result(
     {KeyCount, ObjSize, PressMethod,
         TotalLoadTime,
         TotalHeadTime, TotalGetTime,
-        TotalQueryTime, TotalMiniQueryTime, TotalFullFoldTime, TotalSegFoldTime,
+        TotalQueryTime, TotalMiniQueryTime, RegexQueryTime,
+        TotalFullFoldTime, TotalSegFoldTime,
         TotalUpdateTime,
         DiskSpace,
         {TotalMemoryMB, ProcessMemoryMB, BinaryMemoryMB},
@@ -227,6 +327,7 @@ output_result(
         "TotalGetTime - ~w ms~n"
         "TotalQueryTime - ~w ms~n"
         "TotalMiniQueryTime - ~w ms~n"
+        "TotalRegexQueryTime - ~w ms~n"
         "TotalFullFoldTime - ~w ms~n"
         "TotalAAEFoldTime - ~w ms~n"
         "TotalUpdateTime - ~w ms~n"
@@ -236,7 +337,8 @@ output_result(
         "Closing count of CDB Files - ~w~n",
         [KeyCount, ObjSize, PressMethod,
             TotalLoadTime, TotalHeadTime, TotalGetTime,
-            TotalQueryTime, TotalMiniQueryTime, TotalFullFoldTime, TotalSegFoldTime,
+            TotalQueryTime, TotalMiniQueryTime, RegexQueryTime,
+            TotalFullFoldTime, TotalSegFoldTime,
             TotalUpdateTime,
             DiskSpace,
             TotalMemoryMB, ProcessMemoryMB, BinaryMemoryMB,
@@ -244,13 +346,12 @@ output_result(
     ).
 
 memory_usage() ->
-    garbage_collect(), % GC the test process
     MemoryUsage = erlang:memory(),
     {element(2, lists:keyfind(total, 1, MemoryUsage)),
         element(2, lists:keyfind(processes, 1, MemoryUsage)),
         element(2, lists:keyfind(binary, 1, MemoryUsage))}.
 
-profile_app(Pids, ProfiledFun) ->
+profile_app(Pids, ProfiledFun, P) ->
 
     eprof:start(),
     eprof:start_profiling(Pids),
@@ -258,40 +359,12 @@ profile_app(Pids, ProfiledFun) ->
     ProfiledFun(),
 
     eprof:stop_profiling(),
-    eprof:analyze(total),
-    eprof:stop().
-
-size_estimate_summary(Bookie) ->
-    Loops = 10,
-    ct:log(
-        ?INFO,
-        "Size Estimate Tester (SET) started with Loops ~w",
-        [Loops]
-    ),
-    {{TotalGuessTime, TotalEstimateTime, TotalCountTime}, 
-            {TotalEstimateVariance, TotalGuessVariance}} =
-        lists:foldl(
-            fun(_I, {{GT, ET, CT}, {AET, AGT}}) ->
-                {{GT0, ET0, CT0}, {AE0, AG0}} = size_estimate_tester(Bookie),
-                {{GT + GT0, ET + ET0, CT + CT0}, {AET + AE0, AGT + AG0}}
-            end,
-            {{0, 0, 0}, {0, 0}},
-            lists:seq(1, Loops)
-        ),
-    ct:log(
-        ?INFO,
-        "SET: MeanGuess ~w ms MeanEstimate ~w ms MeanCount ~w ms",
-        [TotalGuessTime div 10000,
-            TotalEstimateTime div 10000,
-            TotalCountTime div 10000]
-    ),
-    ct:log(
-        ?INFO,
-        "Mean variance in Estimate ~w Guess ~w",
-        [TotalEstimateVariance div Loops, TotalGuessVariance div Loops]
-    ),
-    %% Assume that segment-list folds are 10 * as common as all folds
-    {TotalCountTime div 1000, (TotalGuessTime + TotalEstimateTime) div 1000}.
+    eprof:log(atom_to_list(P) ++ ".log"),
+    eprof:analyze(total, [{filter, [{time, 150000}]}]),
+    eprof:stop(),
+    {ok, Analysis} = file:read_file(atom_to_list(P) ++ ".log"),
+    io:format(user, "~n~s~n", [Analysis])
+    .
 
 
 rotate_chunk(Bookie, Bucket, KeyCount, ObjSize) ->
@@ -311,15 +384,6 @@ rotate_chunk(Bookie, Bucket, KeyCount, ObjSize) ->
             end),
     TC div 1000.
 
-load_chunk(Bookie, CountPerList, ObjSize, IndexGenFun, Bucket, Chunk) ->
-    ct:log(?INFO, "Generating and loading ObjList ~w", [Chunk]),
-    ObjList =
-        generate_chunk(CountPerList, ObjSize, IndexGenFun, Bucket, Chunk),
-    {TC, ok} = timer:tc(fun() -> testutil:riakload(Bookie, ObjList) end),
-    garbage_collect(),
-    timer:sleep(2000),
-    TC.
-
 generate_chunk(CountPerList, ObjSize, IndexGenFun, Bucket, Chunk) ->
     testutil:generate_objects(
         CountPerList, 
@@ -329,31 +393,60 @@ generate_chunk(CountPerList, ObjSize, IndexGenFun, Bucket, Chunk) ->
         Bucket
     ).
 
-size_estimate_tester(Bookie) ->
-    %% Data size test - calculate data size, then estimate data size
-    {CountTS, Count} = counter(Bookie, full),
-    {CountTSEstimate, CountEstimate} = counter(Bookie, estimate),
-    {CountTSGuess, CountGuess} = counter(Bookie, guess),
-    {GuessTolerance, EstimateTolerance} =
-        case Count of
-            C when C < 500000 ->
-                {0.20, 0.15};
-            C when C < 1000000 ->
-                {0.12, 0.1};
-            C when C < 2000000 ->
-                {0.1, 0.08};
-            _C ->
-                {0.08, 0.05}
-        end,
+load_chunk(Bookie, CountPerList, ObjSize, IndexGenFun, Bucket, Chunk) ->
+    ct:log(?INFO, "Generating and loading ObjList ~w", [Chunk]),
+    time_load_chunk(
+        Bookie,
+        {Bucket, base64:encode(leveled_rand:rand_bytes(ObjSize)), IndexGenFun(Chunk)},
+        (Chunk - 1) * CountPerList + 1,
+        Chunk * CountPerList,
+        0,
+        0
+    ).
 
-    true =
-        ((CountGuess / Count) > (1.0 - GuessTolerance))
-            and ((CountGuess / Count) < (1.0 + GuessTolerance)),
-    true =
-        ((CountEstimate / Count) > (1.0 - EstimateTolerance))
-            and ((CountEstimate / Count) < (1.0 + EstimateTolerance)),
-    {{CountTSGuess, CountTSEstimate, CountTS},
-        {abs(CountEstimate - Count), abs(CountGuess - Count)}}.
+time_load_chunk(
+        _Bookie, _ObjDetails, KeyNumber, TopKey, TotalTime, PC)
+        when KeyNumber > TopKey ->
+    garbage_collect(),
+    timer:sleep(2000),
+    ct:log(
+        ?INFO,
+        "Count of ~w pauses during chunk load",
+        [PC]
+    ),
+    TotalTime;
+time_load_chunk(
+        Bookie, {Bucket, Value, IndexGen}, KeyNumber, TopKey, TotalTime, PC) ->
+    ThisProcess = self(),
+    spawn(
+        fun() ->
+            {RiakObj, IndexSpecs} =
+                testutil:set_object(
+                    Bucket, testutil:fixed_bin_key(KeyNumber), Value, IndexGen, []),
+            {TC, R} =
+                timer:tc(
+                    testutil, book_riakput, [Bookie, RiakObj, IndexSpecs]
+                ),
+            case R of
+                ok ->
+                    ThisProcess! {TC, 0};
+                pause ->
+                    timer:sleep(40),
+                    ThisProcess ! {TC + 40000, 1}
+            end
+        end
+    ),
+    receive
+        {PutTime, Pause} ->
+            time_load_chunk(
+                Bookie, 
+                {Bucket, Value, IndexGen},
+                KeyNumber + 1,
+                TopKey,
+                TotalTime + PutTime,
+                PC + Pause
+            )
+    end.
 
 counter(Bookie, full) ->
     {async, DataSizeCounter} =
@@ -471,6 +564,42 @@ random_queries(Bookie, Bucket, IDs, IdxCnt, MaxRange, IndexesReturned) ->
     TC div 1000.
 
 
+random_people_queries(Bookie, Bucket, IndexesReturned) ->
+    SeventiesWillowRegex =
+        "[^\\|]*\\|197[0-9]{5}\\|[^\\|]*\\|"
+        "[^\\|]*#Willow[^\\|]*\\|[^\\|]*#LS[^\\|]*",
+        %% born in the 70s with Willow as a given name
+    QueryFun =
+        fun() ->
+            Surname = get_random_surname(),
+            Range =
+                {?PEOPLE_INDEX,
+                    Surname,
+                    <<Surname/binary, 126:8/integer>>
+            },
+            {ok, TermRegex} =
+                re:compile(SeventiesWillowRegex),
+            FoldKeysFun =  fun(_B, _K, Cnt) -> Cnt + 1 end,
+            {async, R} =
+                leveled_bookie:book_indexfold(
+                    Bookie,
+                    {Bucket, <<>>}, 
+                    {FoldKeysFun, 0},
+                    Range,
+                    {true, TermRegex}),
+            R()
+        end,
+    
+    {TC, {QC, EF}} =
+        timer:tc(fun() -> run_queries(QueryFun, 0, 0, IndexesReturned) end),
+    ct:log(
+        ?INFO,
+        "Fetch of ~w index entries by regex in ~w queries took ~w ms",
+        [EF, QC, TC div 1000]
+    ),
+    TC div 1000.
+
+
 run_queries(_QueryFun, QueryCount, EntriesFound, TargetEntries)
         when EntriesFound >= TargetEntries ->
     {QueryCount, EntriesFound};
@@ -486,7 +615,8 @@ profile_fun(
         {Bookie, Bucket, _KeyCount, _ObjSize, IndexCount, IndexesReturned}) ->
     fun() ->
         random_queries(
-            Bookie, Bucket, 10, IndexCount, QuerySize, IndexesReturned div 8)
+            Bookie, Bucket, 10, IndexCount, QuerySize,
+            IndexesReturned div ?MINI_QUERY_DIVISOR)
     end;
 profile_fun(
         {query, QuerySize},
@@ -495,6 +625,13 @@ profile_fun(
         random_queries(
             Bookie, Bucket, 10, IndexCount, QuerySize, IndexesReturned)
     end;
+profile_fun(
+        regex_query,
+        {Bookie, Bucket, _KeyCount, _ObjSize, _IndexCount, IndexesReturned}) ->
+    fun() ->
+        random_people_queries(
+            Bookie, Bucket, IndexesReturned div ?RGEX_QUERY_DIVISOR)
+    end;
 profile_fun(
         {head, HeadFetches},
         {Bookie, Bucket, KeyCount, _ObjSize, _IndexCount, _IndexesReturned}) ->
@@ -524,11 +661,230 @@ profile_fun(
 profile_fun(
         CounterFold,
         {Bookie, _Bucket, _KeyCount, _ObjSize, _IndexCount, _IndexesReturned}) ->
+    Runs =
+        case CounterFold of
+            full ->
+                20;
+            estimate ->
+                40;
+            guess ->
+                100
+        end,
     fun() ->
         lists:foreach(
             fun(_I) ->
                 _ = counter(Bookie, CounterFold)
             end,
-            lists:seq(1, 10)
+            lists:seq(1, Runs)
         )
     end.
+
+random_people_index() ->
+    io_lib:format(
+        "~s|~s|~s|#~s#~s#~s|#~s#~s#~s",
+        [get_random_surname(),
+            get_random_dob(),
+            get_random_dod(),
+            get_random_givenname(), get_random_givenname(), get_random_givenname(),
+            get_random_postcode(), get_random_postcode(), get_random_postcode()
+        ]
+    ).
+
+get_random_surname() ->
+    lists:nth(
+        rand:uniform(100),
+        [<<"Smith">>, <<"Jones">>, <<"Taylor">>, <<"Brown">>, <<"Williams">>,
+            <<"Wilson">>, <<"Johnson">>, <<"Davies">>, <<"Patel">>, <<"Robinson">>,
+            <<"Wright">>, <<"Thompson">>, <<"Evans">>, <<"Walker">>, <<"White">>,
+            <<"Roberts">>, <<"Green">>, <<"Hall">>, <<"Thomas">>, <<"Clarke">>,
+            <<"Jackson">>, <<"Wood">>, <<"Harris">>, <<"Edwards">>, <<"Turner">>,
+            <<"Martin">>, <<"Cooper">>, <<"Hill">>, <<"Ward">>, <<"Hughes">>,
+            <<"Moore">>, <<"Clark">>, <<"King">>, <<"Harrison">>, <<"Lewis">>,
+            <<"Baker">>, <<"Lee">>, <<"Allen">>, <<"Morris">>, <<"Khan">>,
+            <<"Scott">>, <<"Watson">>, <<"Davis">>, <<"Parker">>, <<"James">>,
+            <<"Bennett">>, <<"Young">>, <<"Phillips">>, <<"Richardson">>, <<"Mitchell">>,
+            <<"Bailey">>, <<"Carter">>, <<"Cook">>, <<"Singh">>, <<"Shaw">>,
+            <<"Bell">>, <<"Collins">>, <<"Morgan">>, <<"Kelly">>, <<"Begum">>,
+            <<"Miller">>, <<"Cox">>, <<"Hussain">>, <<"Marshall">>, <<"Simpson">>,
+            <<"Price">>, <<"Anderson">>, <<"Adams">>, <<"Wilkinson">>, <<"Ali">>,
+            <<"Ahmed">>, <<"Foster">>, <<"Ellis">>, <<"Murphy">>, <<"Chapman">>,
+            <<"Mason">>, <<"Gray">>, <<"Richards">>, <<"Webb">>, <<"Griffiths">>,
+            <<"Hunt">>, <<"Palmer">>, <<"Campbell">>, <<"Holmes">>, <<"Mills">>,
+            <<"Rogers">>, <<"Barnes">>, <<"Knight">>, <<"Matthews">>, <<"Barker">>,
+            <<"Powell">>, <<"Stevens">>, <<"Kaur">>, <<"Fisher">>, <<"Butler">>,
+            <<"Dixon">>, <<"Russell">>, <<"Harvey">>, <<"Pearson">>, <<"Graham">>]
+    ).
+
+get_random_givenname() ->
+    lists:nth(
+        rand:uniform(20),
+        [<<"Noah">>, <<"Oliver">>, <<"George">>, <<"Arthur">>, <<"Muhammad">>,
+            <<"Leo">>, <<"Harry">>, <<"Oscar">> , <<"Archie">>, <<"Henry">>,
+            <<"Olivia">>, <<"Amelia">>, <<"Isla">>, <<"Ava">>, <<"Ivy">>,
+            <<"Freya">>, <<"Lily">>, <<"Florence">>, <<"Mia">>, <<"Willow">>
+    ]).
+
+get_random_dob() ->
+    io_lib:format(
+        "~4..0B~2..0B~2..0B",
+        [1900 + rand:uniform(99), rand:uniform(12), rand:uniform(28)]
+    ).
+
+get_random_dod() ->
+    io_lib:format(
+        "~4..0B~2..0B~2..0B",
+        [2000 + rand:uniform(20), rand:uniform(12), rand:uniform(28)]
+    ).
+
+get_random_postcode() ->
+    io_lib:format(
+        "LS~w ~wXX", [rand:uniform(26), rand:uniform(9)]
+    ).
+
+
+memory_tracking(Phase, Timeout) ->
+    spawn(
+        fun() ->
+            memory_tracking(Phase, Timeout, {0, 0, 0}, 0)
+        end
+    ).
+
+memory_tracking(Phase, Timeout, {TAcc, PAcc, BAcc}, Loops) ->
+    receive
+        {stop, Caller} ->
+            {T, P, B} = memory_usage(),
+            TAvg = (T + TAcc) div ((Loops + 1) * 1000000),
+            PAvg = (P + PAcc) div ((Loops + 1) * 1000000),
+            BAvg = (B + BAcc) div ((Loops + 1) * 1000000),
+            print_memory_stats(Phase, TAvg, PAvg, BAvg),
+            Caller ! {TAvg, PAvg, BAvg}
+    after Timeout ->
+        {T, P, B} = memory_usage(),
+        memory_tracking(
+            Phase, Timeout, {TAcc + T, PAcc + P, BAcc + B}, Loops + 1)
+    end.
+
+
+-if(?performance == riak_ctperf).
+print_memory_stats(_Phase, _TAvg, _PAvg, _BAvg) ->
+    ok.
+-else.
+print_memory_stats(Phase, TAvg, PAvg, BAvg) ->
+    io:format(
+        user,
+        "~nFor ~w memory stats: total ~wMB process ~wMB binary ~wMB~n",
+        [Phase, TAvg, PAvg, BAvg]
+    ).
+-endif.
+
+dummy_accountant() ->
+    spawn(fun() -> receive {stop, Caller} -> Caller ! ok end end).
+    
+stop_accounting(Accountant) ->
+    Accountant ! {stop, self()},
+    receive ok -> ok end.
+
+stop_tracker(Tracker) ->
+    garbage_collect(),
+        % Garbage collect the test process, before getting the memory stats
+    Tracker ! {stop, self()},
+    receive MemStats -> MemStats end.
+
+-if(?ACCOUNTING).
+
+-define(ACCT_TYPES, [scheduler, dirty_io_scheduler, dirty_cpu_scheduler, aux]).
+
+accounting(Phase, Timeout, ProfileList) ->
+    case lists:member(Phase, ProfileList) of
+        true ->
+            ZeroCounters =
+                #{
+                    emulator => 0,
+                    aux => 0,
+                    check_io => 0,
+                    gc => 0,
+                    other => 0
+                },
+            InitCounters =
+                lists:map(fun(T) -> {T, ZeroCounters} end, ?ACCT_TYPES),
+            spawn(
+                fun() ->
+                    accounting(Phase, Timeout, maps:from_list(InitCounters), 0)
+                end
+            );
+        false ->
+            dummy_accountant()
+    end.
+
+accounting(Phase, Timeout, Counters, Loops) ->
+    receive
+        {stop, Caller} ->
+            io:format(
+                user,
+                "~n~nStats for Phase ~p after loops ~p:~n",
+                [Phase, Loops]
+            ),
+            lists:foreach(
+                fun(S) ->
+                    scheduler_output(S, maps:get(S, Counters))
+                end,
+                ?ACCT_TYPES
+            ),
+            Caller ! ok
+    after Timeout ->
+        msacc:start(Timeout div 5),
+        UpdCounters =
+            lists:foldl(
+                fun(StatMap, CountersAcc) ->
+                    Type = maps:get(type, StatMap),
+                    case lists:member(Type, ?ACCT_TYPES) of
+                        true ->
+                            TypeAcc =
+                                maps:intersect_with(
+                                    fun(_K, V1, V2) -> V1 + V2 end,
+                                    maps:get(counters, StatMap),
+                                    maps:get(Type, CountersAcc)
+                                ),
+                            maps:update(Type, TypeAcc, CountersAcc);
+                        false ->
+                            CountersAcc
+                    end
+                end,
+                Counters,
+                msacc:stats()
+            ),
+        accounting(Phase, Timeout, UpdCounters, Loops + 1)
+    end.
+
+scheduler_output(Scheduler, CounterMap) ->
+    Total =
+        maps:get(emulator, CounterMap) +
+        maps:get(aux, CounterMap) +
+        maps:get(check_io, CounterMap) +
+        maps:get(gc, CounterMap) +
+        maps:get(other, CounterMap),
+    GC = maps:get(gc, CounterMap),
+    GCperc = case Total > 0 of true -> GC/Total; false -> 0.0 end,
+    io:format(
+        user,
+        "~nFor ~w:~n"
+        "emulator=~w, aux=~w, check_io=~w, gc=~w, other=~w~n"
+        "total ~w~n"
+        "percentage_gc ~.2f %~n",
+        [Scheduler,
+            maps:get(emulator, CounterMap),
+            maps:get(aux, CounterMap),
+            maps:get(check_io, CounterMap),
+            GC,
+            maps:get(other, CounterMap),
+            Total,
+            GCperc
+        ]
+    ).
+
+-else.
+
+accounting(_Phase, _Timeout, _ProfileList) ->
+    dummy_accountant().
+
+-endif.

test/end_to_end/recovery_SUITE.erl

@@ -483,12 +483,14 @@ rotate_wipe_compact(Strategy1, Strategy2) ->
     {ok, Book3} = leveled_bookie:book_start(BookOptsAlt),
 
     {KSpcL2, _V2} = testutil:put_indexed_objects(Book3, "AltBucket6", 3000),
-    Q2 = fun(RT) -> {index_query,
-                        "AltBucket6",
-                        {fun testutil:foldkeysfun/3, []},
-                        {"idx1_bin", "#", "|"},
-                        {RT, undefined}}
-                    end,
+    Q2 =
+        fun(RT) -> 
+            {index_query,
+                "AltBucket6",
+                {fun testutil:foldkeysfun/3, []},
+                {<<"idx1_bin">>, <<"#">>, <<"|">>},
+                {RT, undefined}}
+        end,
     {async, KFolder2A} = leveled_bookie:book_returnfolder(Book3, Q2(false)),
     KeyList2A = lists:usort(KFolder2A()),
     true = length(KeyList2A) == 3000,
@@ -629,12 +631,14 @@ recovr_strategy(_Config) ->
                         true = VCH == VCG
                         end,
                     lists:nthtail(6400, AllSpcL)),
-    Q = fun(RT) -> {index_query,
-                        "Bucket6",
-                        {fun testutil:foldkeysfun/3, []},
-                        {"idx1_bin", "#", "|"},
-                        {RT, undefined}}
-                    end,
+    Q =
+        fun(RT) ->
+            {index_query,
+                "Bucket6",
+                {fun testutil:foldkeysfun/3, []},
+                {<<"idx1_bin">>, <<"#">>, <<"|">>},
+                {RT, undefined}}
+        end,
     {async, TFolder} = leveled_bookie:book_returnfolder(Book1, Q(true)),
     KeyTermList = TFolder(),
     {async, KFolder} = leveled_bookie:book_returnfolder(Book1, Q(false)),
@@ -660,12 +664,14 @@ recovr_strategy(_Config) ->
     KeyList2 = lists:usort(KFolder2()),
     true = length(KeyList2) == 6400,
 
-    Q2 = fun(RT) -> {index_query,
-                        "AltBucket6",
-                        {fun testutil:foldkeysfun/3, []},
-                        {"idx1_bin", "#", "|"},
-                        {RT, undefined}}
-                    end,
+    Q2 =
+        fun(RT) ->
+            {index_query,
+                "AltBucket6",
+                {fun testutil:foldkeysfun/3, []},
+                {<<"idx1_bin">>, <<"#">>, <<"|">>},
+                {RT, undefined}}
+        end,
     {async, KFolder2A} = leveled_bookie:book_returnfolder(Book2, Q2(false)),
     KeyList2A = lists:usort(KFolder2A()),
     true = length(KeyList2A) == 3000,

test/end_to_end/testutil.erl

@@ -68,7 +68,7 @@
             compact_and_wait/1]).
 
 -define(RETURN_TERMS, {true, undefined}).
--define(SLOWOFFER_DELAY, 10).
+-define(SLOWOFFER_DELAY, 40).
 -define(V1_VERS, 1).
 -define(MAGIC, 53). % riak_kv -> riak_object
 -define(MD_VTAG,     <<"X-Riak-VTag">>).
@@ -691,21 +691,24 @@ load_objects(ChunkSize, GenList, Bookie, TestObject, Generator, SubListL) ->
 
 
 get_randomindexes_generator(Count) ->
-    Generator = fun() ->
-            lists:map(fun(X) ->
-                                {add,
-                                    "idx" ++ integer_to_list(X) ++ "_bin",
-                                    get_randomdate() ++ get_randomname()} end,
-                        lists:seq(1, Count))
+    Generator =
+        fun() ->
+            lists:map(
+                fun(X) ->
+                    {add,
+                        list_to_binary("idx" ++ integer_to_list(X) ++ "_bin"),
+                        list_to_binary(get_randomdate() ++ get_randomname())}
+                end,
+                lists:seq(1, Count))
         end,
     Generator.
 
 name_list() ->
     [{1, "Sophia"}, {2, "Emma"}, {3, "Olivia"}, {4, "Ava"},
-            {5, "Isabella"}, {6, "Mia"}, {7, "Zoe"}, {8, "Lily"},
-            {9, "Emily"}, {10, "Madelyn"}, {11, "Madison"}, {12, "Chloe"},
-            {13, "Charlotte"}, {14, "Aubrey"}, {15, "Avery"},
-            {16, "Abigail"}].
+        {5, "Isabella"}, {6, "Mia"}, {7, "Zoe"}, {8, "Lily"},
+        {9, "Emily"}, {10, "Madelyn"}, {11, "Madison"}, {12, "Chloe"},
+        {13, "Charlotte"}, {14, "Aubrey"}, {15, "Avery"},
+        {16, "Abigail"}].
 
 get_randomname() ->
     NameList = name_list(),
@@ -738,7 +741,7 @@ check_indexed_objects(Book, B, KSpecL, V) ->
             fun({K, Spc}) ->
                 {ok, O} = book_riakget(Book, B, K),
                 V = testutil:get_value(O),
-                {add, "idx1_bin", IdxVal} = lists:keyfind(add, 1, Spc),
+                {add, <<"idx1_bin">>, IdxVal} = lists:keyfind(add, 1, Spc),
                 {IdxVal, K}
             end,
             KSpecL),
@@ -749,7 +752,7 @@ check_indexed_objects(Book, B, KSpecL, V) ->
             {index_query,
                 B,
                 {fun foldkeysfun/3, []},
-                {"idx1_bin", "0", "|"},
+                {<<"idx1_bin">>, <<"0">>, <<"|">>},
                 ?RETURN_TERMS}),
     SW = os:timestamp(),
     {async, Fldr} = R,
@@ -796,11 +799,12 @@ put_altered_indexed_objects(Book, Bucket, KSpecL, RemoveOld2i) ->
     put_altered_indexed_objects(Book, Bucket, KSpecL, RemoveOld2i, V).
 
 put_altered_indexed_objects(Book, Bucket, KSpecL, RemoveOld2i, V) ->
+    SW = os:timestamp(),
     IndexGen = get_randomindexes_generator(1),
-    
+    ThisProcess = self(),
     FindAdditionFun = fun(SpcItem) -> element(1, SpcItem) == add end,
     MapFun = 
-        fun({K, Spc}) ->
+        fun({K, Spc}, Acc) ->
             OldSpecs = lists:filter(FindAdditionFun, Spc),
             {RemoveSpc, AddSpc} =
                 case RemoveOld2i of
@@ -809,26 +813,45 @@ put_altered_indexed_objects(Book, Bucket, KSpecL, RemoveOld2i, V) ->
                     false ->
                         {[], OldSpecs}
                 end,
-            {O, DeltaSpecs} =
-                set_object(Bucket, K, V,
-                            IndexGen, RemoveSpc, AddSpc),
-            % DeltaSpecs should be new indexes added, and any old indexes which
-            % have been removed by this change where RemoveOld2i is true.
-            %
-            % The actual indexes within the object should reflect any history
-            % of indexes i.e. when RemoveOld2i is false.
-            %
-            % The [{Key, SpecL}] returned should accrue additions over loops if
-            % RemoveOld2i is false
-            case book_riakput(Book, O, DeltaSpecs) of
-                ok -> ok;
-                pause -> timer:sleep(?SLOWOFFER_DELAY)
-            end,
+            PutFun =
+                fun() ->
+                    {O, DeltaSpecs} =
+                        set_object(
+                            Bucket, K, V, IndexGen, RemoveSpc, AddSpc),
+                    % DeltaSpecs should be new indexes added, and any old
+                    % indexes which have been removed by this change where
+                    % RemoveOld2i is true.
+                    %
+                    % The actual indexes within the object should reflect any
+                    % history of indexes i.e. when RemoveOld2i is false.
+                    %
+                    % The [{Key, SpecL}] returned should accrue additions over
+                    % loops if RemoveOld2i is false
+                    R =
+                        case book_riakput(Book, O, DeltaSpecs) of
+                            ok ->
+                                ok;
+                            pause ->
+                                timer:sleep(?SLOWOFFER_DELAY),
+                                pause
+                        end,
+                    ThisProcess ! {R, DeltaSpecs}
+                end,
+            spawn(PutFun),
+            AccOut =
+                receive
+                    {ok, NewSpecs} -> Acc;
+                    {pause, NewSpecs} -> Acc + 1
+                end,
             % Note that order in the SpecL is important, as
             % check_indexed_objects, needs to find the latest item added
-            {K, DeltaSpecs ++ AddSpc}
+            {{K, NewSpecs ++ AddSpc}, AccOut}
         end,
-    RplKSpecL = lists:map(MapFun, KSpecL),
+    {RplKSpecL, Pauses} = lists:mapfoldl(MapFun, 0, KSpecL),
+    io:format(
+        "Altering ~w objects took ~w ms with ~w pauses~n",
+        [length(KSpecL), timer:now_diff(os:timestamp(), SW) div 1000, Pauses]
+    ),
     {RplKSpecL, V}.
 
 rotating_object_check(RootPath, B, NumberOfObjects) ->

test/end_to_end/tictac_SUITE.erl

@@ -378,10 +378,13 @@ index_compare(_Config) ->
     GetTicTacTreeFun =
         fun(X, Bookie) ->
             SW = os:timestamp(),
-            ST = "!",
-            ET = "|",
+            ST = <<"!">>,
+            ET = <<"|">>,
             Q = {tictactree_idx,
-                    {BucketBin, "idx" ++ integer_to_list(X) ++ "_bin", ST, ET},
+                    {BucketBin,
+                        list_to_binary("idx" ++ integer_to_list(X) ++ "_bin"),
+                        ST,
+                        ET},
                     TreeSize,
                     fun(_B, _K) -> accumulate end},
             {async, Folder} = leveled_bookie:book_returnfolder(Bookie, Q),
@@ -442,12 +445,14 @@ index_compare(_Config) ->
     true = DL2_0 == [],
     true = length(DL2_1) > 100,
 
-    IdxSpc = {add, "idx2_bin", "zz999"},
-    {TestObj, TestSpc} = testutil:generate_testobject(BucketBin,
-                                                        term_to_binary("K9.Z"),
-                                                        "Value1",
-                                                        [IdxSpc],
-                                                        [{"MDK1", "MDV1"}]),
+    IdxSpc = {add, <<"idx2_bin">>, <<"zz999">>},
+    {TestObj, TestSpc} =
+        testutil:generate_testobject(
+            BucketBin,
+            term_to_binary("K9.Z"),
+            "Value1",
+            [IdxSpc],
+            [{"MDK1", "MDV1"}]),
     ok = testutil:book_riakput(Book2C, TestObj, TestSpc),
     testutil:check_forobject(Book2C, TestObj),
 
@@ -457,25 +462,30 @@ index_compare(_Config) ->
     TicTacTree3_P3 = GetTicTacTreeFun(2, Book2D),
 
     % Merge the tree across the partitions
-    TicTacTree3_Joined = lists:foldl(fun leveled_tictac:merge_trees/2,
-                                        TicTacTree3_P1,
-                                        [TicTacTree3_P2, TicTacTree3_P3]),
+    TicTacTree3_Joined =
+        lists:foldl(
+            fun leveled_tictac:merge_trees/2,
+            TicTacTree3_P1,
+            [TicTacTree3_P2, TicTacTree3_P3]),
 
     % Find all keys index, and then just the last key
     IdxQ1 = {index_query,
                 BucketBin,
                 {fun testutil:foldkeysfun/3, []},
-                {"idx2_bin", "zz", "zz|"},
+                {<<"idx2_bin">>, <<"zz">>, <<"zz|">>},
                 {true, undefined}},
     {async, IdxFolder1} = leveled_bookie:book_returnfolder(Book2C, IdxQ1),
     true = IdxFolder1() >= 1,
 
-    DL_3to2B = leveled_tictac:find_dirtyleaves(TicTacTree2_P1,
-                                                TicTacTree3_P1),
-    DL_3to2C = leveled_tictac:find_dirtyleaves(TicTacTree2_P2,
-                                                TicTacTree3_P2),
-    DL_3to2D = leveled_tictac:find_dirtyleaves(TicTacTree2_P3,
-                                                TicTacTree3_P3),
+    DL_3to2B =
+        leveled_tictac:find_dirtyleaves(
+            TicTacTree2_P1, TicTacTree3_P1),
+    DL_3to2C =
+        leveled_tictac:find_dirtyleaves(
+            TicTacTree2_P2, TicTacTree3_P2),
+    DL_3to2D =
+        leveled_tictac:find_dirtyleaves(
+            TicTacTree2_P3, TicTacTree3_P3),
     io:format("Individual tree comparison found dirty leaves of ~w ~w ~w~n",
                 [DL_3to2B, DL_3to2C, DL_3to2D]),
 
@@ -509,7 +519,7 @@ index_compare(_Config) ->
     MismatchQ = {index_query,
                     BucketBin,
                     {FoldKeysIndexQFun, []},
-                    {"idx2_bin", "!", "|"},
+                    {<<"idx2_bin">>, <<"!">>, <<"|">>},
                     {true, undefined}},
     {async, MMFldr_2A} = leveled_bookie:book_returnfolder(Book2A, MismatchQ),
     {async, MMFldr_2B} = leveled_bookie:book_returnfolder(Book2B, MismatchQ),
@@ -531,7 +541,7 @@ index_compare(_Config) ->
     io:format("Differences between lists ~w~n", [Diffs]),
 
     % The actual difference is discovered
-    true = lists:member({"zz999", term_to_binary("K9.Z")}, Diffs),
+    true = lists:member({<<"zz999">>, term_to_binary("K9.Z")}, Diffs),
     % Without discovering too many others
     true = length(Diffs) < 20,