Change refernces to loop state records

Resolve issue with OTP 22 performance https://github.com/martinsumner/leveled/issues/326 - by changing refernces to loop state.

The test perf_SUITE proves the issue.

OTP 22, without fixes:

Fold pre-close 41209 ms post-close 688 ms

OTP 22, with fixes:

Fold pre-close 401 ms post-close 317 ms

src/leveled_penciller.erl

@@ -796,9 +796,15 @@ handle_call({fetch_keys,
                                 [State#state.levelzero_size],
                                 SW,
                                 0.01),
+    
+    %% Rename any reference to loop state that may be used by the function
+    %% to be returned - https://github.com/martinsumner/leveled/issues/326
+    Manifest = State#state.manifest,
+    SnapshotTime = State#state.snapshot_time,
+    
     SetupFoldFun =
         fun(Level, Acc) ->
-            Pointers = leveled_pmanifest:range_lookup(State#state.manifest,
+            Pointers = leveled_pmanifest:range_lookup(Manifest,
                                                         Level,
                                                         StartKey,
                                                         EndKey),
@@ -812,7 +818,7 @@ handle_call({fetch_keys,
         fun() -> 
             keyfolder({FilteredL0, SSTiter},
                         {StartKey, EndKey},
-                        {AccFun, InitAcc, State#state.snapshot_time},
+                        {AccFun, InitAcc, SnapshotTime},
                         {SegmentList, LastModRange0, MaxKeys})
         end,
     case By of 
@@ -1403,9 +1409,9 @@ roll_memory(State, true) ->
     ManSQN = leveled_pmanifest:get_manifest_sqn(State#state.manifest) + 1,
     RootPath = sst_rootpath(State#state.root_path),
     FileName = sst_filename(ManSQN, 0, 0),
-    FetchFun = fun(Slot) -> lists:nth(Slot, State#state.levelzero_cache) end,
+    LZC = State#state.levelzero_cache,
-    KVList = leveled_pmem:to_list(length(State#state.levelzero_cache),
+    FetchFun = fun(Slot) -> lists:nth(Slot, LZC) end,
-                                    FetchFun),
+    KVList = leveled_pmem:to_list(length(LZC), FetchFun),
     R = leveled_sst:sst_new(RootPath,
                             FileName,
                             0,

test/end_to_end/perf_SUITE.erl

@@ -0,0 +1,85 @@
+-module(perf_SUITE).
+-include_lib("common_test/include/ct.hrl").
+-include("include/leveled.hrl").
+-export([all/0]).
+-export([bigpcl_bucketlist/1
+            ]).
+
+all() -> [bigpcl_bucketlist].
+
+    
+bigpcl_bucketlist(_Config) ->
+    %% https://github.com/martinsumner/leveled/issues/326
+    %% In OTP 22+ there appear to be issues with anonymous functions which
+    %% have a reference to loop state, requiring a copy of all the loop state
+    %% to be made when returning the function.
+    %% This test creates  alarge loop state on the leveled_penciller to prove
+    %% this.
+    %% The problem can be resolved simply by renaming the element of the loop
+    %% state using within the anonymous function.
+    RootPath = testutil:reset_filestructure(),
+    BucketCount = 500,
+    ObjectCount = 100,
+    StartOpts1 = [{root_path, RootPath},
+                    {max_journalsize, 50000000},
+                    {cache_size, 4000},
+                    {max_pencillercachesize, 128000},
+                    {max_sstslots, 256},
+                    {sync_strategy, testutil:sync_strategy()},
+                    {compression_point, on_compact}],
+    {ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
+    BucketList =
+        lists:map(fun(I) -> list_to_binary(integer_to_list(I)) end,
+                    lists:seq(1, BucketCount)),
+
+    MapFun =
+        fun(B) ->
+            testutil:generate_objects(ObjectCount, 1, [], 
+                                        leveled_rand:rand_bytes(100), 
+                                        fun() -> [] end, 
+                                        B)
+        end,
+    ObjLofL = lists:map(MapFun, BucketList),
+    lists:foreach(fun(ObjL) -> testutil:riakload(Bookie1, ObjL) end, ObjLofL),
+    BucketFold =
+        fun(B, _K, _V, Acc) ->
+            case sets:is_element(B, Acc) of
+                true ->
+                    Acc;
+                false ->
+                    sets:add_element(B, Acc)
+            end
+        end,
+    FBAccT = {BucketFold, sets:new()},
+
+    {async, BucketFolder1} = 
+        leveled_bookie:book_headfold(Bookie1,
+                                        ?RIAK_TAG,
+                                        {bucket_list, BucketList},
+                                        FBAccT,
+                                        false, false, false),
+
+    {FoldTime1, BucketList1} = timer:tc(BucketFolder1, []),
+    true = BucketCount == sets:size(BucketList1),
+    ok = leveled_bookie:book_close(Bookie1),
+
+    {ok, Bookie2} = leveled_bookie:book_start(StartOpts1),
+    
+    {async, BucketFolder2} = 
+        leveled_bookie:book_headfold(Bookie2,
+                                        ?RIAK_TAG,
+                                        {bucket_list, BucketList},
+                                        FBAccT,
+                                        false, false, false),
+    {FoldTime2, BucketList2} = timer:tc(BucketFolder2, []),
+    true = BucketCount == sets:size(BucketList2),
+
+    io:format("Fold pre-close ~w ms post-close ~w ms~n",
+                [FoldTime1 div 1000, FoldTime2 div 1000]),
+
+    true = FoldTime1 < 10 * FoldTime2,
+    %% The fold in-memory should be the same order of magnitude of response
+    %% time as the fold post-persistence
+
+    ok = leveled_bookie:book_destroy(Bookie2).
+