From 5bef21d9719a561beeaf735e168bd140d9771e38 Mon Sep 17 00:00:00 2001
From: Martin Sumner <martin.sumner@adaptip.co.uk>
Date: Mon, 22 Jul 2019 10:35:55 +0100
Subject: [PATCH] Add unit test to prove binary/copy issue

Need to understand why the binary:copy is necessary - unit test now shows this.
---
 src/leveled_sst.erl | 54 ++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 49 insertions(+), 5 deletions(-)

diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl
index acabb08..35b8690 100644
--- a/src/leveled_sst.erl
+++ b/src/leveled_sst.erl
@@ -2128,7 +2128,7 @@ crc_check_slot(FullBin) ->
         CRC32H:32/integer,
         Rest/binary>> = FullBin,
     PosBL0 = min(PosBL, byte_size(FullBin) - 12), 
-        % If the position has been bit-flipped to beyond the maximum paossible
+        % If the position has been bit-flipped to beyond the maximum possible
         % length, use the maximum possible length
     <<Header:PosBL0/binary, Blocks/binary>> = Rest,
     case {hmac(Header), hmac(PosBL0)} of 
@@ -2630,16 +2630,20 @@ generate_randomkeys(_Seqn, 0, Acc, _BucketLow, _BucketHigh) ->
 generate_randomkeys(Seqn, Count, Acc, BucketLow, BRange) ->
     BRand = leveled_rand:uniform(BRange),
     BNumber =
-        lists:flatten(io_lib:format("K~4..0B", [BucketLow + BRand])),
+        lists:flatten(io_lib:format("B~6..0B", [BucketLow + BRand])),
     KNumber =
-        lists:flatten(io_lib:format("K~6..0B", [leveled_rand:uniform(1000)])),
+        lists:flatten(io_lib:format("K~8..0B", [leveled_rand:uniform(1000000)])),
     LK = leveled_codec:to_ledgerkey("Bucket" ++ BNumber, "Key" ++ KNumber, o),
     Chunk = leveled_rand:rand_bytes(64),
     {_B, _K, MV, _H, _LMs} =
         leveled_codec:generate_ledgerkv(LK, Seqn, Chunk, 64, infinity),
+    MD = element(4, MV),
+    ?assertMatch(undefined, element(3, MD)),
+    MD0 = [{magic_md, [<<0:32/integer>>, base64:encode(Chunk)]}],
+    MV0 = setelement(4, MV, setelement(3, MD, MD0)),
     generate_randomkeys(Seqn + 1,
                         Count - 1,
-                        [{LK, MV}|Acc],
+                        [{LK, MV0}|Acc],
                         BucketLow,
                         BRange).
 
@@ -2661,6 +2665,7 @@ generate_indexkey(Term, Count) ->
                                     Count, 
                                     infinity).
 
+
 form_slot_test() ->
     % If a skip key happens, mustn't switch to loookup by accident as could be
     % over the expected size
@@ -3243,13 +3248,17 @@ simple_persisted_test_bothformats() ->
     simple_persisted_tester(fun testsst_new/6).
 
 simple_persisted_tester(SSTNewFun) ->
+    Level = 3,
     {RP, Filename} = {?TEST_AREA, "simple_test"},
     KVList0 = generate_randomkeys(1, ?LOOK_SLOTSIZE * 32, 1, 20),
     KVList1 = lists:ukeysort(1, KVList0),
     [{FirstKey, _FV}|_Rest] = KVList1,
     {LastKey, _LV} = lists:last(KVList1),
     {ok, Pid, {FirstKey, LastKey}, _Bloom} = 
-        SSTNewFun(RP, Filename, 1, KVList1, length(KVList1), native),
+        SSTNewFun(RP, Filename, Level, KVList1, length(KVList1), native),
+    
+    B0 = check_binary_references(Pid),
+
     SW0 = os:timestamp(),
     lists:foreach(fun({K, V}) ->
                         ?assertMatch({K, V}, sst_get(Pid, K))
@@ -3346,9 +3355,44 @@ simple_persisted_tester(SSTNewFun) ->
     FetchedListB4 = lists:foldl(FoldFun, [], FetchListB4),
     ?assertMatch([{Eight000Key, _v800}], FetchedListB4),
     
+    B1 = check_binary_references(Pid),
+
     ok = sst_close(Pid),
+
+    io:format(user, "Reopen SST file~n", []),
+    OptsSST = #sst_options{press_method=native,
+                            log_options=leveled_log:get_opts()},
+    {ok, OpenP, {FirstKey, LastKey}, _Bloom} =
+        sst_open(RP, Filename ++ ".sst", OptsSST, Level),
+
+    B2 = check_binary_references(OpenP),
+
+    lists:foreach(fun({K, V}) ->
+                        ?assertMatch({K, V}, sst_get(OpenP, K)),
+                        ?assertMatch({K, V}, sst_get(OpenP, K))
+                        end,
+                    KVList1),
+
+    garbage_collect(OpenP),
+    B3 = check_binary_references(OpenP),
+    ?assertMatch(0, B2), % Opens with an empty cache
+    ?assertMatch(true, B3 > B2), % Now has headers in cache
+    ?assertMatch(false, B3 > B0 * 2), 
+        % Not significantly bigger than when created new
+    ?assertMatch(false, B3 > B1 * 2), 
+        % Not significantly bigger than when created new
+
+    ok = sst_close(OpenP),
     ok = file:delete(filename:join(RP, Filename ++ ".sst")).
 
+check_binary_references(Pid) ->
+    garbage_collect(Pid),
+    {binary, BinList} = process_info(Pid, binary),
+    TotalBinMem = 
+        lists:foldl(fun({_R, BM, _RC}, Acc) -> Acc + BM end, 0, BinList),
+    io:format(user, "Total binary memory ~w~n", [TotalBinMem]),
+    TotalBinMem.
+
 key_dominates_test() ->
     KV1 = {{o, "Bucket", "Key1", null}, {5, {active, infinity}, 0, []}},
     KV2 = {{o, "Bucket", "Key3", null}, {6, {active, infinity}, 0, []}},