From b2835aeaeca48d2bceac9e079a88c3da67b1df1a Mon Sep 17 00:00:00 2001
From: martinsumner <martin.sumner@adaptip.co.uk>
Date: Wed, 21 Dec 2016 18:28:14 +0000
Subject: [PATCH] Improve fetching efficiency

Experiment to see if parsing all keys in block can be avoided - and if
so does this make the range scan more efficient.

Unproven change.
---
 src/leveled_penciller.erl |  2 +-
 src/leveled_sft.erl       | 31 ++++++++++++++++++++++++++++++-
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl
index f338076..ad13bfc 100644
--- a/src/leveled_penciller.erl
+++ b/src/leveled_penciller.erl
@@ -202,7 +202,7 @@
 -define(PROMPT_WAIT_ONL0, 5).
 -define(WORKQUEUE_BACKLOG_TOLERANCE, 4).
 -define(COIN_SIDECOUNT, 5).
--define(SLOW_FETCH, 10000).
+-define(SLOW_FETCH, 20000).
 
 -record(state, {manifest = [] :: list(),
 				manifest_sqn = 0 :: integer(),
diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl
index d66dfaf..8c42d23 100644
--- a/src/leveled_sft.erl
+++ b/src/leveled_sft.erl
@@ -644,6 +644,8 @@ acc_list_keysonly(null, empty) ->
     [];
 acc_list_keysonly(null, RList) ->
     RList;
+acc_list_keysonly(R, RList) when is_list(R) ->
+    lists:foldl(fun acc_list_keysonly/2, RList, R);
 acc_list_keysonly(R, RList) ->
     lists:append(RList, [leveled_codec:strip_to_keyseqstatusonly(R)]).
 
@@ -651,6 +653,8 @@ acc_list_kv(null, empty) ->
     [];
 acc_list_kv(null, RList) ->
     RList;
+acc_list_kv(R, RList) when is_list(R) ->
+    RList ++ R;
 acc_list_kv(R, RList) ->
     lists:append(RList, [R]).
 
@@ -713,7 +717,13 @@ fetch_range(Handle, FileMD, StartKey, NearestKey, EndKey,
                         Pointer,
                         Acc) ->
     Block = fetch_block(Handle, LengthList, BlockNumber, Pointer),
-    Results = scan_block(Block, StartKey, EndKey, AccFun, Acc),
+    Results = 
+        case maybe_scan_entire_block(Block, StartKey, EndKey) of
+            true ->
+                {partial, AccFun(Block, Acc), StartKey};
+            false ->
+                scan_block(Block, StartKey, EndKey, AccFun, Acc)
+        end,
     case Results of
         {partial, Acc1, StartKey} ->
             %% Move on to the next block
@@ -741,6 +751,25 @@ scan_block([HeadKV|T], StartKey, EndKey, AccFun, Acc) ->
     end.
 
 
+maybe_scan_entire_block([], _, _) ->
+    true;
+maybe_scan_entire_block(_Block, all, all) ->
+    true;
+maybe_scan_entire_block(Block, StartKey, all) ->
+    [FirstKey|_Tail] = Block,
+    leveled_codec:strip_to_keyonly(FirstKey) >= StartKey;
+maybe_scan_entire_block(Block, StartKey, EndKey) ->
+    [FirstKey|_Tail] = Block,
+    LastKey = leveled_codec:strip_to_keyonly(lists:last(Block)),
+    FromStart = leveled_codec:strip_to_keyonly(FirstKey) >= StartKey,
+    ToEnd = leveled_codec:endkey_passed(EndKey, LastKey),
+    case {FromStart, ToEnd} of
+        {true, true} ->
+            true;
+        _ ->
+            false
+    end.
+
 fetch_keyvalue_fromblock([], _Key, _LengthList, _Handle, _StartOfSlot) ->
     not_present;
 fetch_keyvalue_fromblock([BlockNmb|T], Key, LengthList, Handle, StartOfSlot) ->