TupleBuckets in Riak objects

Adds support with test for tuplebuckets in Riak keys.

This exposed that there was no filter using the seglist on the in-mmemory keys.  This means that if there is no filter applied in the fold_function, many false positives may emerge.

This is probably not a big performance benefit (and indeed for performance it may be better to apply during the leveled_pmem:merge_trees).

Some thought still required as to what is more likely to contribute to future bugs: an extra location using the hash matching found in leveled_sst, or the extra results in the query.

src/leveled_codec.erl

@@ -36,6 +36,7 @@
         inker_reload_strategy/1,
         strip_to_seqonly/1,
         strip_to_statusonly/1,
+        strip_to_segmentonly/1,
         strip_to_keyseqonly/1,
         strip_to_indexdetails/1,
         striphead_to_v1details/1,
@@ -174,12 +175,24 @@ segment_hash(Key) when is_binary(Key) ->
 segment_hash({?RIAK_TAG, Bucket, Key, null}) 
                                     when is_binary(Bucket), is_binary(Key) ->
     segment_hash(<<Bucket/binary, Key/binary>>);
+segment_hash({?RIAK_TAG, {BucketType, Bucket}, Key, SubKey})
+                            when is_binary(BucketType), is_binary(Bucket) ->
+    segment_hash({?RIAK_TAG,
+                    <<BucketType/binary, Bucket/binary>>,
+                    Key,
+                    SubKey});
 segment_hash({?HEAD_TAG, Bucket, Key, SubK})
                     when is_binary(Bucket), is_binary(Key), is_binary(SubK) ->
     segment_hash(<<Bucket/binary, Key/binary, SubK/binary>>);
 segment_hash({?HEAD_TAG, Bucket, Key, _SubK})
                                     when is_binary(Bucket), is_binary(Key) ->
     segment_hash(<<Bucket/binary, Key/binary>>);
+% segment_hash({?HEAD_TAG, {BucketType, Bucket}, Key, SubKey})
+%                             when is_binary(BucketType), is_binary(Bucket) ->
+%     segment_hash({?HEAD_TAG,
+%                     <<BucketType/binary, Bucket/binary>>, 
+%                     Key,
+%                     SubKey});
 segment_hash(Key) ->
     segment_hash(term_to_binary(Key)).
 
@@ -207,6 +220,9 @@ strip_to_statusonly({_, V}) -> element(2, V).
 -spec strip_to_seqonly(ledger_kv()) -> non_neg_integer().
 strip_to_seqonly({_, V}) -> element(1, V).
 
+-spec strip_to_segmentonly(ledger_kv()) -> segment_hash().
+strip_to_segmentonly({_LK, LV}) -> element(3, LV).
+
 -spec strip_to_keyseqonly(ledger_kv()) -> {ledger_key(), integer()}.
 strip_to_keyseqonly({LK, V}) -> {LK, element(1, V)}.
 

src/leveled_penciller.erl

@@ -724,6 +724,22 @@ handle_call({fetch_keys,
             List ->
                 List
         end,
+    FilteredL0 = 
+        case SegmentList of
+            false ->
+                L0AsList;
+            _ ->
+                TunedList = leveled_sst:tune_seglist(SegmentList),
+                FilterFun =
+                    fun(LKV) ->
+                        CheckSeg = 
+                            leveled_sst:extract_hash(
+                                leveled_codec:strip_to_segmentonly(LKV)),
+                        lists:member(CheckSeg, TunedList)
+                    end,
+                lists:filter(FilterFun, L0AsList)
+        end,
+    
     leveled_log:log_randomtimer("P0037",
                                 [State#state.levelzero_size],
                                 SW,
@@ -742,7 +758,7 @@ handle_call({fetch_keys,
     SSTiter = lists:foldl(SetupFoldFun, [], lists:seq(0, ?MAX_LEVELS - 1)),
     Folder = 
         fun() -> 
-            keyfolder({L0AsList, SSTiter},
+            keyfolder({FilteredL0, SSTiter},
                         {StartKey, EndKey},
                         {AccFun, InitAcc},
                         {SegmentList, LastModRange0, MaxKeys})

src/leveled_sst.erl

@@ -121,7 +121,7 @@
             sst_deleteconfirmed/1,
             sst_close/1]).
 
-
+-export([tune_seglist/1, extract_hash/1]).
 
 -record(slot_index_value, {slot_id :: integer(),
                             start_position :: integer(),
@@ -879,7 +879,7 @@ fetch(LedgerKey, Hash, State, Timings0) ->
                 State#state{blockindex_cache = BlockIndexCache}, 
                 Timings3};
         {BlockLengths, _LMD, PosBin} ->
-            PosList = find_pos(PosBin, extra_hash(Hash), [], 0),
+            PosList = find_pos(PosBin, extract_hash(Hash), [], 0),
             case PosList of 
                 [] ->
                     {_SW3, Timings3} =
@@ -1290,7 +1290,7 @@ lookup_slots(StartKey, EndKey, Tree) ->
 accumulate_positions({K, V}, {PosBinAcc, NoHashCount, HashAcc, LMDAcc}) ->
     {_SQN, H1, LMD} = leveled_codec:strip_to_indexdetails({K, V}),
     LMDAcc0 = take_max_lastmoddate(LMD, LMDAcc),
-    PosH1 = extra_hash(H1),
+    PosH1 = extract_hash(H1),
     case is_integer(PosH1) of 
         true ->
             case NoHashCount of 
@@ -1725,7 +1725,7 @@ binaryslot_get(FullBin, Key, Hash, PressMethod, IdxModDate) ->
             {BlockLengths, _LMD, PosBinIndex} =
                 extract_header(Header, IdxModDate),
             PosList = find_pos(PosBinIndex,
-                                extra_hash(Hash), 
+                                extract_hash(Hash), 
                                 [], 
                                 0),
             {fetch_value(PosList, BlockLengths, Blocks, Key, PressMethod),
@@ -1926,9 +1926,9 @@ block_offsetandlength(BlockLengths, BlockID) ->
             {B1L + B2L + B3L + B4L, B5L}
     end.
 
-extra_hash({SegHash, _ExtraHash}) when is_integer(SegHash) ->
+extract_hash({SegHash, _ExtraHash}) when is_integer(SegHash) ->
     tune_hash(SegHash);
-extra_hash(NotHash) ->
+extract_hash(NotHash) ->
     NotHash.
 
 cache_hash({_SegHash, ExtraHash}) when is_integer(ExtraHash) ->
@@ -2658,8 +2658,8 @@ indexed_list_mixedkeys_bitflip_test() ->
     ToList = binaryslot_tolist(SlotBin, native, ?INDEX_MODDATE),
     ?assertMatch(Keys, ToList),
 
-    [Pos1] = find_pos(PosBin, extra_hash(MH1), [], 0),
-    [Pos2] = find_pos(PosBin, extra_hash(MH2), [], 0),
+    [Pos1] = find_pos(PosBin, extract_hash(MH1), [], 0),
+    [Pos2] = find_pos(PosBin, extract_hash(MH2), [], 0),
     {BN1, _BP1} = revert_position(Pos1),
     {BN2, _BP2} = revert_position(Pos2),
     {Offset1, Length1} = block_offsetandlength(Header, BN1),