From dfab33e8da458061e40e00e9899f58793c5b361f Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Mon, 25 Sep 2017 13:07:08 +0100 Subject: [PATCH 01/10] Add smaller trees The "small" tree will serialise to 1.5MB - which seems large. Much smaller trees seem to be more suitable for things like recently modified aae indexes. --- src/leveled_tictac.erl | 21 +++++++++++++++------ test/end_to_end/tictac_SUITE.erl | 4 ++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/leveled_tictac.erl b/src/leveled_tictac.erl index 39d7db8..d61c414 100644 --- a/src/leveled_tictac.erl +++ b/src/leveled_tictac.erl @@ -73,13 +73,15 @@ -include_lib("eunit/include/eunit.hrl"). -define(HASH_SIZE, 4). +-define(XXSMALL, {6, 64, 64 * 64}). +-define(XSMALL, {7, 128, 128 * 128}). -define(SMALL, {8, 256, 256 * 256}). -define(MEDIUM, {9, 512, 512 * 512}). -define(LARGE, {10, 1024, 1024 * 1024}). -define(XLARGE, {11, 2048, 2048 * 2048}). -record(tictactree, {treeID :: any(), - size :: small|medium|large|xlarge, + size :: xxsmall|xsmall|small|medium|large|xlarge, width :: integer(), bitwidth :: integer(), segment_count :: integer(), @@ -139,10 +141,9 @@ import_tree(ExportedTree) -> [{<<"level1">>, L1Base64}, {<<"level2">>, {struct, L2List}}]} = ExportedTree, L1Bin = base64:decode(L1Base64), - Sizes = [{small, element(2, ?SMALL)}, - {medium, element(2, ?MEDIUM)}, - {large, element(2, ?LARGE)}, - {xlarge, element(2, ?XLARGE)}], + Sizes = + lists:map(fun(SizeTag) -> {SizeTag, element(2, get_size(SizeTag))} end, + [xxsmall, xsmall, small, medium, large, xlarge]), Width = byte_size(L1Bin) div ?HASH_SIZE, {Size, Width} = lists:keyfind(Width, 2, Sizes), {BitWidth, Width, SegmentCount} = get_size(Size), @@ -275,7 +276,9 @@ merge_trees(TreeA, TreeB) -> MergedTree#tictactree{level1 = NewLevel1, level2 = NewLevel2}. --spec get_segment(any(), integer()|small|medium|large|xlarge) -> integer(). +-spec get_segment(any(), + integer()|xsmall|xxsmall|small|medium|large|xlarge) -> + integer(). %% @doc %% Return the segment ID for a Key. Can pass the tree size or the actual %% segment count derived from the size @@ -297,6 +300,10 @@ tictac_hash(Key, Term) -> get_size(Size) -> case Size of + xxsmall -> + ?XXSMALL; + xsmall -> + ?XSMALL; small -> ?SMALL; medium -> @@ -357,6 +364,8 @@ merge_binaries(BinA, BinB) -> simple_bysize_test() -> + simple_test_withsize(xxsmall), + simple_test_withsize(xsmall), simple_test_withsize(small), simple_test_withsize(medium), simple_test_withsize(large), diff --git a/test/end_to_end/tictac_SUITE.erl b/test/end_to_end/tictac_SUITE.erl index a837738..c78b27e 100644 --- a/test/end_to_end/tictac_SUITE.erl +++ b/test/end_to_end/tictac_SUITE.erl @@ -305,11 +305,11 @@ many_put_compare(_Config) -> index_compare(_Config) -> - TreeSize = small, + TreeSize = xxsmall, LS = 2000, JS = 50000000, SS = testutil:sync_strategy(), - SegmentCount = 256 * 256, + SegmentCount = 64 * 64, % Test requires multiple different databases, so want to mount them all % on individual file paths From f50a2a19d32eda03d552f89dce1c85e02154c633 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Mon, 25 Sep 2017 15:57:06 +0100 Subject: [PATCH 02/10] File should not be pushed --- .rebar/erlcinfo | Bin 913 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .rebar/erlcinfo diff --git a/.rebar/erlcinfo b/.rebar/erlcinfo deleted file mode 100644 index dc2cd62c4e84fff2719fa64c98ce1aeb5c3a5bcf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 913 zcmV;C18)3-PyhfI9e7@~SjleNFc4+zG;#JlZI)(h(iN8C7)egO6zHKS&_mmxM`0u} z5v|S85+Hxu{z)mtPDEOwXuv$l3(1-H=FLJmO&hgvOn*5E8GQ3lpv3%VHtpYy%< zp_lqtk4bRnVIv4>a5J|}h@H?dDNe_*BXW-;gUTtF6i+O7EE8)p3|+$E9tL8DigrY; zX?V+*GNN!l0Mg?9PGQ~sNf zsv{oBqoB&5v|xdfyZ~Le&PZmIV$q!8RLIrX&D@Yjb0O{5V`nN2)yjiOcltuQM#?~h zt&ryILm3t>ENo%(At1MP(P0W(-^XI-7z0Ggz@PELpV4d3?4pRdZxx zEDkJcn%dRbr_%5SS5uz4k4mFwE|Mr5GO=3V)l$rIl@OMA!^Oc&rQs<~ox|}?hOOPI z1|qZh6*o;kkFGMiUvRshM<|)yGjvhO-9jbL?@BZ}I^)VXzgl?>k=eb*K_A?dR6;*- zk?Ahr4n(&p6RpxqCL8>Qhn-euNlglXcSZc^r z3jctsCC@J!WB}|{Wu-dNtO(kVI;*oB(81e48IC^4^ox42O$#Otb-3Tu_D;d84>r-Z n#Mz<_!`qTFGkULA+_O(>i4?uJWSM*|$k~a~*z5fVHQZjJ5ai0S From 2f9afa14695063d5e15366c5d213b7abd9df1b72 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 26 Sep 2017 16:32:59 +0100 Subject: [PATCH 03/10] Add support for performing a magic hash on a binary Ignore unnecessray term_to_binary if already binary. This will be useful when we use magic_hash in tictac_trees we wish to be exportable. --- src/leveled_codec.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index 9ed2a2c..1f6d6cc 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -87,10 +87,12 @@ magic_hash({?RIAK_TAG, Bucket, Key, _SubKey}) -> magic_hash({Bucket, Key}); magic_hash({?STD_TAG, Bucket, Key, _SubKey}) -> magic_hash({Bucket, Key}); +magic_hash({binary, BinaryKey}) -> + H = 5381, + hash1(H, BinaryKey) band 16#FFFFFFFF; magic_hash(AnyKey) -> BK = term_to_binary(AnyKey), - H = 5381, - hash1(H, BK) band 16#FFFFFFFF. + magic_hash({binary, BK}). hash1(H, <<>>) -> H; From 8e3b7baa18abf5cead7d4cdd0286cdc90c092a8a Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 29 Sep 2017 11:08:37 +0100 Subject: [PATCH 04/10] Change encoding - issue with JSON friendliness Also add compression to reduce penalty on sparse trees --- src/leveled_tictac.erl | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/leveled_tictac.erl b/src/leveled_tictac.erl index 6b2aeb1..cd39055 100644 --- a/src/leveled_tictac.erl +++ b/src/leveled_tictac.erl @@ -121,13 +121,13 @@ new_tree(TreeID, Size) -> %% Export the tree into a tuple list, with the level1 binary, and then for %% level2 {branchID, binary()} export_tree(Tree) -> + EncodeL2Fun = + fun(X, L2Acc) -> + L2Element = zlib:compress(array:get(X, Tree#tictactree.level2)), + [{integer_to_list(X), base64:encode_to_string(L2Element)}|L2Acc] + end, L2 = - lists:foldl(fun(X, L2Acc) -> - [{integer_to_binary(X), - array:get(X, Tree#tictactree.level2)}|L2Acc] - end, - [], - lists:seq(0, Tree#tictactree.width - 1)), + lists:foldl(EncodeL2Fun, [], lists:seq(0, Tree#tictactree.width - 1)), {struct, [{<<"level1">>, base64:encode_to_string(Tree#tictactree.level1)}, {<<"level2">>, {struct, lists:reverse(L2)}} @@ -149,8 +149,9 @@ import_tree(ExportedTree) -> {BitWidth, Width, SegmentCount} = get_size(Size), Lv2Init = array:new([{size, Width}]), FoldFun = - fun({X, L2SegBin}, L2Array) -> - array:set(binary_to_integer(X), L2SegBin, L2Array) + fun({X, EncodedL2SegBin}, L2Array) -> + L2SegBin = zlib:uncompress(base64:decode(EncodedL2SegBin)), + array:set(list_to_integer(X), L2SegBin, L2Array) end, Lv2 = lists:foldl(FoldFun, Lv2Init, L2List), #tictactree{treeID = import, @@ -386,7 +387,10 @@ merge_binaries(BinA, BinB) -> -ifdef(TEST). -simple_bysize_test() -> +simple_bysize_test_() -> + {timeout, 60, fun simple_bysize_test_allsizes/0}. + +simple_bysize_test_allsizes() -> simple_test_withsize(xxsmall), simple_test_withsize(xsmall), simple_test_withsize(small), From 4d6f816ab2868bf3e46c52341d6d627e13921560 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 29 Sep 2017 11:17:02 +0100 Subject: [PATCH 05/10] Switch back to binary --- src/leveled_tictac.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/leveled_tictac.erl b/src/leveled_tictac.erl index cd39055..8e28c46 100644 --- a/src/leveled_tictac.erl +++ b/src/leveled_tictac.erl @@ -124,7 +124,7 @@ export_tree(Tree) -> EncodeL2Fun = fun(X, L2Acc) -> L2Element = zlib:compress(array:get(X, Tree#tictactree.level2)), - [{integer_to_list(X), base64:encode_to_string(L2Element)}|L2Acc] + [{integer_to_binary(X), base64:encode_to_string(L2Element)}|L2Acc] end, L2 = lists:foldl(EncodeL2Fun, [], lists:seq(0, Tree#tictactree.width - 1)), @@ -151,7 +151,7 @@ import_tree(ExportedTree) -> FoldFun = fun({X, EncodedL2SegBin}, L2Array) -> L2SegBin = zlib:uncompress(base64:decode(EncodedL2SegBin)), - array:set(list_to_integer(X), L2SegBin, L2Array) + array:set(binary_to_integer(X), L2SegBin, L2Array) end, Lv2 = lists:foldl(FoldFun, Lv2Init, L2List), #tictactree{treeID = import, From 5e6534fb491bb5c3b10d8e7d88541125bf7fa22f Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 29 Sep 2017 15:01:16 +0100 Subject: [PATCH 06/10] Initialise empty trees When new trees are initialised they are started with 1 byte binaries at Level2 - and become full-size following a merge or add event. The idea is that when trees are distributed before they are added to, or when over-sized trees are used - the output may be smaller on the network. --- src/leveled_tictac.erl | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/leveled_tictac.erl b/src/leveled_tictac.erl index 8e28c46..2edd636 100644 --- a/src/leveled_tictac.erl +++ b/src/leveled_tictac.erl @@ -79,6 +79,7 @@ -define(MEDIUM, {9, 512, 512 * 512}). -define(LARGE, {10, 1024, 1024 * 1024}). -define(XLARGE, {11, 2048, 2048 * 2048}). +-define(EMPTY, <<0:8/integer>>). -record(tictactree, {treeID :: any(), size :: xxsmall|xsmall|small|medium|large|xlarge, @@ -105,9 +106,7 @@ new_tree(TreeID, Size) -> {BitWidth, Width, SegmentCount} = get_size(Size), Lv1Width = Width * ?HASH_SIZE * 8, Lv1Init = <<0:Lv1Width/integer>>, - Lv2SegBinSize = Width * ?HASH_SIZE * 8, - Lv2SegBinInit = <<0:Lv2SegBinSize/integer>>, - Lv2Init = array:new([{size, Width}, {default, Lv2SegBinInit}]), + Lv2Init = array:new([{size, Width}, {default, ?EMPTY}]), #tictactree{treeID = TreeID, size = Size, width = Width, @@ -190,7 +189,7 @@ add_kv(TicTacTree, Key, Value, BinExtractFun, Exportable) -> Level2BytePos = ?HASH_SIZE * Level2Pos, Level1BytePos = ?HASH_SIZE * Level1Pos, - Level2 = array:get(Level1Pos, TicTacTree#tictactree.level2), + Level2 = get_level2(TicTacTree, Level1Pos), HashIntLength = ?HASH_SIZE * 8, < fetch_leaves(TicTacTree, BranchList) -> MapFun = fun(Idx) -> - {Idx, array:get(Idx, TicTacTree#tictactree.level2)} + {Idx, get_level2(TicTacTree, Idx)} end, lists:map(MapFun, BranchList). @@ -277,8 +276,8 @@ merge_trees(TreeA, TreeB) -> MergeFun = fun(SQN, MergeL2) -> - L2A = array:get(SQN, TreeA#tictactree.level2), - L2B = array:get(SQN, TreeB#tictactree.level2), + L2A = get_level2(TreeA, SQN), + L2B = get_level2(TreeB, SQN), NewLevel2 = merge_binaries(L2A, L2B), array:set(SQN, NewLevel2, MergeL2) end, @@ -322,6 +321,15 @@ tictac_hash(BinKey, BinVal, false) -> %%% Internal functions %%%============================================================================ +get_level2(TicTacTree, L1Pos) -> + case array:get(L1Pos, TicTacTree#tictactree.level2) of + ?EMPTY -> + Lv2SegBinSize = TicTacTree#tictactree.width * ?HASH_SIZE * 8, + <<0:Lv2SegBinSize/integer>>; + SrcL2 -> + SrcL2 + end. + get_size(Size) -> case Size of xxsmall -> @@ -361,7 +369,7 @@ checktree(<<>>, TicTacTree, Counter) -> checktree(Level1Bin, TicTacTree, Counter) -> BitSize = ?HASH_SIZE * 8, <> = Level1Bin, - L2Bin = array:get(Counter, TicTacTree#tictactree.level2), + L2Bin = get_level2(TicTacTree, Counter), true = TopHash == segmentsummarise(L2Bin, 0), checktree(Tail, TicTacTree, Counter + 1). From fd4fbf7ea87bbf55712c92ca5f777d24a35ea999 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 29 Sep 2017 15:28:17 +0100 Subject: [PATCH 07/10] Keep trees empty on merge Done't blow out a tree unnecessarily on merge --- src/leveled_tictac.erl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/leveled_tictac.erl b/src/leveled_tictac.erl index 2edd636..39bf983 100644 --- a/src/leveled_tictac.erl +++ b/src/leveled_tictac.erl @@ -278,7 +278,14 @@ merge_trees(TreeA, TreeB) -> fun(SQN, MergeL2) -> L2A = get_level2(TreeA, SQN), L2B = get_level2(TreeB, SQN), - NewLevel2 = merge_binaries(L2A, L2B), + BothEmpty = (L2A == ?EMPTY) and (L2B == ?EMPTY), + NewLevel2 = + case BothEmpty of + true -> + ?EMPTY; + false -> + merge_binaries(L2A, L2B) + end, array:set(SQN, NewLevel2, MergeL2) end, NewLevel2 = lists:foldl(MergeFun, From 7912742a84fb4b42e76b3901362651ded42b1ebc Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 29 Sep 2017 17:35:15 +0100 Subject: [PATCH 08/10] Add valid_size/1 --- src/leveled_tictac.erl | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/leveled_tictac.erl b/src/leveled_tictac.erl index 39bf983..4bf83fb 100644 --- a/src/leveled_tictac.erl +++ b/src/leveled_tictac.erl @@ -66,7 +66,8 @@ get_segment/2, tictac_hash/3, export_tree/1, - import_tree/1 + import_tree/1, + valid_size/1 ]). @@ -80,6 +81,7 @@ -define(LARGE, {10, 1024, 1024 * 1024}). -define(XLARGE, {11, 2048, 2048 * 2048}). -define(EMPTY, <<0:8/integer>>). +-define(VALID_SIZES, [xxsmall, xsmall, small, medium, large, xlarge]). -record(tictactree, {treeID :: any(), size :: xxsmall|xsmall|small|medium|large|xlarge, @@ -96,6 +98,12 @@ %%% External functions %%%============================================================================ +-spec valid_size(any()) -> boolean(). +%% @doc +%% For validation of input +valid_size(Size) -> + lists:member(Size, ?VALID_SIZES). + -spec new_tree(any()) -> tictactree(). %% @doc %% Create a new tree, zeroed out. @@ -142,7 +150,7 @@ import_tree(ExportedTree) -> L1Bin = base64:decode(L1Base64), Sizes = lists:map(fun(SizeTag) -> {SizeTag, element(2, get_size(SizeTag))} end, - [xxsmall, xsmall, small, medium, large, xlarge]), + ?VALID_SIZES), Width = byte_size(L1Bin) div ?HASH_SIZE, {Size, Width} = lists:keyfind(Width, 2, Sizes), {BitWidth, Width, SegmentCount} = get_size(Size), @@ -414,6 +422,7 @@ simple_bysize_test_allsizes() -> simple_test_withsize(xlarge). simple_test_withsize(Size) -> + ?assertMatch(true, valid_size(Size)), BinFun = fun(K, V) -> {term_to_binary(K), term_to_binary(V)} end, K1 = {o, "B1", "K1", null}, From 0c5f5cdb65879731a094a90053887286d35ec137 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 6 Oct 2017 15:02:14 +0100 Subject: [PATCH 09/10] Add key range to fold_heads queries --- src/leveled_bookie.erl | 48 +++++++++++++++++++++++++++++--- test/end_to_end/tictac_SUITE.erl | 3 ++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index eb422e1..8886a81 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -604,10 +604,11 @@ handle_call({return_folder, FolderType}, _From, State) -> CheckPresence, SnapPreFold), State}; - {foldheads_bybucket, Tag, Bucket, FoldHeadsFun, + {foldheads_bybucket, Tag, Bucket, KeyRange, FoldHeadsFun, CheckPresence, SnapPreFold} -> {reply, foldheads_bybucket(State, Tag, Bucket, + KeyRange, FoldHeadsFun, CheckPresence, SnapPreFold), @@ -1000,9 +1001,20 @@ foldobjects_bybucket(State, Tag, Bucket, FoldObjectsFun) -> foldobjects(State, Tag, StartKey, EndKey, FoldObjectsFun, false, true). -foldheads_bybucket(State, Tag, Bucket, FoldHeadsFun, CheckPresence, SnapPreFold) -> - StartKey = leveled_codec:to_ledgerkey(Bucket, null, Tag), - EndKey = leveled_codec:to_ledgerkey(Bucket, null, Tag), +foldheads_bybucket(State, Tag, Bucket, KeyRange, FoldHeadsFun, + CheckPresence, SnapPreFold) -> + {StartKey, EndKey} = + case KeyRange of + all -> + {leveled_codec:to_ledgerkey(Bucket, null, Tag), + leveled_codec:to_ledgerkey(Bucket, null, Tag)}; + {StartTerm, <<"$all">>} -> + {leveled_codec:to_ledgerkey(Bucket, StartTerm, Tag), + leveled_codec:to_ledgerkey(Bucket, null, Tag)}; + {StartTerm, EndTerm} -> + {leveled_codec:to_ledgerkey(Bucket, StartTerm, Tag), + leveled_codec:to_ledgerkey(Bucket, EndTerm, Tag)} + end, foldobjects(State, Tag, StartKey, EndKey, FoldHeadsFun, {true, CheckPresence}, SnapPreFold). @@ -1876,6 +1888,7 @@ foldobjects_vs_foldheads_bybucket_testto() -> {foldheads_bybucket, ?STD_TAG, "BucketA", + all, FoldHeadsFun, true, true}), @@ -1885,15 +1898,42 @@ foldobjects_vs_foldheads_bybucket_testto() -> {foldheads_bybucket, ?STD_TAG, "BucketB", + all, FoldHeadsFun, false, false}), KeyHashList2B = HTFolder2B(), + ?assertMatch(true, lists:usort(KeyHashList1A) == lists:usort(KeyHashList2A)), ?assertMatch(true, lists:usort(KeyHashList1B) == lists:usort(KeyHashList2B)), + {async, HTFolder2C} = + book_returnfolder(Bookie1, + {foldheads_bybucket, + ?STD_TAG, + "BucketB", + {"Key", <<"$all">>}, + FoldHeadsFun, + false, + false}), + KeyHashList2C = HTFolder2C(), + {async, HTFolder2D} = + book_returnfolder(Bookie1, + {foldheads_bybucket, + ?STD_TAG, + "BucketB", + {"Key", "Keyzzzzz"}, + FoldHeadsFun, + false, + false}), + KeyHashList2D = HTFolder2D(), + ?assertMatch(true, + lists:usort(KeyHashList2B) == lists:usort(KeyHashList2C)), + ?assertMatch(true, + lists:usort(KeyHashList2B) == lists:usort(KeyHashList2D)), + ok = book_close(Bookie1), reset_filestructure(). diff --git a/test/end_to_end/tictac_SUITE.erl b/test/end_to_end/tictac_SUITE.erl index 2215b6b..1a3c9f8 100644 --- a/test/end_to_end/tictac_SUITE.erl +++ b/test/end_to_end/tictac_SUITE.erl @@ -129,6 +129,7 @@ many_put_compare(_Config) -> FoldQ0 = {foldheads_bybucket, o_rkv, "Bucket", + all, {FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)}, false, true}, {async, TreeAObjFolder0} = @@ -143,6 +144,7 @@ many_put_compare(_Config) -> FoldQ1 = {foldheads_bybucket, o_rkv, "Bucket", + all, {FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)}, true, true}, {async, TreeAObjFolder1} = @@ -170,6 +172,7 @@ many_put_compare(_Config) -> AltFoldQ0 = {foldheads_bybucket, o_rkv, "Bucket", + all, {AltFoldObjectsFun, leveled_tictac:new_tree(0, TreeSize)}, false, true}, From 5c8eea3f0e960d74b1af12fd09b9124e9497aa2a Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 6 Oct 2017 15:07:36 +0100 Subject: [PATCH 10/10] Extend foldheads_bybucket test Now explicitly checking key ranges --- src/leveled_bookie.erl | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 8886a81..53f931e 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -1933,6 +1933,33 @@ foldobjects_vs_foldheads_bybucket_testto() -> lists:usort(KeyHashList2B) == lists:usort(KeyHashList2C)), ?assertMatch(true, lists:usort(KeyHashList2B) == lists:usort(KeyHashList2D)), + + {async, HTFolder2E} = + book_returnfolder(Bookie1, + {foldheads_bybucket, + ?STD_TAG, + "BucketB", + {"Key", "Key4zzzz"}, + FoldHeadsFun, + false, + false}), + KeyHashList2E = HTFolder2E(), + {async, HTFolder2F} = + book_returnfolder(Bookie1, + {foldheads_bybucket, + ?STD_TAG, + "BucketB", + {"Key5", <<"all">>}, + FoldHeadsFun, + false, + false}), + KeyHashList2F = HTFolder2F(), + + ?assertMatch(true, length(KeyHashList2E) > 0), + ?assertMatch(true, length(KeyHashList2F) > 0), + ?assertMatch(true, + lists:usort(KeyHashList2B) == + lists:usort(KeyHashList2E ++ KeyHashList2F)), ok = book_close(Bookie1), reset_filestructure().