From 0d8ab0899e808aacfcd89e93fecdd853c66cba60 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 23 May 2017 11:59:44 +0100 Subject: [PATCH] Add test for is_empty Bucket listing didn't care if keys were active - now does. --- src/leveled_bookie.erl | 55 ++++++++++++++++++----------- test/end_to_end/basic_SUITE.erl | 62 +++++++++++++++++++++++++++++++-- 2 files changed, 94 insertions(+), 23 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 970b54c..d12687c 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -179,7 +179,7 @@ book_start(Opts) -> %% @doc Put an object with an expiry time %% %% Put an item in the store but with a Time To Live - the time when the object -%% should expire, in gregorian_sconds (add the required number of seconds to +%% should expire, in gregorian_seconds (add the required number of seconds to %% leveled_codec:integer_time/1). %% %% There exists the possibility of per object expiry times, not just whole @@ -729,6 +729,7 @@ binary_bucketlist(State, Tag, {FoldBucketsFun, InitAcc}) -> no_lookup), Folder = fun() -> BucketAcc = get_nextbucket(null, + null, Tag, LedgerSnapshot, []), @@ -739,26 +740,40 @@ binary_bucketlist(State, Tag, {FoldBucketsFun, InitAcc}) -> end, {async, Folder}. -get_nextbucket(NextBucket, Tag, LedgerSnapshot, BKList) -> - StartKey = leveled_codec:to_ledgerkey(NextBucket, null, Tag), +get_nextbucket(NextBucket, NextKey, Tag, LedgerSnapshot, BKList) -> + Now = leveled_codec:integer_now(), + StartKey = leveled_codec:to_ledgerkey(NextBucket, NextKey, Tag), EndKey = leveled_codec:to_ledgerkey(null, null, Tag), - ExtractFun = fun(LK, _V, _Acc) -> leveled_codec:from_ledgerkey(LK) end, - BK = leveled_penciller:pcl_fetchnextkey(LedgerSnapshot, - StartKey, - EndKey, - ExtractFun, - null), - case BK of + ExtractFun = + fun(LK, V, _Acc) -> + {leveled_codec:from_ledgerkey(LK), V} + end, + R = leveled_penciller:pcl_fetchnextkey(LedgerSnapshot, + StartKey, + EndKey, + ExtractFun, + null), + case R of null -> leveled_log:log("B0008",[]), BKList; - {B, K} when is_binary(B) -> - leveled_log:log("B0009",[B]), - get_nextbucket(<>, - Tag, - LedgerSnapshot, - [{B, K}|BKList]); - NB -> + {{B, K}, V} when is_binary(B), is_binary(K) -> + case leveled_codec:is_active({B, K}, V, Now) of + true -> + leveled_log:log("B0009",[B]), + get_nextbucket(<>, + null, + Tag, + LedgerSnapshot, + [{B, K}|BKList]); + false -> + get_nextbucket(B, + <>, + Tag, + LedgerSnapshot, + BKList) + end; + {NB, _V} -> leveled_log:log("B0010",[NB]), [] end. @@ -1308,11 +1323,11 @@ maybe_withjitter(CacheSize, MaxCacheSize) -> -load_fun(KeyInLedger, ValueInLedger, _Position, Acc0, ExtractFun) -> +load_fun(KeyInJournal, ValueInJournal, _Position, Acc0, ExtractFun) -> {MinSQN, MaxSQN, OutputTree} = Acc0, - {SQN, Type, PK} = KeyInLedger, + {SQN, Type, PK} = KeyInJournal, % VBin may already be a term - {VBin, VSize} = ExtractFun(ValueInLedger), + {VBin, VSize} = ExtractFun(ValueInJournal), {Obj, IndexSpecs} = leveled_codec:split_inkvalue(VBin), case SQN of SQN when SQN < MinSQN -> diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 37424c2..932e062 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -8,7 +8,8 @@ fetchput_snapshot/1, load_and_count/1, load_and_count_withdelete/1, - space_clear_ondelete/1 + space_clear_ondelete/1, + is_empty_test/1 ]). all() -> [ @@ -18,7 +19,8 @@ all() -> [ fetchput_snapshot, load_and_count, load_and_count_withdelete, - space_clear_ondelete + space_clear_ondelete, + is_empty_test ]. @@ -591,4 +593,58 @@ space_clear_ondelete(_Config) -> true = length(FNsD_L) < length(FNsA_L), true = length(FNsD_L) < length(FNsB_L), true = length(FNsD_L) < length(FNsC_L), - true = length(FNsD_L) == 0. \ No newline at end of file + true = length(FNsD_L) == 0. + + + +is_empty_test(_Config) -> + RootPath = testutil:reset_filestructure(), + StartOpts1 = [{root_path, RootPath}, + {sync_strategy, testutil:sync_strategy()}], + {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), + + {B1, K1, V1, Spec, MD} = {term_to_binary("Bucket1"), + term_to_binary("Key1"), + "Value1", + [], + [{"MDK1", "MDV1"}]}, + {TestObject1, TestSpec1} = + testutil:generate_testobject(B1, K1, V1, Spec, MD), + {B1, K2, V2, Spec, MD} = {term_to_binary("Bucket1"), + term_to_binary("Key2"), + "Value2", + [], + [{"MDK1", "MDV1"}]}, + {TestObject2, TestSpec2} = + testutil:generate_testobject(B1, K2, V2, Spec, MD), + {B2, K3, V3, Spec, MD} = {term_to_binary("Bucket2"), + term_to_binary("Key3"), + "Value3", + [], + [{"MDK1", "MDV1"}]}, + {TestObject3, TestSpec3} = + testutil:generate_testobject(B2, K3, V3, Spec, MD), + ok = testutil:book_riakput(Bookie1, TestObject1, TestSpec1), + ok = testutil:book_riakput(Bookie1, TestObject2, TestSpec2), + ok = testutil:book_riakput(Bookie1, TestObject3, TestSpec3), + + FoldBucketsFun = fun(B, Acc) -> sets:add_element(B, Acc) end, + BucketListQuery = {binary_bucketlist, + ?RIAK_TAG, + {FoldBucketsFun, sets:new()}}, + {async, BL} = leveled_bookie:book_returnfolder(Bookie1, BucketListQuery), + true = sets:size(BL()) == 2, + + ok = leveled_bookie:book_put(Bookie1, B2, K3, delete, [], ?RIAK_TAG), + {async, BLpd1} = leveled_bookie:book_returnfolder(Bookie1, BucketListQuery), + true = sets:size(BLpd1()) == 1, + + ok = leveled_bookie:book_put(Bookie1, B1, K2, delete, [], ?RIAK_TAG), + {async, BLpd2} = leveled_bookie:book_returnfolder(Bookie1, BucketListQuery), + true = sets:size(BLpd2()) == 1, + + ok = leveled_bookie:book_put(Bookie1, B1, K1, delete, [], ?RIAK_TAG), + {async, BLpd3} = leveled_bookie:book_returnfolder(Bookie1, BucketListQuery), + true = sets:size(BLpd3()) == 0, + + ok = leveled_bookie:book_close(Bookie1).