From eeeee07081155ad61fe86b6f9fb2c7f797cf18fe Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 4 Nov 2016 14:23:37 +0000 Subject: [PATCH] Fold Objects - Check values test Test that summed values in fold objects before and after restart --- src/leveled_bookie.erl | 10 +++++-- test/end_to_end/iterator_SUITE.erl | 42 +++++++++++++++++++++++++----- test/end_to_end/testutil.erl | 4 +++ 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index dec8ceb..82abb86 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -528,16 +528,22 @@ foldobjects(State, Tag, StartKey, EndKey, FoldObjectsFun) -> {ok, {LedgerSnapshot, LedgerCache}, JournalSnapshot} = snapshot_store(State, store), + {FoldFun, InitAcc} = case is_tuple(FoldObjectsFun) of + true -> + FoldObjectsFun; + false -> + {FoldObjectsFun, []} + end, Folder = fun() -> leveled_log:log("B0004", [gb_trees:size(LedgerCache)]), ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, LedgerCache), - AccFun = accumulate_objects(FoldObjectsFun, JournalSnapshot, Tag), + AccFun = accumulate_objects(FoldFun, JournalSnapshot, Tag), Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, StartKey, EndKey, AccFun, - []), + InitAcc), ok = leveled_penciller:pcl_close(LedgerSnapshot), ok = leveled_inker:ink_close(JournalSnapshot), Acc diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index 73e9ab4..78d19bd 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -11,27 +11,30 @@ rotating_objects/1]). all() -> [ - small_load_with2i, - query_count, - rotating_objects + small_load_with2i %, + % query_count, + % rotating_objects ]. small_load_with2i(_Config) -> RootPath = testutil:reset_filestructure(), StartOpts1 = [{root_path, RootPath}, - {max_journalsize, 50000000}], + {max_journalsize, 5000000}], + % low journal size to make sure > 1 created {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), testutil:check_forobject(Bookie1, TestObject), testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"), testutil:check_forobject(Bookie1, TestObject), + ObjectGen = testutil:get_compressiblevalue_andinteger(), + IndexGen = testutil:get_randomindexes_generator(8), ObjL1 = testutil:generate_objects(10000, uuid, [], - testutil:get_compressiblevalue(), - testutil:get_randomindexes_generator(8)), + ObjectGen, + IndexGen), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, ObjL1), @@ -74,7 +77,34 @@ small_load_with2i(_Config) -> true = 9900 == length(KeyHashList2), true = 9900 == length(KeyHashList3), + SumIntegerFun = fun(_B, _K, V, Acc) -> + [C] = V#r_object.contents, + {I, _Bin} = C#r_content.value, + Acc + I + end, + {async, Sum1} = leveled_bookie:book_returnfolder(Bookie1, + {foldobjects_bybucket, + ?RIAK_TAG, + "Bucket", + {SumIntegerFun, + 0}}), + Total1 = Sum1(), + true = Total1 > 100000, + ok = leveled_bookie:book_close(Bookie1), + + {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), + + {async, Sum2} = leveled_bookie:book_returnfolder(Bookie2, + {foldobjects_bybucket, + ?RIAK_TAG, + "Bucket", + {SumIntegerFun, + 0}}), + Total2 = Sum2(), + true = Total2 == Total1, + + ok = leveled_bookie:book_close(Bookie2), testutil:reset_filestructure(). diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index 9d9f278..755cf88 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -21,6 +21,7 @@ get_key/1, get_value/1, get_compressiblevalue/0, + get_compressiblevalue_andinteger/0, get_randomindexes_generator/1, name_list/0, load_objects/5, @@ -147,6 +148,9 @@ generate_compressibleobjects(Count, KeyNumber) -> generate_objects(Count, KeyNumber, [], V). +get_compressiblevalue_andinteger() -> + {random:uniform(1000), get_compressiblevalue()}. + get_compressiblevalue() -> S1 = "111111111111111", S2 = "222222222222222",