Switch to binary format Riak object
Initial change to try and test assuming that leveled received the binary format of Riak objects (and parses that for metadata).
This commit is contained in:
martinsumner
parent
b0a515553f
commit
e8c1d39df9
6 changed files with 212 additions and 177 deletions
|
|
@ -968,112 +968,6 @@ generate_multiple_objects(Count, KeyNumber, ObjL) ->
|
|||
KeyNumber + 1,
|
||||
ObjL ++ [{Key, Value, IndexSpec}]).
|
||||
|
||||
|
||||
generate_multiple_robjects(Count, KeyNumber) ->
|
||||
generate_multiple_robjects(Count, KeyNumber, []).
|
||||
|
||||
generate_multiple_robjects(0, _KeyNumber, ObjL) ->
|
||||
ObjL;
|
||||
generate_multiple_robjects(Count, KeyNumber, ObjL) ->
|
||||
Obj = {"Bucket",
|
||||
"Key" ++ integer_to_list(KeyNumber),
|
||||
crypto:rand_bytes(1024),
|
||||
[],
|
||||
[{"MDK", "MDV" ++ integer_to_list(KeyNumber)},
|
||||
{"MDK2", "MDV" ++ integer_to_list(KeyNumber)}]},
|
||||
{B1, K1, V1, Spec1, MD} = Obj,
|
||||
Content = #r_content{metadata=MD, value=V1},
|
||||
Obj1 = #r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]},
|
||||
generate_multiple_robjects(Count - 1, KeyNumber + 1, ObjL ++ [{Obj1, Spec1}]).
|
||||
|
||||
|
||||
single_key_test() ->
|
||||
RootPath = reset_filestructure(),
|
||||
{ok, Bookie1} = book_start([{root_path, RootPath}]),
|
||||
{B1, K1, V1, Spec1, MD} = {"Bucket1",
|
||||
"Key1",
|
||||
"Value1",
|
||||
[],
|
||||
{"MDK1", "MDV1"}},
|
||||
Content = #r_content{metadata=MD, value=V1},
|
||||
Object = #r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]},
|
||||
ok = book_put(Bookie1, B1, K1, Object, Spec1, ?RIAK_TAG),
|
||||
{ok, F1} = book_get(Bookie1, B1, K1, ?RIAK_TAG),
|
||||
?assertMatch(F1, Object),
|
||||
ok = book_close(Bookie1),
|
||||
{ok, Bookie2} = book_start([{root_path, RootPath}]),
|
||||
{ok, F2} = book_get(Bookie2, B1, K1, ?RIAK_TAG),
|
||||
?assertMatch(F2, Object),
|
||||
ok = book_close(Bookie2),
|
||||
reset_filestructure().
|
||||
|
||||
multi_key_test() ->
|
||||
RootPath = reset_filestructure(),
|
||||
{ok, Bookie1} = book_start([{root_path, RootPath}]),
|
||||
{B1, K1, V1, Spec1, MD1} = {"Bucket",
|
||||
"Key1",
|
||||
"Value1",
|
||||
[],
|
||||
{"MDK1", "MDV1"}},
|
||||
C1 = #r_content{metadata=MD1, value=V1},
|
||||
Obj1 = #r_object{bucket=B1, key=K1, contents=[C1], vclock=[{'a',1}]},
|
||||
{B2, K2, V2, Spec2, MD2} = {"Bucket",
|
||||
"Key2",
|
||||
"Value2",
|
||||
[],
|
||||
{"MDK2", "MDV2"}},
|
||||
C2 = #r_content{metadata=MD2, value=V2},
|
||||
Obj2 = #r_object{bucket=B2, key=K2, contents=[C2], vclock=[{'a',1}]},
|
||||
ok = book_put(Bookie1, B1, K1, Obj1, Spec1, ?RIAK_TAG),
|
||||
ObjL1 = generate_multiple_robjects(20, 3),
|
||||
SW1 = os:timestamp(),
|
||||
lists:foreach(fun({O, S}) ->
|
||||
{B, K} = leveled_codec:riakto_keydetails(O),
|
||||
ok = book_put(Bookie1, B, K, O, S, ?RIAK_TAG)
|
||||
end,
|
||||
ObjL1),
|
||||
io:format("PUT of 20 objects completed in ~w microseconds~n",
|
||||
[timer:now_diff(os:timestamp(),SW1)]),
|
||||
ok = book_put(Bookie1, B2, K2, Obj2, Spec2, ?RIAK_TAG),
|
||||
{ok, F1A} = book_get(Bookie1, B1, K1, ?RIAK_TAG),
|
||||
?assertMatch(F1A, Obj1),
|
||||
{ok, F2A} = book_get(Bookie1, B2, K2, ?RIAK_TAG),
|
||||
?assertMatch(F2A, Obj2),
|
||||
ObjL2 = generate_multiple_robjects(20, 23),
|
||||
SW2 = os:timestamp(),
|
||||
lists:foreach(fun({O, S}) ->
|
||||
{B, K} = leveled_codec:riakto_keydetails(O),
|
||||
ok = book_put(Bookie1, B, K, O, S, ?RIAK_TAG)
|
||||
end,
|
||||
ObjL2),
|
||||
io:format("PUT of 20 objects completed in ~w microseconds~n",
|
||||
[timer:now_diff(os:timestamp(),SW2)]),
|
||||
{ok, F1B} = book_get(Bookie1, B1, K1, ?RIAK_TAG),
|
||||
?assertMatch(F1B, Obj1),
|
||||
{ok, F2B} = book_get(Bookie1, B2, K2, ?RIAK_TAG),
|
||||
?assertMatch(F2B, Obj2),
|
||||
ok = book_close(Bookie1),
|
||||
% Now reopen the file, and confirm that a fetch is still possible
|
||||
{ok, Bookie2} = book_start([{root_path, RootPath}]),
|
||||
{ok, F1C} = book_get(Bookie2, B1, K1, ?RIAK_TAG),
|
||||
?assertMatch(F1C, Obj1),
|
||||
{ok, F2C} = book_get(Bookie2, B2, K2, ?RIAK_TAG),
|
||||
?assertMatch(F2C, Obj2),
|
||||
ObjL3 = generate_multiple_robjects(20, 43),
|
||||
SW3 = os:timestamp(),
|
||||
lists:foreach(fun({O, S}) ->
|
||||
{B, K} = leveled_codec:riakto_keydetails(O),
|
||||
ok = book_put(Bookie2, B, K, O, S, ?RIAK_TAG)
|
||||
end,
|
||||
ObjL3),
|
||||
io:format("PUT of 20 objects completed in ~w microseconds~n",
|
||||
[timer:now_diff(os:timestamp(),SW3)]),
|
||||
{ok, F1D} = book_get(Bookie2, B1, K1, ?RIAK_TAG),
|
||||
?assertMatch(F1D, Obj1),
|
||||
{ok, F2D} = book_get(Bookie2, B2, K2, ?RIAK_TAG),
|
||||
?assertMatch(F2D, Obj2),
|
||||
ok = book_close(Bookie2),
|
||||
reset_filestructure().
|
||||
|
||||
ttl_test() ->
|
||||
RootPath = reset_filestructure(),
|
||||
|
|
|
|||
|
|
@ -60,9 +60,12 @@
|
|||
get_size/2,
|
||||
get_keyandhash/2,
|
||||
convert_indexspecs/5,
|
||||
riakto_keydetails/1,
|
||||
generate_uuid/0,
|
||||
integer_now/0]).
|
||||
integer_now/0,
|
||||
riak_extract_metadata/2]).
|
||||
|
||||
-define(V1_VERS, 1).
|
||||
-define(MAGIC, 53). % riak_kv -> riak_object
|
||||
|
||||
|
||||
%% Credit to
|
||||
|
|
@ -325,44 +328,67 @@ get_keyandhash(LK, Value) ->
|
|||
|
||||
|
||||
build_metadata_object(PrimaryKey, MD) ->
|
||||
{Tag, Bucket, Key, null} = PrimaryKey,
|
||||
{Tag, _Bucket, _Key, null} = PrimaryKey,
|
||||
case Tag of
|
||||
?RIAK_TAG ->
|
||||
riak_metadata_object(Bucket, Key, MD);
|
||||
{SibMetaBinList, Vclock, _Hash, _Size} = MD,
|
||||
riak_metadata_to_binary(Vclock, SibMetaBinList);
|
||||
?STD_TAG ->
|
||||
MD
|
||||
end.
|
||||
|
||||
|
||||
|
||||
|
||||
riak_metadata_object(Bucket, Key, MD) ->
|
||||
{RMD, VC, _Hash, _Size} = MD,
|
||||
Contents = lists:foldl(fun(X, Acc) -> Acc ++ [#r_content{metadata=X}] end,
|
||||
[],
|
||||
RMD),
|
||||
#r_object{contents=Contents, bucket=Bucket, key=Key, vclock=VC}.
|
||||
|
||||
riak_extract_metadata(delete, Size) ->
|
||||
{delete, null, null, Size};
|
||||
riak_extract_metadata(Obj, Size) ->
|
||||
{get_metadatas(Obj), vclock(Obj), riak_hash(Obj), Size}.
|
||||
riak_extract_metadata(ObjBin, Size) ->
|
||||
{Vclock, SibMetaBinList} = riak_metadata_from_binary(ObjBin),
|
||||
{SibMetaBinList, Vclock, erlang:phash2(ObjBin), Size}.
|
||||
|
||||
riak_hash(Obj=#r_object{}) ->
|
||||
Vclock = vclock(Obj),
|
||||
UpdObj = set_vclock(Obj, lists:sort(Vclock)),
|
||||
erlang:phash2(term_to_binary(UpdObj)).
|
||||
%% <<?MAGIC:8/integer, ?V1_VERS:8/integer, VclockLen:32/integer,
|
||||
%%% VclockBin/binary, SibCount:32/integer, SibsBin/binary>>.
|
||||
|
||||
riakto_keydetails(Object) ->
|
||||
{Object#r_object.bucket, Object#r_object.key}.
|
||||
riak_metadata_to_binary(Vclock, SibMetaBinList) ->
|
||||
VclockBin = term_to_binary(Vclock),
|
||||
VclockLen = byte_size(VclockBin),
|
||||
SibCount = length(SibMetaBinList),
|
||||
SibsBin = slimbin_contents(SibMetaBinList),
|
||||
<<?MAGIC:8/integer, ?V1_VERS:8/integer, VclockLen:32/integer,
|
||||
VclockBin:VclockLen/binary, SibCount:32/integer, SibsBin/binary>>.
|
||||
|
||||
% Fixes the value length for each sibling to be zero, and so includes no value
|
||||
slimbin_content(MetaBin) ->
|
||||
MetaLen = byte_size(MetaBin),
|
||||
<<0:32/integer, MetaLen:32/integer, MetaBin:MetaLen/binary>>.
|
||||
|
||||
get_metadatas(#r_object{contents=Contents}) ->
|
||||
[Content#r_content.metadata || Content <- Contents].
|
||||
|
||||
set_vclock(Object=#r_object{}, VClock) -> Object#r_object{vclock=VClock}.
|
||||
|
||||
vclock(#r_object{vclock=VClock}) -> VClock.
|
||||
slimbin_contents(SibMetaBinList) ->
|
||||
F = fun(MetaBin, Acc) ->
|
||||
<<Acc/binary, (slimbin_content(MetaBin))/binary>>
|
||||
end,
|
||||
lists:foldl(F, <<>>, SibMetaBinList).
|
||||
|
||||
riak_metadata_from_binary(V1Binary) ->
|
||||
<<?MAGIC:8/integer, ?V1_VERS:8/integer, VclockLen:32/integer,
|
||||
Rest/binary>> = V1Binary,
|
||||
<<VclockBin:VclockLen/binary, SibCount:32/integer, SibsBin/binary>> = Rest,
|
||||
SibMetaBinList =
|
||||
case SibCount of
|
||||
0 ->
|
||||
[];
|
||||
SC when is_integer(SC) ->
|
||||
get_metadata_from_siblings(SibsBin, SibCount, [])
|
||||
end,
|
||||
{binary_to_term(VclockBin), SibMetaBinList}.
|
||||
|
||||
get_metadata_from_siblings(<<>>, 0, SibMetaBinList) ->
|
||||
SibMetaBinList;
|
||||
get_metadata_from_siblings(<<ValLen:32/integer, Rest0/binary>>,
|
||||
SibCount,
|
||||
SibMetaBinList) ->
|
||||
<<_ValBin:ValLen/binary, MetaLen:32/integer, Rest1/binary>> = Rest0,
|
||||
<<MetaBin:MetaLen/binary, Rest2/binary>> = Rest1,
|
||||
get_metadata_from_siblings(Rest2,
|
||||
SibCount - 1,
|
||||
[MetaBin|SibMetaBinList]).
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue