umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Switch to binary metadata

Browse source 
Trya nd maintain binary format when stored in Ledger so less
swapping/changing as added and removed.
This commit is contained in:
martinsumner 2017-04-04 10:02:35 +00:00
parent 1146e843bc
commit 400f65f557
2 changed files with 20 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

49
src/leveled_codec.erl
View file

@ -445,57 +445,44 @@ build_metadata_object(PrimaryKey, MD) ->
riak_extract_metadata(delete, Size) -> riak_extract_metadata(delete, Size) ->
{delete, null, null, Size}; {delete, null, null, Size};
riak_extract_metadata(ObjBin, Size) -> riak_extract_metadata(ObjBin, Size) ->
{Vclock, SibData} = riak_metadata_from_binary(ObjBin), {Vclock, SibBin} = riak_metadata_from_binary(ObjBin),
{SibData, Vclock, erlang:phash2(ObjBin), Size}. {SibBin, Vclock, erlang:phash2(ObjBin), Size}.
%% <<?MAGIC:8/integer, ?V1_VERS:8/integer, VclockLen:32/integer, %% <<?MAGIC:8/integer, ?V1_VERS:8/integer, VclockLen:32/integer,
%%% VclockBin/binary, SibCount:32/integer, SibsBin/binary>>. %%% VclockBin/binary, SibCount:32/integer, SibsBin/binary>>.
riak_metadata_to_binary(Vclock, SibData) -> riak_metadata_to_binary(VclockBin, SibMetaBin) ->
VclockBin = term_to_binary(Vclock),
VclockLen = byte_size(VclockBin), VclockLen = byte_size(VclockBin),
% <<?MAGIC:8/integer, ?V1_VERS:8/integer, VclockLen:32/integer, <<?MAGIC:8/integer, ?V1_VERS:8/integer,
% VclockBin:VclockLen/binary, SibData:32/integer>>. VclockLen:32/integer, VclockBin/binary,
SibCount = length(SibData), SibMetaBin/binary>>.
SibsBin = slimbin_contents(SibData),
<<?MAGIC:8/integer, ?V1_VERS:8/integer, VclockLen:32/integer,
VclockBin:VclockLen/binary, SibCount:32/integer, SibsBin/binary>>.
riak_metadata_from_binary(V1Binary) -> riak_metadata_from_binary(V1Binary) ->
<<?MAGIC:8/integer, ?V1_VERS:8/integer, VclockLen:32/integer, <<?MAGIC:8/integer, ?V1_VERS:8/integer, VclockLen:32/integer,
Rest/binary>> = V1Binary, Rest/binary>> = V1Binary,
% Just grab the Sibling count and not the full metadata
% <<VclockBin:VclockLen/binary, SibCount:32/integer, _Rest/binary>> = Rest,
% {binary_to_term(VclockBin), SibCount}.
<<VclockBin:VclockLen/binary, SibCount:32/integer, SibsBin/binary>> = Rest, <<VclockBin:VclockLen/binary, SibCount:32/integer, SibsBin/binary>> = Rest,
SibMetaBinList = SibMetaBin =
case SibCount of case SibCount of
SC when is_integer(SC) -> SC when is_integer(SC) ->
get_metadata_from_siblings(SibsBin, SibCount, []) get_metadata_from_siblings(SibsBin,
SibCount,
<<SibCount:32/integer>>)
end, end,
{binary_to_term(VclockBin), SibMetaBinList}. {VclockBin, SibMetaBin}.
% Fixes the value length for each sibling to be zero, and so includes no value get_metadata_from_siblings(<<>>, 0, SibMetaBin) ->
slimbin_content(MetaBin) -> SibMetaBin;
MetaLen = byte_size(MetaBin),
<<0:32/integer, MetaLen:32/integer, MetaBin:MetaLen/binary>>.
slimbin_contents(SibMetaBinList) ->
F = fun(MetaBin, Acc) ->
<<Acc/binary, (slimbin_content(MetaBin))/binary>>
end,
lists:foldl(F, <<>>, SibMetaBinList).
get_metadata_from_siblings(<<>>, 0, SibMetaBinList) ->
SibMetaBinList;
get_metadata_from_siblings(<<ValLen:32/integer, Rest0/binary>>, get_metadata_from_siblings(<<ValLen:32/integer, Rest0/binary>>,
SibCount, SibCount,
SibMetaBinList) -> SibMetaBin) ->
<<_ValBin:ValLen/binary, MetaLen:32/integer, Rest1/binary>> = Rest0, <<_ValBin:ValLen/binary, MetaLen:32/integer, Rest1/binary>> = Rest0,
<<MetaBin:MetaLen/binary, Rest2/binary>> = Rest1, <<MetaBin:MetaLen/binary, Rest2/binary>> = Rest1,
get_metadata_from_siblings(Rest2, get_metadata_from_siblings(Rest2,
SibCount - 1, SibCount - 1,
[MetaBin|SibMetaBinList]). <<SibMetaBin/binary,
0:32/integer,
MetaLen:32/integer,
MetaBin:MetaLen/binary>>).

4
test/end_to_end/testutil.erl
View file

@ -258,11 +258,11 @@ check_forobject(Bookie, TestObject) ->
{ok, HeadBinary} = book_riakhead(Bookie, {ok, HeadBinary} = book_riakhead(Bookie,
TestObject#r_object.bucket, TestObject#r_object.bucket,
TestObject#r_object.key), TestObject#r_object.key),
{_SibMetaBinList, {_SibMetaBin,
Vclock, Vclock,
_Hash, _Hash,
size} = leveled_codec:riak_extract_metadata(HeadBinary, size), size} = leveled_codec:riak_extract_metadata(HeadBinary, size),
true = Vclock == TestObject#r_object.vclock. true = binary_to_term(Vclock) == TestObject#r_object.vclock.
check_formissingobject(Bookie, Bucket, Key) -> check_formissingobject(Bookie, Bucket, Key) ->
not_found = book_riakget(Bookie, Bucket, Key), not_found = book_riakget(Bookie, Bucket, Key),