umgeher/jsx
0
Fork 0
Code Issues Pull requests Activity

fixed bug related to partial utfx sequences

Browse source
This commit is contained in:
alisdair sullivan 2010-06-01 19:42:43 -07:00
parent c287315ebf
commit ce503823c8
4 changed files with 61 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/jsx.erl
View file

@ -36,7 +36,7 @@ decoder(Opts) ->
decoder({F, _} = Callbacks, OptsList) when is_list(OptsList), is_function(F) ->
start(Callbacks, OptsList);
decoder({{Mod, Fun}, State}, OptsList) when is_list(OptsList), is_atom(Mod), is_atom(Fun) ->
decoder({Mod, Fun, State}, OptsList) when is_list(OptsList), is_atom(Mod), is_atom(Fun) ->
start({fun(E, S) -> Mod:Fun(E, S) end, State}, OptsList).
start(Callbacks, OptsList) ->

17
src/jsx_utf16.erl
View file

@ -193,8 +193,21 @@ string(<<?rsolidus/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) ->
escape(Rest, Stack, Callbacks, Opts, Acc);
string(<<S/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) when ?is_noncontrol(S) ->
string(Rest, Stack, Callbacks, Opts, [S] ++ Acc);
string(Bin, Stack, Callbacks, Opts, Acc) when byte_size(Bin) < 2 ->
{incomplete, fun(Stream) -> string(<<Bin/binary, Stream/binary>>, Stack, Callbacks, Opts, Acc) end}.
string(Bin, Stack, Callbacks, Opts, Acc) ->
case partial_utf16(Bin) of
true ->
{incomplete, fun(Stream) -> string(<<Bin/binary, Stream/binary>>, Stack, Callbacks, Opts, Acc) end}
; false ->
erlang:error(function_clause)
end.
partial_utf16(<<>>) -> true;
%% this case is not strictly true, there are single bytes that should be rejected, but
%% they're rare enough they can be ignored
partial_utf16(<<_X>>) -> true;
partial_utf16(<<X, _Y>>) when X >= 16#d8, X =< 16#df -> true;
partial_utf16(<<X, _Y, Z>>) when X >= 16#d8, X =< 16#df, Z >= 16#dc, Z =< 16#df -> true;
partial_utf16(_) -> false.
%% only thing to note here is the additional accumulator passed to escaped_unicode used

17
src/jsx_utf16le.erl
View file

@ -193,8 +193,21 @@ string(<<?rsolidus/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) ->
escape(Rest, Stack, Callbacks, Opts, Acc);
string(<<S/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) when ?is_noncontrol(S) ->
string(Rest, Stack, Callbacks, Opts, [S] ++ Acc);
string(Bin, Stack, Callbacks, Opts, Acc) when byte_size(Bin) < 2 ->
{incomplete, fun(Stream) -> string(<<Bin/binary, Stream/binary>>, Stack, Callbacks, Opts, Acc) end}.
string(Bin, Stack, Callbacks, Opts, Acc) ->
case partial_utf16(Bin) of
true ->
{incomplete, fun(Stream) -> string(<<Bin/binary, Stream/binary>>, Stack, Callbacks, Opts, Acc) end}
; false ->
erlang:error(function_clause)
end.
partial_utf16(<<>>) -> true;
%% this case is not strictly true, there are single bytes that should be rejected, but
%% they're rare enough they can be ignored
partial_utf16(<<_X>>) -> true;
partial_utf16(<<_Y, X>>) when X >= 16#d8, X =< 16#df -> true;
partial_utf16(<<_Y, X, _Z>>) when X >= 16#d8, X =< 16#df -> true;
partial_utf16(_) -> false.
%% only thing to note here is the additional accumulator passed to escaped_unicode used

34
src/jsx_utf8.erl
View file

@ -183,7 +183,8 @@ key(<<>>, Stack, Callbacks, Opts) ->
%% converted back to lists by the user anyways.
%% the clause starting with Bin is necessary for cases where a stream is broken at a
%% point where it contains only a partial utf-8 sequence.
%% point where it contains only a partial utf-8 sequence. we emulate a function_clause
%% error if the partial sequence is not valid utf-8 to maintain consistency of errors
string(<<?quote/?encoding, Rest/binary>>, [key|_] = Stack, Callbacks, Opts, Acc) ->
colon(Rest, Stack, fold({key, lists:reverse(Acc)}, Callbacks), Opts);
@ -193,8 +194,31 @@ string(<<?rsolidus/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) ->
escape(Rest, Stack, Callbacks, Opts, Acc);
string(<<S/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) when ?is_noncontrol(S) ->
string(Rest, Stack, Callbacks, Opts, [S] ++ Acc);
string(<<>>, Stack, Callbacks, Opts, Acc) ->
{incomplete, fun(Stream) -> string(Stream, Stack, Callbacks, Opts, Acc) end}.
string(Bin, Stack, Callbacks, Opts, Acc) ->
case partial_utf8(Bin) of
true ->
{incomplete, fun(Stream) -> string(<<Bin/binary, Stream/binary>>, Stack, Callbacks, Opts, Acc) end}
; false ->
erlang:error(function_clause)
end.
%% in the case of broken (as in split over two halves of a stream) utf-8 input,
%% ensure that the half present is *possibly* valid
partial_utf8(<<>>) -> true;
partial_utf8(<<X>>) when X >= 16#c2, X =< 16#df -> true;
partial_utf8(<<X, Rest/binary>>) when X >= 16#e0, X =< 16#ef ->
case Rest of
<<>> -> true
; <<Y>> when Y >= 16#80, Y =< 16#bf -> true
end;
partial_utf8(<<X, Rest/binary>>) when X >= 16#f0, X =< 16#f4 ->
case Rest of
<<>> -> true
; <<Y>> when Y >= 16#80, Y =< 16#bf -> true
; <<Y, Z>> when Y >= 16#80, Y =< 16#bf, Z >= 16#80, Z =< 16#bf -> true
end;
partial_utf8(_) -> false.
%% only thing to note here is the additional accumulator passed to escaped_unicode used
@ -346,9 +370,9 @@ decimal(<<?comma/?encoding, Rest/binary>>, [array|_] = Stack, Callbacks, Opts, A
decimal(<<?zero/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) ->
decimal(Rest, Stack, Callbacks, Opts, [?zero] ++ Acc);
decimal(<<$e/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) ->
e(Rest, Stack, Callbacks, Opts, "e0." ++ Acc);
e(Rest, Stack, Callbacks, Opts, "e" ++ Acc);
decimal(<<$E/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) ->
e(Rest, Stack, Callbacks, Opts, "e0." ++ Acc);
e(Rest, Stack, Callbacks, Opts, "e" ++ Acc);
decimal(<<S/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) when ?is_whitespace(S) ->
maybe_done(Rest, Stack, fold({float, lists:reverse(Acc)}, Callbacks), Opts);
decimal(<<?solidus/?encoding, Rest/binary>>, Stack, Callbacks, ?comments_enabled(Opts), Acc) ->