umgeher/jsx
0
Fork 0
Code Issues Pull requests Activity

add tests for bad utf sequences, fix failures

Browse source
This commit is contained in:
alisdair sullivan 2012-03-28 23:31:07 -07:00
parent 2d2dd5f7c1
commit dc6a026e94
3 changed files with 622 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

201
src/jsx_decoder.erl
View file

@ -1041,6 +1041,207 @@ done(Bin, Handler, Stack, Opts) -> ?error([Bin, Handler, Stack, Opts]).
-include_lib("eunit/include/eunit.hrl").
xcode(Bin) -> xcode(Bin, []).
xcode(Bin, Opts) ->
Size = size(Bin),
try jsx:to_term(<<34, Bin:Size/binary, 34>>, Opts)
catch error:badarg -> {error, badarg}
end.
is_bad({error, badarg}) -> true;
is_bad(_) -> false.
bad_utf8_test_() ->
[
{"orphan continuation byte u+0080",
?_assert(is_bad(xcode(<<16#0080>>)))
},
{"orphan continuation byte u+0080 replaced",
?_assertEqual(xcode(<<16#0080>>, [loose_unicode]), <<16#fffd/utf8>>)
},
{"orphan continuation byte u+00bf",
?_assert(is_bad(xcode(<<16#00bf>>)))
},
{"orphan continuation byte u+00bf replaced",
?_assertEqual(xcode(<<16#00bf>>, [loose_unicode]), <<16#fffd/utf8>>)
},
{"2 continuation bytes",
?_assert(is_bad(xcode(<<(binary:copy(<<16#0080>>, 2))/binary>>)))
},
{"2 continuation bytes replaced",
?_assertEqual(
xcode(<<(binary:copy(<<16#0080>>, 2))/binary>>, [loose_unicode]),
binary:copy(<<16#fffd/utf8>>, 2)
)
},
{"3 continuation bytes",
?_assert(is_bad(xcode(<<(binary:copy(<<16#0080>>, 3))/binary>>)))
},
{"3 continuation bytes replaced",
?_assertEqual(
xcode(<<(binary:copy(<<16#0080>>, 3))/binary>>, [loose_unicode]),
binary:copy(<<16#fffd/utf8>>, 3)
)
},
{"4 continuation bytes",
?_assert(is_bad(xcode(<<(binary:copy(<<16#0080>>, 4))/binary>>)))
},
{"4 continuation bytes replaced",
?_assertEqual(
xcode(<<(binary:copy(<<16#0080>>, 4))/binary>>, [loose_unicode]),
binary:copy(<<16#fffd/utf8>>, 4)
)
},
{"5 continuation bytes",
?_assert(is_bad(xcode(<<(binary:copy(<<16#0080>>, 5))/binary>>)))
},
{"5 continuation bytes replaced",
?_assertEqual(
xcode(<<(binary:copy(<<16#0080>>, 5))/binary>>, [loose_unicode]),
binary:copy(<<16#fffd/utf8>>, 5)
)
},
{"6 continuation bytes",
?_assert(is_bad(xcode(<<(binary:copy(<<16#0080>>, 6))/binary>>)))
},
{"6 continuation bytes replaced",
?_assertEqual(
xcode(<<(binary:copy(<<16#0080>>, 6))/binary>>, [loose_unicode]),
binary:copy(<<16#fffd/utf8>>, 6)
)
},
{"all continuation bytes",
?_assert(is_bad(xcode(<<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>)))
},
{"all continuation bytes replaced",
?_assertEqual(
xcode(<<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, [loose_unicode]),
binary:copy(<<16#fffd/utf8>>, length(lists:seq(16#0080, 16#00bf)))
)
},
{"lonely start byte",
?_assert(is_bad(xcode(<<16#00c0>>)))
},
{"lonely start byte replaced",
?_assertEqual(
xcode(<<16#00c0>>, [loose_unicode]),
<<16#fffd/utf8>>
)
},
{"lonely start bytes (2 byte)",
?_assert(is_bad(xcode(<<16#00c0, 32, 16#00df>>)))
},
{"lonely start bytes (2 byte) replaced",
?_assertEqual(
xcode(<<16#00c0, 32, 16#00df>>, [loose_unicode]),
<<16#fffd/utf8, 32, 16#fffd/utf8>>
)
},
{"lonely start bytes (3 byte)",
?_assert(is_bad(xcode(<<16#00e0, 32, 16#00ef>>)))
},
{"lonely start bytes (3 byte) replaced",
?_assertEqual(
xcode(<<16#00e0, 32, 16#00ef>>, [loose_unicode]),
<<16#fffd/utf8, 32, 16#fffd/utf8>>
)
},
{"lonely start bytes (4 byte)",
?_assert(is_bad(xcode(<<16#00f0, 32, 16#00f7>>)))
},
{"lonely start bytes (4 byte) replaced",
?_assertEqual(
xcode(<<16#00f0, 32, 16#00f7>>, [loose_unicode]),
<<16#fffd/utf8, 32, 16#fffd/utf8>>
)
},
{"missing continuation byte (3 byte)",
?_assert(is_bad(xcode(<<224, 160, 32>>)))
},
{"missing continuation byte (3 byte) replaced",
?_assertEqual(
xcode(<<224, 160, 32>>, [loose_unicode]),
<<16#fffd/utf8, 32>>
)
},
{"missing continuation byte (4 byte missing one)",
?_assert(is_bad(xcode(<<240, 144, 128, 32>>)))
},
{"missing continuation byte2 (4 byte missing one) replaced",
?_assertEqual(
xcode(<<240, 144, 128, 32>>, [loose_unicode]),
<<16#fffd/utf8, 32>>
)
},
{"missing continuation byte (4 byte missing two)",
?_assert(is_bad(xcode(<<240, 144, 32>>)))
},
{"missing continuation byte2 (4 byte missing two) replaced",
?_assertEqual(
xcode(<<240, 144, 32>>, [loose_unicode]),
<<16#fffd/utf8, 32>>
)
},
{"overlong encoding of u+002f (2 byte)",
?_assert(is_bad(xcode(<<16#c0, 16#af, 32>>)))
},
{"overlong encoding of u+002f (2 byte) replaced",
?_assertEqual(
xcode(<<16#c0, 16#af, 32>>, [loose_unicode]),
<<16#fffd/utf8, 32>>
)
},
{"overlong encoding of u+002f (3 byte)",
?_assert(is_bad(xcode(<<16#e0, 16#80, 16#af, 32>>)))
},
{"overlong encoding of u+002f (3 byte) replaced",
?_assertEqual(
xcode(<<16#e0, 16#80, 16#af, 32>>, [loose_unicode]),
<<16#fffd/utf8, 32>>
)
},
{"overlong encoding of u+002f (4 byte)",
?_assert(is_bad(xcode(<<16#f0, 16#80, 16#80, 16#af, 32>>)))
},
{"overlong encoding of u+002f (4 byte) replaced",
?_assertEqual(
xcode(<<16#f0, 16#80, 16#80, 16#af, 32>>, [loose_unicode]),
<<16#fffd/utf8, 32>>
)
},
{"highest overlong 2 byte sequence",
?_assert(is_bad(xcode(<<16#c1, 16#bf, 32>>)))
},
{"highest overlong 2 byte sequence replaced",
?_assertEqual(
xcode(<<16#c1, 16#bf, 32>>, [loose_unicode]),
<<16#fffd/utf8, 32>>
)
},
{"highest overlong 3 byte sequence",
?_assert(is_bad(xcode(<<16#e0, 16#9f, 16#bf, 32>>)))
},
{"highest overlong 3 byte sequence replaced",
?_assertEqual(
xcode(<<16#e0, 16#9f, 16#bf, 32>>, [loose_unicode]),
<<16#fffd/utf8, 32>>
)
},
{"highest overlong 4 byte sequence",
?_assert(is_bad(xcode(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>)))
},
{"highest overlong 4 byte sequence replaced",
?_assertEqual(
xcode(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, [loose_unicode]),
<<16#fffd/utf8, 32>>
)
}
].
comments_test_() ->
[
{"preceeding // comment", ?_assertEqual(