From f137ff484bd5440eadad9a92243b3a0f0119e472 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sun, 20 Oct 2013 03:16:56 +0000 Subject: [PATCH] fix bad utf8 encodings by default (convert to u+FFFD) this (temporarily, i hope) removes the fast path for handling strings internally without escaping or replacing bad utf8 --- src/jsx_config.erl | 16 ++- src/jsx_config.hrl | 2 +- src/jsx_decoder.erl | 167 ++++++++++++------------ src/jsx_encoder.erl | 4 +- src/jsx_parser.erl | 301 +++++++++++--------------------------------- src/jsx_tests.hrl | 108 ++++++++-------- 6 files changed, 220 insertions(+), 378 deletions(-) diff --git a/src/jsx_config.erl b/src/jsx_config.erl index 2aa9107..0427ed5 100644 --- a/src/jsx_config.erl +++ b/src/jsx_config.erl @@ -40,8 +40,8 @@ parse_config(Config) -> parse_config([], Config) -> Config; -parse_config([replaced_bad_utf8|Rest], Config) -> - parse_config(Rest, Config#config{replaced_bad_utf8=true}); +parse_config([strict_utf8|Rest], Config) -> + parse_config(Rest, Config#config{strict_utf8=true}); parse_config([escaped_forward_slashes|Rest], Config) -> parse_config(Rest, Config#config{escaped_forward_slashes=true}); parse_config([stream|Rest], Config) -> @@ -60,7 +60,6 @@ parse_config([ignored_bad_escapes|Rest], Config) -> parse_config(Rest, Config#config{ignored_bad_escapes=true}); parse_config([relax|Rest], Config) -> parse_config(Rest, Config#config{ - replaced_bad_utf8 = true, single_quoted_strings = true, comments = true, ignored_bad_escapes = true @@ -95,7 +94,7 @@ config_to_list(Config) -> valid_flags() -> [ - replaced_bad_utf8, + strict_utf8, escaped_forward_slashes, single_quoted_strings, unescaped_jsonp, @@ -136,7 +135,7 @@ config_test_() -> {"all flags", ?_assertEqual( #config{ - replaced_bad_utf8=true, + strict_utf8=true, escaped_forward_slashes=true, stream=true, single_quoted_strings=true, @@ -146,7 +145,7 @@ config_test_() -> ignored_bad_escapes=true }, parse_config([ - replaced_bad_utf8, + strict_utf8, escaped_forward_slashes, stream, single_quoted_strings, @@ -160,7 +159,6 @@ config_test_() -> {"relax flag", ?_assertEqual( #config{ - replaced_bad_utf8=true, single_quoted_strings=true, comments=true, ignored_bad_escapes=true @@ -202,7 +200,7 @@ config_to_list_test_() -> )}, {"all flags", ?_assertEqual( [ - replaced_bad_utf8, + strict_utf8, escaped_forward_slashes, single_quoted_strings, unescaped_jsonp, @@ -213,7 +211,7 @@ config_to_list_test_() -> ], config_to_list( #config{ - replaced_bad_utf8=true, + strict_utf8=true, escaped_forward_slashes=true, stream=true, single_quoted_strings=true, diff --git a/src/jsx_config.hrl b/src/jsx_config.hrl index 09ebce6..5b0f21e 100644 --- a/src/jsx_config.hrl +++ b/src/jsx_config.hrl @@ -1,5 +1,5 @@ -record(config, { - replaced_bad_utf8 = false, + strict_utf8 = false, escaped_forward_slashes = false, single_quoted_strings = false, unescaped_jsonp = false, diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 26e5aa6..f521a04 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -548,36 +548,35 @@ string(<>, Handler, Acc, Stack, Config) when X >= 16#f0, X =< 16#f4, Y >= 16#80, Y =< 16#bf, Z >= 16#80, Z =< 16#bf -> - incomplete(string, <>, Handler, Acc, Stack, Config); + incomplete(string, <>, Handler, Acc, Stack, Config); %% surrogates -string(<<237, X, _, Rest/binary>>, Handler, Acc, Stack, Config=#config{replaced_bad_utf8=true}) +string(<<237, X, _, Rest/binary>>, Handler, Acc, Stack, Config=#config{strict_utf8=false}) when X >= 160 -> string(Rest, Handler, acc_seq(Acc, 16#fffd), Stack, Config); %% u+xfffe, u+xffff, control codes and other noncharacters -string(<<_/utf8, Rest/binary>>, Handler, Acc, Stack, Config=#config{replaced_bad_utf8=true}) -> +string(<<_/utf8, Rest/binary>>, Handler, Acc, Stack, Config=#config{strict_utf8=false}) -> string(Rest, Handler, acc_seq(Acc, 16#fffd), Stack, Config); %% u+fffe and u+ffff for R14BXX (subsequent runtimes will happily match the %% preceeding clause -string(<<239, 191, X, Rest/binary>>, Handler, Acc, Stack, Config=#config{replaced_bad_utf8=true}) +string(<<239, 191, X, Rest/binary>>, Handler, Acc, Stack, Config=#config{strict_utf8=false}) when X == 190; X == 191 -> string(Rest, Handler, acc_seq(Acc, 16#fffd), Stack, Config); %% overlong encodings and missing continuations of a 2 byte sequence -string(<>, Handler, Acc, Stack, Config=#config{replaced_bad_utf8=true}) +string(<>, Handler, Acc, Stack, Config=#config{strict_utf8=false}) when X >= 192, X =< 223 -> strip_continuations(Rest, Handler, Acc, Stack, Config, 1); %% overlong encodings and missing continuations of a 3 byte sequence -string(<>, Handler, Acc, Stack, Config=#config{replaced_bad_utf8=true}) +string(<>, Handler, Acc, Stack, Config=#config{strict_utf8=false}) when X >= 224, X =< 239 -> strip_continuations(Rest, Handler, Acc, Stack, Config, 2); %% overlong encodings and missing continuations of a 4 byte sequence -string(<>, Handler, Acc, Stack, Config=#config{replaced_bad_utf8=true}) +string(<>, Handler, Acc, Stack, Config=#config{strict_utf8=false}) when X >= 240, X =< 247 -> strip_continuations(Rest, Handler, Acc, Stack, Config, 3); %% incompletes and unexpected bytes, including orphan continuations -string(<<_, Rest/binary>>, Handler, Acc, Stack, Config=#config{replaced_bad_utf8=true}) -> +string(<<_, Rest/binary>>, Handler, Acc, Stack, Config=#config{strict_utf8=false}) -> string(Rest, Handler, acc_seq(Acc, 16#fffd), Stack, Config); -string(Bin, Handler, Acc, Stack, Config) -> - ?error(string, Bin, Handler, Acc, Stack, Config). +string(Bin, Handler, Acc, Stack, Config) -> ?error(string, Bin, Handler, Acc, Stack, Config). doublequote(<>, Handler, Acc, [key|_] = Stack, Config) -> @@ -658,9 +657,9 @@ unescape(<<$u, $d, A, B, C, ?rsolidus, $u, W, X, Y, Z, Rest/binary>>, Handler, A when (A == $8 orelse A == $9 orelse A == $a orelse A == $b), ?is_hex(B), ?is_hex(C), ?is_hex(W), ?is_hex(X), ?is_hex(Y), ?is_hex(Z) -> - case Config#config.replaced_bad_utf8 of - true -> string(Rest, Handler, acc_seq(Acc, [16#fffd, 16#fffd]), Stack, Config); - false -> ?error(<<$u, $d, A, B, C, ?rsolidus, $u, W, X, Y, Z, Rest/binary>>, Handler, Acc, Stack, Config) + case Config#config.strict_utf8 of + true -> ?error(<<$u, $d, A, B, C, ?rsolidus, $u, W, X, Y, Z, Rest/binary>>, Handler, Acc, Stack, Config); + false -> string(Rest, Handler, acc_seq(Acc, [16#fffd, 16#fffd]), Stack, Config) end; unescape(<<$u, $d, A, B, C, ?rsolidus, Rest/binary>>, Handler, Acc, Stack, Config) when (A == $8 orelse A == $9 orelse A == $a orelse A == $b) andalso @@ -677,9 +676,9 @@ unescape(<<$u, A, B, C, D, Rest/binary>>, Handler, Acc, Stack, Config) case erlang:list_to_integer([A, B, C, D], 16) of Codepoint when Codepoint < 16#d800; Codepoint > 16#dfff -> string(Rest, Handler, acc_seq(Acc, maybe_replace(Codepoint, Config)), Stack, Config); - _ when Config#config.replaced_bad_utf8 -> - string(Rest, Handler, acc_seq(Acc, 16#fffd), Stack, Config); - _ -> ?error(string, <>, Handler, Acc, Stack, Config) + _ when Config#config.strict_utf8 -> + ?error(string, <>, Handler, Acc, Stack, Config); + _ -> string(Rest, Handler, acc_seq(Acc, 16#fffd), Stack, Config) end; unescape(Bin, Handler, Acc, Stack, Config=#config{ignored_bad_escapes=true}) -> string(Bin, Handler, acc_seq(Acc, ?rsolidus), Stack, Config); @@ -904,7 +903,7 @@ comment(<>, Handler, Resume, [multicomment|_] = Stack, Config) -> incomplete(comment, <>, Handler, Resume, Stack, Config); comment(<<_/utf8, Rest/binary>>, Handler, Resume, Stack, Config) -> comment(Rest, Handler, Resume, Stack, Config); -comment(<<_, Rest/binary>>, Handler, Resume, Stack, Config=#config{replaced_bad_utf8=true}) -> +comment(<<_, Rest/binary>>, Handler, Resume, Stack, Config=#config{strict_utf8=false}) -> comment(Rest, Handler, Resume, Stack, Config); comment(<<>>, Handler, done, [Comment], Config=#config{stream=false}) when Comment == comment; Comment == multicomment -> @@ -1250,15 +1249,15 @@ comments_test_() -> )}, {"// comment with badutf", ?_assertEqual( [start_array, {literal, true}, end_array, end_json], - decode(<<"[ // comment ", 16#00c0, " ", ?newline, "true]">>, [comments, replaced_bad_utf8]) + decode(<<"[ // comment ", 16#00c0, " ", ?newline, "true]">>, [comments]) )}, {"/**/ comment with badutf", ?_assertEqual( [start_array, {literal, true}, end_array, end_json], - decode(<<"[ /* comment ", 16#00c0, " */ true]">>, [comments, replaced_bad_utf8]) + decode(<<"[ /* comment ", 16#00c0, " */ true]">>, [comments]) )}, {"/**/ comment with badutf preceeded by /", ?_assertEqual( [start_array, {literal, true}, end_array, end_json], - decode(<<"[ /* comment /", 16#00c0, " */ true]">>, [comments, replaced_bad_utf8]) + decode(<<"[ /* comment /", 16#00c0, " */ true]">>, [comments]) )} ]. @@ -1326,35 +1325,35 @@ clean_string_test_() -> )}, {"error reserved space", ?_assertEqual( lists:duplicate(length(reserved_space()), {error, badarg}), - lists:map(fun(Codepoint) -> decode(Codepoint, []) end, reserved_space()) + lists:map(fun(Codepoint) -> decode(Codepoint, [strict_utf8]) end, reserved_space()) )}, {"error surrogates", ?_assertEqual( lists:duplicate(length(surrogates()), {error, badarg}), - lists:map(fun(Codepoint) -> decode(Codepoint, []) end, surrogates()) + lists:map(fun(Codepoint) -> decode(Codepoint, [strict_utf8]) end, surrogates()) )}, {"error noncharacters", ?_assertEqual( lists:duplicate(length(noncharacters()), {error, badarg}), - lists:map(fun(Codepoint) -> decode(Codepoint, []) end, noncharacters()) + lists:map(fun(Codepoint) -> decode(Codepoint, [strict_utf8]) end, noncharacters()) )}, {"error extended noncharacters", ?_assertEqual( lists:duplicate(length(extended_noncharacters()), {error, badarg}), - lists:map(fun(Codepoint) -> decode(Codepoint, []) end, extended_noncharacters()) + lists:map(fun(Codepoint) -> decode(Codepoint, [strict_utf8]) end, extended_noncharacters()) )}, {"clean reserved space", ?_assertEqual( lists:duplicate(length(reserved_space()), [{string, <<16#fffd/utf8>>}, end_json]), - lists:map(fun(Codepoint) -> decode(Codepoint, [replaced_bad_utf8]) end, reserved_space()) + lists:map(fun(Codepoint) -> decode(Codepoint, []) end, reserved_space()) )}, {"clean surrogates", ?_assertEqual( lists:duplicate(length(surrogates()), [{string, <<16#fffd/utf8>>}, end_json]), - lists:map(fun(Codepoint) -> decode(Codepoint, [replaced_bad_utf8]) end, surrogates()) + lists:map(fun(Codepoint) -> decode(Codepoint, []) end, surrogates()) )}, {"clean noncharacters", ?_assertEqual( lists:duplicate(length(noncharacters()), [{string, <<16#fffd/utf8>>}, end_json]), - lists:map(fun(Codepoint) -> decode(Codepoint, [replaced_bad_utf8]) end, noncharacters()) + lists:map(fun(Codepoint) -> decode(Codepoint, []) end, noncharacters()) )}, {"clean extended noncharacters", ?_assertEqual( lists:duplicate(length(extended_noncharacters()), [{string, <<16#fffd/utf8>>}, end_json]), - lists:map(fun(Codepoint) -> decode(Codepoint, [replaced_bad_utf8]) end, extended_noncharacters()) + lists:map(fun(Codepoint) -> decode(Codepoint, []) end, extended_noncharacters()) )}, {"dirty \\uwxyz", ?_assertEqual( [{string, <<"\\uwxyz">>}, end_json], @@ -1405,190 +1404,190 @@ bad_utf8_test_() -> [ {"noncharacter u+fffe", ?_assertError( badarg, - decode_bad_utf(<<239, 191, 190>>, []) + decode_bad_utf(<<239, 191, 190>>, [strict_utf8]) )}, {"noncharacter u+fffe replaced", ?_assertEqual( <<16#fffd/utf8>>, - decode_bad_utf(<<239, 191, 190>>, [replaced_bad_utf8]) + decode_bad_utf(<<239, 191, 190>>, []) )}, {"noncharacter u+ffff", ?_assertError( badarg, - decode_bad_utf(<<239, 191, 191>>, []) + decode_bad_utf(<<239, 191, 191>>, [strict_utf8]) )}, {"noncharacter u+ffff replaced", ?_assertEqual( <<16#fffd/utf8>>, - decode_bad_utf(<<239, 191, 191>>, [replaced_bad_utf8]) + decode_bad_utf(<<239, 191, 191>>, []) )}, {"orphan continuation byte u+0080", ?_assertError( badarg, - decode_bad_utf(<<16#0080>>, []) + decode_bad_utf(<<16#0080>>, [strict_utf8]) )}, {"orphan continuation byte u+0080 replaced", ?_assertEqual( <<16#fffd/utf8>>, - decode_bad_utf(<<16#0080>>, [replaced_bad_utf8]) + decode_bad_utf(<<16#0080>>, []) )}, {"orphan continuation byte u+00bf", ?_assertError( badarg, - decode_bad_utf(<<16#00bf>>, []) + decode_bad_utf(<<16#00bf>>, [strict_utf8]) )}, {"orphan continuation byte u+00bf replaced", ?_assertEqual( <<16#fffd/utf8>>, - decode_bad_utf(<<16#00bf>>, [replaced_bad_utf8]) + decode_bad_utf(<<16#00bf>>, []) )}, {"2 continuation bytes", ?_assertError( badarg, - decode_bad_utf(<<(binary:copy(<<16#0080>>, 2))/binary>>, []) + decode_bad_utf(<<(binary:copy(<<16#0080>>, 2))/binary>>, [strict_utf8]) )}, {"2 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 2), - decode_bad_utf(<<(binary:copy(<<16#0080>>, 2))/binary>>, [replaced_bad_utf8]) + decode_bad_utf(<<(binary:copy(<<16#0080>>, 2))/binary>>, []) )}, {"3 continuation bytes", ?_assertError( badarg, - decode_bad_utf(<<(binary:copy(<<16#0080>>, 3))/binary>>, []) + decode_bad_utf(<<(binary:copy(<<16#0080>>, 3))/binary>>, [strict_utf8]) )}, {"3 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 3), - decode_bad_utf(<<(binary:copy(<<16#0080>>, 3))/binary>>, [replaced_bad_utf8]) + decode_bad_utf(<<(binary:copy(<<16#0080>>, 3))/binary>>, []) )}, {"4 continuation bytes", ?_assertError( badarg, - decode_bad_utf(<<(binary:copy(<<16#0080>>, 4))/binary>>, []) + decode_bad_utf(<<(binary:copy(<<16#0080>>, 4))/binary>>, [strict_utf8]) )}, {"4 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 4), - decode_bad_utf(<<(binary:copy(<<16#0080>>, 4))/binary>>, [replaced_bad_utf8]) + decode_bad_utf(<<(binary:copy(<<16#0080>>, 4))/binary>>, []) )}, {"5 continuation bytes", ?_assertError( badarg, - decode_bad_utf(<<(binary:copy(<<16#0080>>, 5))/binary>>, []) + decode_bad_utf(<<(binary:copy(<<16#0080>>, 5))/binary>>, [strict_utf8]) )}, {"5 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 5), - decode_bad_utf(<<(binary:copy(<<16#0080>>, 5))/binary>>, [replaced_bad_utf8]) + decode_bad_utf(<<(binary:copy(<<16#0080>>, 5))/binary>>, []) )}, {"6 continuation bytes", ?_assertError( badarg, - decode_bad_utf(<<(binary:copy(<<16#0080>>, 6))/binary>>, []) + decode_bad_utf(<<(binary:copy(<<16#0080>>, 6))/binary>>, [strict_utf8]) )}, {"6 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 6), - decode_bad_utf(<<(binary:copy(<<16#0080>>, 6))/binary>>, [replaced_bad_utf8]) + decode_bad_utf(<<(binary:copy(<<16#0080>>, 6))/binary>>, []) )}, {"all continuation bytes", ?_assertError( badarg, - decode_bad_utf(<<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, []) + decode_bad_utf(<<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, [strict_utf8]) )}, {"all continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, length(lists:seq(16#0080, 16#00bf))), decode_bad_utf( <<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, - [replaced_bad_utf8] + [] ) )}, {"lonely start byte", ?_assertError( badarg, - decode_bad_utf(<<16#00c0>>, []) + decode_bad_utf(<<16#00c0>>, [strict_utf8]) )}, {"lonely start byte replaced", ?_assertEqual( <<16#fffd/utf8>>, - decode_bad_utf(<<16#00c0>>, [replaced_bad_utf8]) + decode_bad_utf(<<16#00c0>>, []) )}, {"lonely start bytes (2 byte)", ?_assertError( badarg, - decode_bad_utf(<<16#00c0, 32, 16#00df>>, []) + decode_bad_utf(<<16#00c0, 32, 16#00df>>, [strict_utf8]) )}, {"lonely start bytes (2 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, - decode_bad_utf(<<16#00c0, 32, 16#00df>>, [replaced_bad_utf8]) + decode_bad_utf(<<16#00c0, 32, 16#00df>>, []) )}, {"lonely start bytes (3 byte)", ?_assertError( badarg, - decode_bad_utf(<<16#00e0, 32, 16#00ef>>, []) + decode_bad_utf(<<16#00e0, 32, 16#00ef>>, [strict_utf8]) )}, {"lonely start bytes (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, - decode_bad_utf(<<16#00e0, 32, 16#00ef>>, [replaced_bad_utf8]) + decode_bad_utf(<<16#00e0, 32, 16#00ef>>, []) )}, {"lonely start bytes (4 byte)", ?_assertError( badarg, - decode_bad_utf(<<16#00f0, 32, 16#00f7>>, []) + decode_bad_utf(<<16#00f0, 32, 16#00f7>>, [strict_utf8]) )}, {"lonely start bytes (4 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, - decode_bad_utf(<<16#00f0, 32, 16#00f7>>, [replaced_bad_utf8]) + decode_bad_utf(<<16#00f0, 32, 16#00f7>>, []) )}, {"missing continuation byte (3 byte)", ?_assertError( badarg, - decode_bad_utf(<<224, 160, 32>>, []) + decode_bad_utf(<<224, 160, 32>>, [strict_utf8]) )}, {"missing continuation byte (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - decode_bad_utf(<<224, 160, 32>>, [replaced_bad_utf8]) + decode_bad_utf(<<224, 160, 32>>, []) )}, {"missing continuation byte (4 byte missing one)", ?_assertError( badarg, - decode_bad_utf(<<240, 144, 128, 32>>, []) + decode_bad_utf(<<240, 144, 128, 32>>, [strict_utf8]) )}, {"missing continuation byte (4 byte missing one) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - decode_bad_utf(<<240, 144, 128, 32>>, [replaced_bad_utf8]) + decode_bad_utf(<<240, 144, 128, 32>>, []) )}, {"missing continuation byte (4 byte missing two)", ?_assertError( badarg, - decode_bad_utf(<<240, 144, 32>>, []) + decode_bad_utf(<<240, 144, 32>>, [strict_utf8]) )}, {"missing continuation byte (4 byte missing two) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - decode_bad_utf(<<240, 144, 32>>, [replaced_bad_utf8]) + decode_bad_utf(<<240, 144, 32>>, []) )}, {"overlong encoding of u+002f (2 byte)", ?_assertError( badarg, - decode_bad_utf(<<16#c0, 16#af, 32>>, []) + decode_bad_utf(<<16#c0, 16#af, 32>>, [strict_utf8]) )}, {"overlong encoding of u+002f (2 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - decode_bad_utf(<<16#c0, 16#af, 32>>, [replaced_bad_utf8]) + decode_bad_utf(<<16#c0, 16#af, 32>>, []) )}, {"overlong encoding of u+002f (3 byte)", ?_assertError( badarg, - decode_bad_utf(<<16#e0, 16#80, 16#af, 32>>, []) + decode_bad_utf(<<16#e0, 16#80, 16#af, 32>>, [strict_utf8]) )}, {"overlong encoding of u+002f (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - decode_bad_utf(<<16#e0, 16#80, 16#af, 32>>, [replaced_bad_utf8]) + decode_bad_utf(<<16#e0, 16#80, 16#af, 32>>, []) )}, {"overlong encoding of u+002f (4 byte)", ?_assertError( badarg, - decode_bad_utf(<<16#f0, 16#80, 16#80, 16#af, 32>>, []) + decode_bad_utf(<<16#f0, 16#80, 16#80, 16#af, 32>>, [strict_utf8]) )}, {"overlong encoding of u+002f (4 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - decode_bad_utf(<<16#f0, 16#80, 16#80, 16#af, 32>>, [replaced_bad_utf8]) + decode_bad_utf(<<16#f0, 16#80, 16#80, 16#af, 32>>, []) )}, {"highest overlong 2 byte sequence", ?_assertError( badarg, - decode_bad_utf(<<16#c1, 16#bf, 32>>, []) + decode_bad_utf(<<16#c1, 16#bf, 32>>, [strict_utf8]) )}, {"highest overlong 2 byte sequence replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - decode_bad_utf(<<16#c1, 16#bf, 32>>, [replaced_bad_utf8]) + decode_bad_utf(<<16#c1, 16#bf, 32>>, []) )}, {"highest overlong 3 byte sequence", ?_assertError( badarg, - decode_bad_utf(<<16#e0, 16#9f, 16#bf, 32>>, []) + decode_bad_utf(<<16#e0, 16#9f, 16#bf, 32>>, [strict_utf8]) )}, {"highest overlong 3 byte sequence replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - decode_bad_utf(<<16#e0, 16#9f, 16#bf, 32>>, [replaced_bad_utf8]) + decode_bad_utf(<<16#e0, 16#9f, 16#bf, 32>>, []) )}, {"highest overlong 4 byte sequence", ?_assertError( badarg, - decode_bad_utf(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, []) + decode_bad_utf(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, [strict_utf8]) )}, {"highest overlong 4 byte sequence replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - decode_bad_utf(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, [replaced_bad_utf8]) + decode_bad_utf(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, []) )} ]. @@ -1647,35 +1646,35 @@ unescape_test_() -> )}, {"replace bad high surrogate", ?_assertEqual( <<16#fffd/utf8>>, - unescape(<<"\\udc00"/utf8>>, [replaced_bad_utf8]) + unescape(<<"\\udc00"/utf8>>, []) )}, {"do not unescape bad high surrogate", ?_assertError( badarg, - unescape(<<"\\udc00"/utf8>>, []) + unescape(<<"\\udc00"/utf8>>, [strict_utf8]) )}, {"replace naked high surrogate", ?_assertEqual( <<16#fffd/utf8, "hello world">>, - unescape(<<"\\ud800hello world"/utf8>>, [replaced_bad_utf8]) + unescape(<<"\\ud800hello world"/utf8>>, []) )}, {"do not unescape naked high surrogate", ?_assertError( badarg, - unescape(<<"\\ud800hello world"/utf8>>, []) + unescape(<<"\\ud800hello world"/utf8>>, [strict_utf8]) )}, {"replace naked low surrogate", ?_assertEqual( <<16#fffd/utf8, "hello world">>, - unescape(<<"\\udc00hello world"/utf8>>, [replaced_bad_utf8]) + unescape(<<"\\udc00hello world"/utf8>>, []) )}, {"do not unescape naked low surrogate", ?_assertError( badarg, - unescape(<<"\\udc00hello world"/utf8>>, []) + unescape(<<"\\udc00hello world"/utf8>>, [strict_utf8]) )}, {"replace bad surrogate pair", ?_assertEqual( <<16#fffd/utf8, 16#fffd/utf8>>, - unescape(<<"\\ud800\\u0000">>, [replaced_bad_utf8]) + unescape(<<"\\ud800\\u0000">>, []) )}, {"do not unescape bad surrogate pair", ?_assertError( badarg, - unescape(<<"\\ud800\\u0000">>, []) + unescape(<<"\\ud800\\u0000">>, [strict_utf8]) )}, {"bad pseudo escape sequence", ?_assertError( badarg, @@ -2104,11 +2103,11 @@ custom_error_handler_test_() -> )}, {"single_comment error", ?_assertEqual( {comment, <<192>>}, - Decode(<<"[ //"/utf8, 192>>, [{error_handler, Error}, comments]) + Decode(<<"[ //"/utf8, 192>>, [{error_handler, Error}, comments, strict_utf8]) )}, {"multi_comment error", ?_assertEqual( {comment, <<192>>}, - Decode(<<"[ /*"/utf8, 192>>, [{error_handler, Error}, comments]) + Decode(<<"[ /*"/utf8, 192>>, [{error_handler, Error}, comments, strict_utf8]) )} ]. diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 6c04d27..517be4c 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -78,7 +78,7 @@ err(Term, Opts) -> (jsx:parser(jsx, [], Opts))(Term). error_test_() -> [ {"value error", ?_assertError(badarg, err(self(), []))}, - {"string error", ?_assertError(badarg, err(<<239, 191, 191>>, []))} + {"string error", ?_assertError(badarg, err(<<239, 191, 191>>, [strict_utf8]))} ]. custom_error_handler_test_() -> @@ -90,7 +90,7 @@ custom_error_handler_test_() -> )}, {"string error", ?_assertEqual( {string, [{string, <<239, 191, 191>>}]}, - err(<<239, 191, 191>>, [{error_handler, Error}]) + err(<<239, 191, 191>>, [{error_handler, Error}, strict_utf8]) )} ]. diff --git a/src/jsx_parser.erl b/src/jsx_parser.erl index 6b27e35..350624f 100644 --- a/src/jsx_parser.erl +++ b/src/jsx_parser.erl @@ -193,209 +193,46 @@ clean_string(Bin, Tokens, Handler, Stack, Config) -> end. clean_string(Bin, #config{dirty_strings=true}) -> Bin; -clean_string(Bin, Config) -> - case Config#config.replaced_bad_utf8 orelse Config#config.escaped_strings of - true -> clean(Bin, [], Config); - false -> ensure_clean(Bin) - end. - - -ensure_clean(Bin) -> - case is_clean(Bin) of - ok -> Bin; - {error, badarg} -> {error, badarg} - end. - - -%% fast path for no escaping and no correcting, throws error if string is 'bad' -is_clean(<<>>) -> ok; -is_clean(<<0, Rest/binary>>) -> is_clean(Rest); -is_clean(<<1, Rest/binary>>) -> is_clean(Rest); -is_clean(<<2, Rest/binary>>) -> is_clean(Rest); -is_clean(<<3, Rest/binary>>) -> is_clean(Rest); -is_clean(<<4, Rest/binary>>) -> is_clean(Rest); -is_clean(<<5, Rest/binary>>) -> is_clean(Rest); -is_clean(<<6, Rest/binary>>) -> is_clean(Rest); -is_clean(<<7, Rest/binary>>) -> is_clean(Rest); -is_clean(<<8, Rest/binary>>) -> is_clean(Rest); -is_clean(<<9, Rest/binary>>) -> is_clean(Rest); -is_clean(<<10, Rest/binary>>) -> is_clean(Rest); -is_clean(<<11, Rest/binary>>) -> is_clean(Rest); -is_clean(<<12, Rest/binary>>) -> is_clean(Rest); -is_clean(<<13, Rest/binary>>) -> is_clean(Rest); -is_clean(<<14, Rest/binary>>) -> is_clean(Rest); -is_clean(<<15, Rest/binary>>) -> is_clean(Rest); -is_clean(<<16, Rest/binary>>) -> is_clean(Rest); -is_clean(<<17, Rest/binary>>) -> is_clean(Rest); -is_clean(<<18, Rest/binary>>) -> is_clean(Rest); -is_clean(<<19, Rest/binary>>) -> is_clean(Rest); -is_clean(<<20, Rest/binary>>) -> is_clean(Rest); -is_clean(<<21, Rest/binary>>) -> is_clean(Rest); -is_clean(<<22, Rest/binary>>) -> is_clean(Rest); -is_clean(<<23, Rest/binary>>) -> is_clean(Rest); -is_clean(<<24, Rest/binary>>) -> is_clean(Rest); -is_clean(<<25, Rest/binary>>) -> is_clean(Rest); -is_clean(<<26, Rest/binary>>) -> is_clean(Rest); -is_clean(<<27, Rest/binary>>) -> is_clean(Rest); -is_clean(<<28, Rest/binary>>) -> is_clean(Rest); -is_clean(<<29, Rest/binary>>) -> is_clean(Rest); -is_clean(<<30, Rest/binary>>) -> is_clean(Rest); -is_clean(<<31, Rest/binary>>) -> is_clean(Rest); -is_clean(<<32, Rest/binary>>) -> is_clean(Rest); -is_clean(<<33, Rest/binary>>) -> is_clean(Rest); -is_clean(<<34, Rest/binary>>) -> is_clean(Rest); -is_clean(<<35, Rest/binary>>) -> is_clean(Rest); -is_clean(<<36, Rest/binary>>) -> is_clean(Rest); -is_clean(<<37, Rest/binary>>) -> is_clean(Rest); -is_clean(<<38, Rest/binary>>) -> is_clean(Rest); -is_clean(<<39, Rest/binary>>) -> is_clean(Rest); -is_clean(<<40, Rest/binary>>) -> is_clean(Rest); -is_clean(<<41, Rest/binary>>) -> is_clean(Rest); -is_clean(<<42, Rest/binary>>) -> is_clean(Rest); -is_clean(<<43, Rest/binary>>) -> is_clean(Rest); -is_clean(<<44, Rest/binary>>) -> is_clean(Rest); -is_clean(<<45, Rest/binary>>) -> is_clean(Rest); -is_clean(<<46, Rest/binary>>) -> is_clean(Rest); -is_clean(<<47, Rest/binary>>) -> is_clean(Rest); -is_clean(<<48, Rest/binary>>) -> is_clean(Rest); -is_clean(<<49, Rest/binary>>) -> is_clean(Rest); -is_clean(<<50, Rest/binary>>) -> is_clean(Rest); -is_clean(<<51, Rest/binary>>) -> is_clean(Rest); -is_clean(<<52, Rest/binary>>) -> is_clean(Rest); -is_clean(<<53, Rest/binary>>) -> is_clean(Rest); -is_clean(<<54, Rest/binary>>) -> is_clean(Rest); -is_clean(<<55, Rest/binary>>) -> is_clean(Rest); -is_clean(<<56, Rest/binary>>) -> is_clean(Rest); -is_clean(<<57, Rest/binary>>) -> is_clean(Rest); -is_clean(<<58, Rest/binary>>) -> is_clean(Rest); -is_clean(<<59, Rest/binary>>) -> is_clean(Rest); -is_clean(<<60, Rest/binary>>) -> is_clean(Rest); -is_clean(<<61, Rest/binary>>) -> is_clean(Rest); -is_clean(<<62, Rest/binary>>) -> is_clean(Rest); -is_clean(<<63, Rest/binary>>) -> is_clean(Rest); -is_clean(<<64, Rest/binary>>) -> is_clean(Rest); -is_clean(<<65, Rest/binary>>) -> is_clean(Rest); -is_clean(<<66, Rest/binary>>) -> is_clean(Rest); -is_clean(<<67, Rest/binary>>) -> is_clean(Rest); -is_clean(<<68, Rest/binary>>) -> is_clean(Rest); -is_clean(<<69, Rest/binary>>) -> is_clean(Rest); -is_clean(<<70, Rest/binary>>) -> is_clean(Rest); -is_clean(<<71, Rest/binary>>) -> is_clean(Rest); -is_clean(<<72, Rest/binary>>) -> is_clean(Rest); -is_clean(<<73, Rest/binary>>) -> is_clean(Rest); -is_clean(<<74, Rest/binary>>) -> is_clean(Rest); -is_clean(<<75, Rest/binary>>) -> is_clean(Rest); -is_clean(<<76, Rest/binary>>) -> is_clean(Rest); -is_clean(<<77, Rest/binary>>) -> is_clean(Rest); -is_clean(<<78, Rest/binary>>) -> is_clean(Rest); -is_clean(<<79, Rest/binary>>) -> is_clean(Rest); -is_clean(<<80, Rest/binary>>) -> is_clean(Rest); -is_clean(<<81, Rest/binary>>) -> is_clean(Rest); -is_clean(<<82, Rest/binary>>) -> is_clean(Rest); -is_clean(<<83, Rest/binary>>) -> is_clean(Rest); -is_clean(<<84, Rest/binary>>) -> is_clean(Rest); -is_clean(<<85, Rest/binary>>) -> is_clean(Rest); -is_clean(<<86, Rest/binary>>) -> is_clean(Rest); -is_clean(<<87, Rest/binary>>) -> is_clean(Rest); -is_clean(<<88, Rest/binary>>) -> is_clean(Rest); -is_clean(<<89, Rest/binary>>) -> is_clean(Rest); -is_clean(<<90, Rest/binary>>) -> is_clean(Rest); -is_clean(<<91, Rest/binary>>) -> is_clean(Rest); -is_clean(<<92, Rest/binary>>) -> is_clean(Rest); -is_clean(<<93, Rest/binary>>) -> is_clean(Rest); -is_clean(<<94, Rest/binary>>) -> is_clean(Rest); -is_clean(<<95, Rest/binary>>) -> is_clean(Rest); -is_clean(<<96, Rest/binary>>) -> is_clean(Rest); -is_clean(<<97, Rest/binary>>) -> is_clean(Rest); -is_clean(<<98, Rest/binary>>) -> is_clean(Rest); -is_clean(<<99, Rest/binary>>) -> is_clean(Rest); -is_clean(<<100, Rest/binary>>) -> is_clean(Rest); -is_clean(<<101, Rest/binary>>) -> is_clean(Rest); -is_clean(<<102, Rest/binary>>) -> is_clean(Rest); -is_clean(<<103, Rest/binary>>) -> is_clean(Rest); -is_clean(<<104, Rest/binary>>) -> is_clean(Rest); -is_clean(<<105, Rest/binary>>) -> is_clean(Rest); -is_clean(<<106, Rest/binary>>) -> is_clean(Rest); -is_clean(<<107, Rest/binary>>) -> is_clean(Rest); -is_clean(<<108, Rest/binary>>) -> is_clean(Rest); -is_clean(<<109, Rest/binary>>) -> is_clean(Rest); -is_clean(<<110, Rest/binary>>) -> is_clean(Rest); -is_clean(<<111, Rest/binary>>) -> is_clean(Rest); -is_clean(<<112, Rest/binary>>) -> is_clean(Rest); -is_clean(<<113, Rest/binary>>) -> is_clean(Rest); -is_clean(<<114, Rest/binary>>) -> is_clean(Rest); -is_clean(<<115, Rest/binary>>) -> is_clean(Rest); -is_clean(<<116, Rest/binary>>) -> is_clean(Rest); -is_clean(<<117, Rest/binary>>) -> is_clean(Rest); -is_clean(<<118, Rest/binary>>) -> is_clean(Rest); -is_clean(<<119, Rest/binary>>) -> is_clean(Rest); -is_clean(<<120, Rest/binary>>) -> is_clean(Rest); -is_clean(<<121, Rest/binary>>) -> is_clean(Rest); -is_clean(<<122, Rest/binary>>) -> is_clean(Rest); -is_clean(<<123, Rest/binary>>) -> is_clean(Rest); -is_clean(<<124, Rest/binary>>) -> is_clean(Rest); -is_clean(<<125, Rest/binary>>) -> is_clean(Rest); -is_clean(<<126, Rest/binary>>) -> is_clean(Rest); -is_clean(<<127, Rest/binary>>) -> is_clean(Rest); -is_clean(<>) when X < 16#d800 -> is_clean(Rest); -is_clean(<>) when X > 16#dfff, X < 16#fdd0 -> is_clean(Rest); -is_clean(<>) when X > 16#fdef, X < 16#fffe -> is_clean(Rest); -is_clean(<>) when X >= 16#10000, X < 16#1fffe -> is_clean(Rest); -is_clean(<>) when X >= 16#20000, X < 16#2fffe -> is_clean(Rest); -is_clean(<>) when X >= 16#30000, X < 16#3fffe -> is_clean(Rest); -is_clean(<>) when X >= 16#40000, X < 16#4fffe -> is_clean(Rest); -is_clean(<>) when X >= 16#50000, X < 16#5fffe -> is_clean(Rest); -is_clean(<>) when X >= 16#60000, X < 16#6fffe -> is_clean(Rest); -is_clean(<>) when X >= 16#70000, X < 16#7fffe -> is_clean(Rest); -is_clean(<>) when X >= 16#80000, X < 16#8fffe -> is_clean(Rest); -is_clean(<>) when X >= 16#90000, X < 16#9fffe -> is_clean(Rest); -is_clean(<>) when X >= 16#a0000, X < 16#afffe -> is_clean(Rest); -is_clean(<>) when X >= 16#b0000, X < 16#bfffe -> is_clean(Rest); -is_clean(<>) when X >= 16#c0000, X < 16#cfffe -> is_clean(Rest); -is_clean(<>) when X >= 16#d0000, X < 16#dfffe -> is_clean(Rest); -is_clean(<>) when X >= 16#e0000, X < 16#efffe -> is_clean(Rest); -is_clean(<>) when X >= 16#f0000, X < 16#ffffe -> is_clean(Rest); -is_clean(<>) when X >= 16#100000, X < 16#10fffe -> is_clean(Rest); -is_clean(_Bin) -> {error, badarg}. +clean_string(Bin, Config) -> clean(Bin, [], Config). %% escape and/or replace bad codepoints if requested clean(<<>>, Acc, _Config) -> unicode:characters_to_binary(lists:reverse(Acc)); -clean(<<0, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(0, Config) ++ Acc, Config); -clean(<<1, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(1, Config) ++ Acc, Config); -clean(<<2, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(2, Config) ++ Acc, Config); -clean(<<3, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(3, Config) ++ Acc, Config); -clean(<<4, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(4, Config) ++ Acc, Config); -clean(<<5, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(5, Config) ++ Acc, Config); -clean(<<6, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(6, Config) ++ Acc, Config); -clean(<<7, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(7, Config) ++ Acc, Config); -clean(<<8, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(8, Config) ++ Acc, Config); -clean(<<9, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(9, Config) ++ Acc, Config); -clean(<<10, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(10, Config) ++ Acc, Config); -clean(<<11, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(11, Config) ++ Acc, Config); -clean(<<12, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(12, Config) ++ Acc, Config); -clean(<<13, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(13, Config) ++ Acc, Config); -clean(<<14, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(14, Config) ++ Acc, Config); -clean(<<15, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(15, Config) ++ Acc, Config); -clean(<<16, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(16, Config) ++ Acc, Config); -clean(<<17, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(17, Config) ++ Acc, Config); -clean(<<18, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(18, Config) ++ Acc, Config); -clean(<<19, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(19, Config) ++ Acc, Config); -clean(<<20, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(20, Config) ++ Acc, Config); -clean(<<21, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(21, Config) ++ Acc, Config); -clean(<<22, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(22, Config) ++ Acc, Config); -clean(<<23, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(23, Config) ++ Acc, Config); -clean(<<24, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(24, Config) ++ Acc, Config); -clean(<<25, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(25, Config) ++ Acc, Config); -clean(<<26, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(26, Config) ++ Acc, Config); -clean(<<27, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(27, Config) ++ Acc, Config); -clean(<<28, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(28, Config) ++ Acc, Config); -clean(<<29, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(29, Config) ++ Acc, Config); -clean(<<30, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(30, Config) ++ Acc, Config); -clean(<<31, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(31, Config) ++ Acc, Config); +clean(<<0, Rest/binary>>, Acc, Config) -> maybe_replace(0, Rest, Acc, Config); +clean(<<1, Rest/binary>>, Acc, Config) -> maybe_replace(1, Rest, Acc, Config); +clean(<<2, Rest/binary>>, Acc, Config) -> maybe_replace(2, Rest, Acc, Config); +clean(<<3, Rest/binary>>, Acc, Config) -> maybe_replace(3, Rest, Acc, Config); +clean(<<4, Rest/binary>>, Acc, Config) -> maybe_replace(4, Rest, Acc, Config); +clean(<<5, Rest/binary>>, Acc, Config) -> maybe_replace(5, Rest, Acc, Config); +clean(<<6, Rest/binary>>, Acc, Config) -> maybe_replace(6, Rest, Acc, Config); +clean(<<7, Rest/binary>>, Acc, Config) -> maybe_replace(7, Rest, Acc, Config); +clean(<<8, Rest/binary>>, Acc, Config) -> maybe_replace(8, Rest, Acc, Config); +clean(<<9, Rest/binary>>, Acc, Config) -> maybe_replace(9, Rest, Acc, Config); +clean(<<10, Rest/binary>>, Acc, Config) -> maybe_replace(10, Rest, Acc, Config); +clean(<<11, Rest/binary>>, Acc, Config) -> maybe_replace(11, Rest, Acc, Config); +clean(<<12, Rest/binary>>, Acc, Config) -> maybe_replace(12, Rest, Acc, Config); +clean(<<13, Rest/binary>>, Acc, Config) -> maybe_replace(13, Rest, Acc, Config); +clean(<<14, Rest/binary>>, Acc, Config) -> maybe_replace(14, Rest, Acc, Config); +clean(<<15, Rest/binary>>, Acc, Config) -> maybe_replace(15, Rest, Acc, Config); +clean(<<16, Rest/binary>>, Acc, Config) -> maybe_replace(16, Rest, Acc, Config); +clean(<<17, Rest/binary>>, Acc, Config) -> maybe_replace(17, Rest, Acc, Config); +clean(<<18, Rest/binary>>, Acc, Config) -> maybe_replace(18, Rest, Acc, Config); +clean(<<19, Rest/binary>>, Acc, Config) -> maybe_replace(19, Rest, Acc, Config); +clean(<<20, Rest/binary>>, Acc, Config) -> maybe_replace(20, Rest, Acc, Config); +clean(<<21, Rest/binary>>, Acc, Config) -> maybe_replace(21, Rest, Acc, Config); +clean(<<22, Rest/binary>>, Acc, Config) -> maybe_replace(22, Rest, Acc, Config); +clean(<<23, Rest/binary>>, Acc, Config) -> maybe_replace(23, Rest, Acc, Config); +clean(<<24, Rest/binary>>, Acc, Config) -> maybe_replace(24, Rest, Acc, Config); +clean(<<25, Rest/binary>>, Acc, Config) -> maybe_replace(25, Rest, Acc, Config); +clean(<<26, Rest/binary>>, Acc, Config) -> maybe_replace(26, Rest, Acc, Config); +clean(<<27, Rest/binary>>, Acc, Config) -> maybe_replace(27, Rest, Acc, Config); +clean(<<28, Rest/binary>>, Acc, Config) -> maybe_replace(28, Rest, Acc, Config); +clean(<<29, Rest/binary>>, Acc, Config) -> maybe_replace(29, Rest, Acc, Config); +clean(<<30, Rest/binary>>, Acc, Config) -> maybe_replace(30, Rest, Acc, Config); +clean(<<31, Rest/binary>>, Acc, Config) -> maybe_replace(31, Rest, Acc, Config); clean(<<32, Rest/binary>>, Acc, Config) -> clean(Rest, [32] ++ Acc, Config); clean(<<33, Rest/binary>>, Acc, Config) -> clean(Rest, [33] ++ Acc, Config); -clean(<<34, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(34, Config) ++ Acc, Config); +clean(<<34, Rest/binary>>, Acc, Config) -> maybe_replace(34, Rest, Acc, Config); clean(<<35, Rest/binary>>, Acc, Config) -> clean(Rest, [35] ++ Acc, Config); clean(<<36, Rest/binary>>, Acc, Config) -> clean(Rest, [36] ++ Acc, Config); clean(<<37, Rest/binary>>, Acc, Config) -> clean(Rest, [37] ++ Acc, Config); @@ -408,7 +245,7 @@ clean(<<43, Rest/binary>>, Acc, Config) -> clean(Rest, [43] ++ Acc, Config); clean(<<44, Rest/binary>>, Acc, Config) -> clean(Rest, [44] ++ Acc, Config); clean(<<45, Rest/binary>>, Acc, Config) -> clean(Rest, [45] ++ Acc, Config); clean(<<46, Rest/binary>>, Acc, Config) -> clean(Rest, [46] ++ Acc, Config); -clean(<<47, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(47, Config) ++ Acc, Config); +clean(<<47, Rest/binary>>, Acc, Config) -> maybe_replace(47, Rest, Acc, Config); clean(<<48, Rest/binary>>, Acc, Config) -> clean(Rest, [48] ++ Acc, Config); clean(<<49, Rest/binary>>, Acc, Config) -> clean(Rest, [49] ++ Acc, Config); clean(<<50, Rest/binary>>, Acc, Config) -> clean(Rest, [50] ++ Acc, Config); @@ -453,7 +290,7 @@ clean(<<88, Rest/binary>>, Acc, Config) -> clean(Rest, [88] ++ Acc, Config); clean(<<89, Rest/binary>>, Acc, Config) -> clean(Rest, [89] ++ Acc, Config); clean(<<90, Rest/binary>>, Acc, Config) -> clean(Rest, [90] ++ Acc, Config); clean(<<91, Rest/binary>>, Acc, Config) -> clean(Rest, [91] ++ Acc, Config); -clean(<<92, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(92, Config) ++ Acc, Config); +clean(<<92, Rest/binary>>, Acc, Config) -> maybe_replace(92, Rest, Acc, Config); clean(<<93, Rest/binary>>, Acc, Config) -> clean(Rest, [93] ++ Acc, Config); clean(<<94, Rest/binary>>, Acc, Config) -> clean(Rest, [94] ++ Acc, Config); clean(<<95, Rest/binary>>, Acc, Config) -> clean(Rest, [95] ++ Acc, Config); @@ -490,7 +327,7 @@ clean(<<125, Rest/binary>>, Acc, Config) -> clean(Rest, [125] ++ Acc, Config); clean(<<126, Rest/binary>>, Acc, Config) -> clean(Rest, [126] ++ Acc, Config); clean(<<127, Rest/binary>>, Acc, Config) -> clean(Rest, [127] ++ Acc, Config); clean(<>, Acc, Config) when X == 16#2028; X == 16#2029 -> - clean(Rest, maybe_replace(X, Config) ++ Acc, Config); + maybe_replace(X, Rest, Acc, Config); clean(<>, Acc, Config) when X < 16#d800 -> clean(Rest, [X] ++ Acc, Config); clean(<>, Acc, Config) when X > 16#dfff, X < 16#fdd0 -> @@ -531,24 +368,24 @@ clean(<>, Acc, Config) when X >= 16#100000, X < 16#10fffe - clean(Rest, [X] ++ Acc, Config); %% surrogates clean(<<237, X, _, Rest/binary>>, Acc, Config) when X >= 160 -> - clean(Rest, maybe_replace(surrogate, Config) ++ Acc, Config); + maybe_replace(surrogate, Rest, Acc, Config); %% noncharacters clean(<<_/utf8, Rest/binary>>, Acc, Config) -> - clean(Rest, maybe_replace(noncharacter, Config) ++ Acc, Config); + maybe_replace(noncharacter, Rest, Acc, Config); %% u+fffe and u+ffff for R14BXX clean(<<239, 191, X, Rest/binary>>, Acc, Config) when X == 190; X == 191 -> - clean(Rest, maybe_replace(noncharacter, Config) ++ Acc, Config); + maybe_replace(noncharacter, Rest, Acc, Config); %% overlong encodings and missing continuations of a 2 byte sequence clean(<>, Acc, Config) when X >= 192, X =< 223 -> - clean(strip_continuations(Rest, 1), maybe_replace(badutf, Config) ++ Acc, Config); + maybe_replace(badutf, strip_continuations(Rest, 1), Acc, Config); %% overlong encodings and missing continuations of a 3 byte sequence clean(<>, Acc, Config) when X >= 224, X =< 239 -> - clean(strip_continuations(Rest, 2), maybe_replace(badutf, Config) ++ Acc, Config); + maybe_replace(badutf, strip_continuations(Rest, 2), Acc, Config); %% overlong encodings and missing continuations of a 4 byte sequence clean(<>, Acc, Config) when X >= 240, X =< 247 -> - clean(strip_continuations(Rest, 3), maybe_replace(badutf, Config) ++ Acc, Config); + maybe_replace(badutf, strip_continuations(Rest, 3), Acc, Config); clean(<<_, Rest/binary>>, Acc, Config) -> - clean(Rest, maybe_replace(badutf, Config) ++ Acc, Config). + maybe_replace(badutf, Rest, Acc, Config). strip_continuations(Bin, 0) -> Bin; @@ -558,29 +395,37 @@ strip_continuations(<>, N) when X >= 128, X =< 191 -> strip_continuations(Bin, _) -> Bin. -maybe_replace($\b, #config{escaped_strings=true}) -> [$b, $\\]; -maybe_replace($\t, #config{escaped_strings=true}) -> [$t, $\\]; -maybe_replace($\n, #config{escaped_strings=true}) -> [$n, $\\]; -maybe_replace($\f, #config{escaped_strings=true}) -> [$f, $\\]; -maybe_replace($\r, #config{escaped_strings=true}) -> [$r, $\\]; -maybe_replace($\", #config{escaped_strings=true}) -> [$\", $\\]; -maybe_replace($/, Config=#config{escaped_strings=true}) -> +maybe_replace($\b, Rest, Acc, Config=#config{escaped_strings=true}) -> + clean(Rest, [$b, $\\] ++ Acc, Config); +maybe_replace($\t, Rest, Acc, Config=#config{escaped_strings=true}) -> + clean(Rest, [$t, $\\] ++ Acc, Config); +maybe_replace($\n, Rest, Acc, Config=#config{escaped_strings=true}) -> + clean(Rest, [$n, $\\] ++ Acc, Config); +maybe_replace($\f, Rest, Acc, Config=#config{escaped_strings=true}) -> + clean(Rest, [$f, $\\] ++ Acc, Config); +maybe_replace($\r, Rest, Acc, Config=#config{escaped_strings=true}) -> + clean(Rest, [$r, $\\] ++ Acc, Config); +maybe_replace($\", Rest, Acc, Config=#config{escaped_strings=true}) -> + clean(Rest, [$\", $\\] ++ Acc, Config); +maybe_replace($/, Rest, Acc, Config=#config{escaped_strings=true}) -> case Config#config.escaped_forward_slashes of - true -> [$/, $\\]; - false -> [$/] + true -> clean(Rest, [$/, $\\] ++ Acc, Config); + false -> clean(Rest, [$/] ++ Acc, Config) end; -maybe_replace($\\, #config{escaped_strings=true}) -> [$\\, $\\]; -maybe_replace(X, Config=#config{escaped_strings=true}) when X == 16#2028; X == 16#2029 -> +maybe_replace($\\, Rest, Acc, Config=#config{escaped_strings=true}) -> + clean(Rest, [$\\, $\\] ++ Acc, Config); +maybe_replace(X, Rest, Acc, Config=#config{escaped_strings=true}) when X == 16#2028; X == 16#2029 -> case Config#config.unescaped_jsonp of - true -> [X]; - false -> lists:reverse(json_escape_sequence(X)) + true -> clean(Rest, [X] ++ Acc, Config); + false -> clean(Rest, lists:reverse(json_escape_sequence(X)) ++ Acc, Config) end; -maybe_replace(X, #config{escaped_strings=true}) when X < 32 -> - lists:reverse(json_escape_sequence(X)); -maybe_replace(noncharacter, #config{replaced_bad_utf8=true}) -> [16#fffd]; -maybe_replace(surrogate, #config{replaced_bad_utf8=true}) -> [16#fffd]; -maybe_replace(badutf, #config{replaced_bad_utf8=true}) -> [16#fffd]; -maybe_replace(_, _) -> {error, badarg}. +maybe_replace(X, Rest, Acc, Config=#config{escaped_strings=true}) when X < 32 -> + clean(Rest, lists:reverse(json_escape_sequence(X)) ++ Acc, Config); +maybe_replace(Atom, _, _, #config{strict_utf8=true}) when is_atom(Atom) -> {error, badarg}; +maybe_replace(noncharacter, Rest, Acc, Config) -> clean(Rest, [16#fffd] ++ Acc, Config); +maybe_replace(surrogate, Rest, Acc, Config) -> clean(Rest, [16#fffd] ++ Acc, Config); +maybe_replace(badutf, Rest, Acc, Config) -> clean(Rest, [16#fffd] ++ Acc, Config); +maybe_replace(X, Rest, Acc, Config) -> clean(Rest, [X] ++ Acc, Config). %% convert a codepoint to it's \uXXXX equiv. @@ -650,7 +495,7 @@ error_test_() -> {"value error", ?_assertError(badarg, parse_error([self()], []))}, {"maybe_done error", ?_assertError(badarg, parse_error([start_array, end_array, start_array, end_json], []))}, {"done error", ?_assertError(badarg, parse_error([{string, <<"">>}, {literal, true}, end_json], []))}, - {"string error", ?_assertError(badarg, parse_error([{string, <<239, 191, 191>>}, end_json], []))} + {"string error", ?_assertError(badarg, parse_error([{string, <<239, 191, 191>>}, end_json], [strict_utf8]))} ]. @@ -671,7 +516,7 @@ custom_error_handler_test_() -> )}, {"string error", ?_assertEqual( {string, [{string, <<239, 191, 191>>}, end_json]}, - parse_error([{string, <<239, 191, 191>>}, end_json], [{error_handler, Error}]) + parse_error([{string, <<239, 191, 191>>}, end_json], [{error_handler, Error}, strict_utf8]) )} ]. diff --git a/src/jsx_tests.hrl b/src/jsx_tests.hrl index fb4bf30..e6ec7aa 100644 --- a/src/jsx_tests.hrl +++ b/src/jsx_tests.hrl @@ -284,35 +284,35 @@ clean_string_test_() -> )}, {"error reserved space", ?_assertEqual( lists:duplicate(length(reserved_space()), {error, badarg}), - lists:map(fun(Codepoint) -> jsx_parser:clean_string(Codepoint, #config{}) end, reserved_space()) + lists:map(fun(Codepoint) -> jsx_parser:clean_string(Codepoint, #config{strict_utf8=true}) end, reserved_space()) )}, {"error surrogates", ?_assertEqual( lists:duplicate(length(surrogates()), {error, badarg}), - lists:map(fun(Codepoint) -> jsx_parser:clean_string(Codepoint, #config{}) end, surrogates()) + lists:map(fun(Codepoint) -> jsx_parser:clean_string(Codepoint, #config{strict_utf8=true}) end, surrogates()) )}, {"error noncharacters", ?_assertEqual( lists:duplicate(length(noncharacters()), {error, badarg}), - lists:map(fun(Codepoint) -> jsx_parser:clean_string(Codepoint, #config{}) end, noncharacters()) + lists:map(fun(Codepoint) -> jsx_parser:clean_string(Codepoint, #config{strict_utf8=true}) end, noncharacters()) )}, {"error extended noncharacters", ?_assertEqual( lists:duplicate(length(extended_noncharacters()), {error, badarg}), - lists:map(fun(Codepoint) -> jsx_parser:clean_string(Codepoint, #config{}) end, extended_noncharacters()) + lists:map(fun(Codepoint) -> jsx_parser:clean_string(Codepoint, #config{strict_utf8=true}) end, extended_noncharacters()) )}, {"clean reserved space", ?_assertEqual( lists:duplicate(length(reserved_space()), <<16#fffd/utf8>>), - lists:map(fun(Codepoint) -> jsx_parser:clean_string(Codepoint, #config{replaced_bad_utf8=true}) end, reserved_space()) + lists:map(fun(Codepoint) -> jsx_parser:clean_string(Codepoint, #config{}) end, reserved_space()) )}, {"clean surrogates", ?_assertEqual( lists:duplicate(length(surrogates()), <<16#fffd/utf8>>), - lists:map(fun(Codepoint) -> jsx_parser:clean_string(Codepoint, #config{replaced_bad_utf8=true}) end, surrogates()) + lists:map(fun(Codepoint) -> jsx_parser:clean_string(Codepoint, #config{}) end, surrogates()) )}, {"clean noncharacters", ?_assertEqual( lists:duplicate(length(noncharacters()), <<16#fffd/utf8>>), - lists:map(fun(Codepoint) -> jsx_parser:clean_string(Codepoint, #config{replaced_bad_utf8=true}) end, noncharacters()) + lists:map(fun(Codepoint) -> jsx_parser:clean_string(Codepoint, #config{}) end, noncharacters()) )}, {"clean extended noncharacters", ?_assertEqual( lists:duplicate(length(extended_noncharacters()), <<16#fffd/utf8>>), - lists:map(fun(Codepoint) -> jsx_parser:clean_string(Codepoint, #config{replaced_bad_utf8=true}) end, extended_noncharacters()) + lists:map(fun(Codepoint) -> jsx_parser:clean_string(Codepoint, #config{}) end, extended_noncharacters()) )} ]. @@ -492,190 +492,190 @@ bad_utf8_test_() -> [ {"noncharacter u+fffe", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(to_fake_utf8(16#fffe), #config{}) + jsx_parser:clean_string(to_fake_utf8(16#fffe), #config{strict_utf8=true}) )}, {"noncharacter u+fffe replaced", ?_assertEqual( <<16#fffd/utf8>>, - jsx_parser:clean_string(to_fake_utf8(16#fffe), #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(to_fake_utf8(16#fffe), #config{}) )}, {"noncharacter u+ffff", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(to_fake_utf8(16#ffff), #config{}) + jsx_parser:clean_string(to_fake_utf8(16#ffff), #config{strict_utf8=true}) )}, {"noncharacter u+ffff replaced", ?_assertEqual( <<16#fffd/utf8>>, - jsx_parser:clean_string(to_fake_utf8(16#ffff), #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(to_fake_utf8(16#ffff), #config{}) )}, {"orphan continuation byte u+0080", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<16#0080>>, #config{}) + jsx_parser:clean_string(<<16#0080>>, #config{strict_utf8=true}) )}, {"orphan continuation byte u+0080 replaced", ?_assertEqual( <<16#fffd/utf8>>, - jsx_parser:clean_string(<<16#0080>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<16#0080>>, #config{}) )}, {"orphan continuation byte u+00bf", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<16#00bf>>, #config{}) + jsx_parser:clean_string(<<16#00bf>>, #config{strict_utf8=true}) )}, {"orphan continuation byte u+00bf replaced", ?_assertEqual( <<16#fffd/utf8>>, - jsx_parser:clean_string(<<16#00bf>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<16#00bf>>, #config{}) )}, {"2 continuation bytes", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #config{}) + jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #config{strict_utf8=true}) )}, {"2 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 2), - jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #config{}) )}, {"3 continuation bytes", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{}) + jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{strict_utf8=true}) )}, {"3 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 3), - jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{}) )}, {"4 continuation bytes", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{}) + jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{strict_utf8=true}) )}, {"4 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 4), - jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{}) )}, {"5 continuation bytes", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{}) + jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{strict_utf8=true}) )}, {"5 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 5), - jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{}) )}, {"6 continuation bytes", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{}) + jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{strict_utf8=true}) )}, {"6 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 6), - jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{}) )}, {"all continuation bytes", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, #config{}) + jsx_parser:clean_string(<<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, #config{strict_utf8=true}) )}, {"all continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, length(lists:seq(16#0080, 16#00bf))), jsx_parser:clean_string( <<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, - #config{replaced_bad_utf8=true} + #config{} ) )}, {"lonely start byte", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<16#00c0>>, #config{}) + jsx_parser:clean_string(<<16#00c0>>, #config{strict_utf8=true}) )}, {"lonely start byte replaced", ?_assertEqual( <<16#fffd/utf8>>, - jsx_parser:clean_string(<<16#00c0>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<16#00c0>>, #config{}) )}, {"lonely start bytes (2 byte)", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<16#00c0, 32, 16#00df>>, #config{}) + jsx_parser:clean_string(<<16#00c0, 32, 16#00df>>, #config{strict_utf8=true}) )}, {"lonely start bytes (2 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, - jsx_parser:clean_string(<<16#00c0, 32, 16#00df>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<16#00c0, 32, 16#00df>>, #config{}) )}, {"lonely start bytes (3 byte)", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<16#00e0, 32, 16#00ef>>, #config{}) + jsx_parser:clean_string(<<16#00e0, 32, 16#00ef>>, #config{strict_utf8=true}) )}, {"lonely start bytes (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, - jsx_parser:clean_string(<<16#00e0, 32, 16#00ef>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<16#00e0, 32, 16#00ef>>, #config{}) )}, {"lonely start bytes (4 byte)", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<16#00f0, 32, 16#00f7>>, #config{}) + jsx_parser:clean_string(<<16#00f0, 32, 16#00f7>>, #config{strict_utf8=true}) )}, {"lonely start bytes (4 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, - jsx_parser:clean_string(<<16#00f0, 32, 16#00f7>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<16#00f0, 32, 16#00f7>>, #config{}) )}, {"missing continuation byte (3 byte)", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<224, 160, 32>>, #config{}) + jsx_parser:clean_string(<<224, 160, 32>>, #config{strict_utf8=true}) )}, {"missing continuation byte (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - jsx_parser:clean_string(<<224, 160, 32>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<224, 160, 32>>, #config{}) )}, {"missing continuation byte (4 byte missing one)", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<240, 144, 128, 32>>, #config{}) + jsx_parser:clean_string(<<240, 144, 128, 32>>, #config{strict_utf8=true}) )}, {"missing continuation byte (4 byte missing one) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - jsx_parser:clean_string(<<240, 144, 128, 32>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<240, 144, 128, 32>>, #config{}) )}, {"missing continuation byte (4 byte missing two)", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<240, 144, 32>>, #config{}) + jsx_parser:clean_string(<<240, 144, 32>>, #config{strict_utf8=true}) )}, {"missing continuation byte (4 byte missing two) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - jsx_parser:clean_string(<<240, 144, 32>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<240, 144, 32>>, #config{}) )}, {"overlong encoding of u+002f (2 byte)", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<16#c0, 16#af, 32>>, #config{}) + jsx_parser:clean_string(<<16#c0, 16#af, 32>>, #config{strict_utf8=true}) )}, {"overlong encoding of u+002f (2 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - jsx_parser:clean_string(<<16#c0, 16#af, 32>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<16#c0, 16#af, 32>>, #config{}) )}, {"overlong encoding of u+002f (3 byte)", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<16#e0, 16#80, 16#af, 32>>, #config{}) + jsx_parser:clean_string(<<16#e0, 16#80, 16#af, 32>>, #config{strict_utf8=true}) )}, {"overlong encoding of u+002f (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - jsx_parser:clean_string(<<16#e0, 16#80, 16#af, 32>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<16#e0, 16#80, 16#af, 32>>, #config{}) )}, {"overlong encoding of u+002f (4 byte)", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #config{}) + jsx_parser:clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #config{strict_utf8=true}) )}, {"overlong encoding of u+002f (4 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - jsx_parser:clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #config{}) )}, {"highest overlong 2 byte sequence", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<16#c1, 16#bf, 32>>, #config{}) + jsx_parser:clean_string(<<16#c1, 16#bf, 32>>, #config{strict_utf8=true}) )}, {"highest overlong 2 byte sequence replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - jsx_parser:clean_string(<<16#c1, 16#bf, 32>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<16#c1, 16#bf, 32>>, #config{}) )}, {"highest overlong 3 byte sequence", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #config{}) + jsx_parser:clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #config{strict_utf8=true}) )}, {"highest overlong 3 byte sequence replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - jsx_parser:clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #config{}) )}, {"highest overlong 4 byte sequence", ?_assertEqual( {error, badarg}, - jsx_parser:clean_string(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, #config{}) + jsx_parser:clean_string(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, #config{strict_utf8=true}) )}, {"highest overlong 4 byte sequence replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - jsx_parser:clean_string(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, #config{replaced_bad_utf8=true}) + jsx_parser:clean_string(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, #config{}) )} ].