umgeher/jsx
0
Fork 0
Code Issues Pull requests Activity

move all tests relating to cleaning of binary strings to jsx_utils

Browse source
This commit is contained in:
alisdair sullivan 2013-02-05 16:40:15 -08:00
parent 052a92d325
commit c99cbd9d12
3 changed files with 321 additions and 297 deletions
Download patch file Download diff file Expand all files Collapse all files

135
src/jsx_parser.erl
View file

@ -352,8 +352,6 @@ escapes_test_() ->
{"carriage return escape", ?_assertEqual(encode(<<"\r">>, [escaped_strings]), [{string, <<"\\r">>}, end_json])},
{"tab escape", ?_assertEqual(encode(<<"\t">>, [escaped_strings]), [{string, <<"\\t">>}, end_json])},
{"quote escape", ?_assertEqual(encode(<<"\"">>, [escaped_strings]), [{string, <<"\\\"">>}, end_json])},
{"single quote escape", ?_assertEqual(encode(<<"'">>, [escaped_strings, single_quoted_strings]), [{string, <<"\\'">>}, end_json])},
{"no single quote escape", ?_assertEqual(encode(<<"'">>, [escaped_strings]), [{string, <<"'">>}, end_json])},
{"forward slash escape", ?_assertEqual(encode(<<"/">>, [escaped_strings, escaped_forward_slashes]), [{string, <<"\\/">>}, end_json])},
{"no forward slash escape", ?_assertEqual(encode(<<"/">>, [escaped_strings]), [{string, <<"/">>}, end_json])},
{"back slash escape", ?_assertEqual(encode(<<"\\">>, [escaped_strings]), [{string, <<"\\\\">>}, end_json])},
@ -371,137 +369,4 @@ escapes_test_() ->
].
surrogates_test_() ->
[
{"surrogates - badarg",
?_assert(check_bad(surrogates()))
},
{"surrogates - replaced",
?_assert(check_replaced(surrogates()))
}
].
good_characters_test_() ->
[
{"acceptable codepoints",
?_assert(check_good(good()))
},
{"acceptable codepoints - escaped_strings",
?_assert(check_good(good(), [escaped_strings]))
},
{"acceptable codepoints - replaced_bad_utf8",
?_assert(check_good(good(), [escaped_strings]))
},
{"acceptable codepoints - escaped_strings + replaced_bad_utf8",
?_assert(check_good(good(), [escaped_strings, replaced_bad_utf8]))
},
{"acceptable extended",
?_assert(check_good(good_extended()))
},
{"acceptable extended - escaped_strings",
?_assert(check_good(good_extended(), [escaped_strings]))
},
{"acceptable extended - escaped_strings",
?_assert(check_good(good_extended(), [replaced_bad_utf8]))
}
].
reserved_test_() ->
[
{"reserved noncharacters - badarg",
?_assert(check_bad(reserved_space()))
},
{"reserved noncharacters - replaced",
?_assert(check_replaced(reserved_space()))
}
].
noncharacters_test_() ->
[
{"noncharacters - badarg",
?_assert(check_bad(noncharacters()))
},
{"noncharacters - replaced",
?_assert(check_replaced(noncharacters()))
}
].
extended_noncharacters_test_() ->
[
{"extended noncharacters - badarg",
?_assert(check_bad(extended_noncharacters()))
},
{"extended noncharacters - replaced",
?_assert(check_replaced(extended_noncharacters()))
}
].
check_bad(List) ->
[] == lists:dropwhile(fun({_, {error, badarg}}) -> true ; (_) -> false end,
check(List, [], [])
).
check_replaced(List) ->
[] == lists:dropwhile(fun({_, [{string, <<16#fffd/utf8>>}|_]}) -> true ; (_) -> false
end,
check(List, [replaced_bad_utf8], [])
).
check_good(List) -> check_good(List, []).
check_good(List, Opts) ->
[] == lists:dropwhile(fun({_, [{string, _}|_]}) -> true ; (_) -> false end,
check(List, Opts, [])
).
check([], _Opts, Acc) -> Acc;
check([H|T], Opts, Acc) ->
R = encode(to_fake_utf(H, utf8), Opts),
check(T, Opts, [{H, R}] ++ Acc).
noncharacters() -> lists:seq(16#fffe, 16#ffff).
extended_noncharacters() ->
[16#1fffe, 16#1ffff, 16#2fffe, 16#2ffff]
++ [16#3fffe, 16#3ffff, 16#4fffe, 16#4ffff]
++ [16#5fffe, 16#5ffff, 16#6fffe, 16#6ffff]
++ [16#7fffe, 16#7ffff, 16#8fffe, 16#8ffff]
++ [16#9fffe, 16#9ffff, 16#afffe, 16#affff]
++ [16#bfffe, 16#bffff, 16#cfffe, 16#cffff]
++ [16#dfffe, 16#dffff, 16#efffe, 16#effff]
++ [16#ffffe, 16#fffff, 16#10fffe, 16#10ffff].
surrogates() -> lists:seq(16#d800, 16#dfff).
reserved_space() -> lists:seq(16#fdd0, 16#fdef).
good() -> lists:seq(16#0000, 16#d7ff) ++ lists:seq(16#e000, 16#fdcf) ++ lists:seq(16#fdf0, 16#fffd).
good_extended() -> [16#10000, 16#20000, 16#30000, 16#40000, 16#50000,
16#60000, 16#70000, 16#80000, 16#90000, 16#a0000,
16#b0000, 16#c0000, 16#d0000, 16#e0000, 16#f0000
] ++ lists:seq(16#100000, 16#10fffd).
%% erlang refuses to encode certain codepoints, so fake them all
to_fake_utf(N, utf8) when N < 16#0080 -> <<N:8>>;
to_fake_utf(N, utf8) when N < 16#0800 ->
<<0:5, Y:5, X:6>> = <<N:16>>,
<<2#110:3, Y:5, 2#10:2, X:6>>;
to_fake_utf(N, utf8) when N < 16#10000 ->
<<Z:4, Y:6, X:6>> = <<N:16>>,
<<2#1110:4, Z:4, 2#10:2, Y:6, 2#10:2, X:6>>;
to_fake_utf(N, utf8) ->
<<0:3, W:3, Z:6, Y:6, X:6>> = <<N:24>>,
<<2#11110:5, W:3, 2#10:2, Z:6, 2#10:2, Y:6, 2#10:2, X:6>>.
-endif.