diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 4784b96..914b35a 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -53,7 +53,12 @@ start(Term, {Handler, State}, Config) -> value(String, {Handler, State}, Config) when is_binary(String) -> - Handler:handle_event({string, clean_string(String, Config)}, State); + case clean_string(String, Config) of + {error, badarg} -> + ?error([String, {Handler, State}, Config]); + CleanString -> + Handler:handle_event({string, CleanString}, State) + end; value(Float, {Handler, State}, _Config) when is_float(Float) -> Handler:handle_event({float, Float}, State); value(Int, {Handler, State}, _Config) when is_integer(Int) -> diff --git a/src/jsx_parser.erl b/src/jsx_parser.erl index 556d8ff..0cb146b 100644 --- a/src/jsx_parser.erl +++ b/src/jsx_parser.erl @@ -101,9 +101,19 @@ value([Number|Tokens], Handler, Stack, Config) when is_integer(Number) -> value([Number|Tokens], Handler, Stack, Config) when is_float(Number) -> value([{float, Number}] ++ Tokens, Handler, Stack, Config); value([{string, String}|Tokens], Handler, [], Config) when is_binary(String) -> - done(Tokens, handle_event({string, clean_string(String, Config)}, Handler, Config), [], Config); + case clean_string(String, Config) of + {error, badarg} -> + ?error([[{string, String}|Tokens], Handler, [], Config]); + CleanString -> + done(Tokens, handle_event({string, CleanString}, Handler, Config), [], Config) + end; value([{string, String}|Tokens], Handler, Stack, Config) when is_binary(String) -> - maybe_done(Tokens, handle_event({string, clean_string(String, Config)}, Handler, Config), Stack, Config); + case clean_string(String, Config) of + {error, badarg} -> + ?error([[{string, String}|Tokens], Handler, Stack, Config]); + CleanString -> + maybe_done(Tokens, handle_event({string, CleanString}, Handler, Config), Stack, Config) + end; value([String|Tokens], Handler, Stack, Config) when is_binary(String) -> value([{string, String}] ++ Tokens, Handler, Stack, Config); value([], Handler, Stack, Config) -> @@ -116,9 +126,14 @@ value(Token, Handler, Stack, Config) -> object([end_object|Tokens], Handler, [object|Stack], Config) -> maybe_done(Tokens, handle_event(end_object, Handler, Config), Stack, Config); object([{key, Key}|Tokens], Handler, Stack, Config) when is_atom(Key); is_binary(Key) -> - value(Tokens, handle_event({key, clean_string(fix_key(Key), Config)}, Handler, Config), Stack, Config); + case clean_string(fix_key(Key), Config) of + {error, badarg} -> + ?error([[{key, Key}|Tokens], Handler, Stack, Config]); + CleanString -> + value(Tokens, handle_event({key, CleanString}, Handler, Config), Stack, Config) + end; object([Key|Tokens], Handler, Stack, Config) when is_atom(Key); is_binary(Key) -> - value(Tokens, handle_event({key, clean_string(fix_key(Key), Config)}, Handler, Config), Stack, Config); + object([{key, Key}] ++ Tokens, Handler, Stack, Config); object([], Handler, Stack, Config) -> ?incomplete(object, Handler, Stack, Config); object(BadTokens, Handler, Stack, Config) when is_list(BadTokens) -> diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index 105082d..391d890 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -145,367 +145,80 @@ to_hex(X) -> X + 48. %% ascii "1" is [49], "2" is [50], etc... clean_string(Bin, #config{dirty_strings=true}) -> Bin; -clean_string(Bin, Config) -> - case Config#config.replaced_bad_utf8 orelse Config#config.escaped_strings of - true -> clean(Bin, [], Config) - ; false -> ensure_clean(Bin), Bin +clean_string(Bin, Config) -> clean_string(Bin, <<>>, Config). + + +clean_string(Bin, Acc, Config) -> + case cut(Bin, 0) of + {_, finished} -> <>; + {X, escape} -> + <> = Bin, + Escaped = maybe_escape(Codepoint, Config), + clean_string(Rest, <>, Config); + {X, replace, Y} -> + <> = Bin, + case maybe_replace(Bad, Config) of + {error, badarg} -> {error, badarg}; + Replaced -> clean_string(Rest, <>, Config) + end end. -%% fast path for no escaping and no correcting, throws error if string is 'bad' -ensure_clean(<<>>) -> ok; -ensure_clean(<<0, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<1, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<2, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<3, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<4, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<5, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<6, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<7, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<8, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<9, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<10, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<11, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<12, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<13, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<14, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<15, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<16, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<17, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<18, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<19, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<20, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<21, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<22, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<23, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<24, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<25, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<26, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<27, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<28, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<29, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<30, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<31, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<32, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<33, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<34, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<35, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<36, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<37, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<38, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<39, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<40, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<41, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<42, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<43, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<44, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<45, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<46, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<47, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<48, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<49, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<50, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<51, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<52, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<53, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<54, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<55, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<56, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<57, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<58, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<59, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<60, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<61, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<62, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<63, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<64, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<65, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<66, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<67, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<68, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<69, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<70, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<71, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<72, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<73, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<74, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<75, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<76, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<77, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<78, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<79, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<80, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<81, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<82, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<83, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<84, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<85, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<86, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<87, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<88, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<89, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<90, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<91, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<92, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<93, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<94, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<95, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<96, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<97, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<98, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<99, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<100, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<101, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<102, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<103, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<104, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<105, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<106, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<107, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<108, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<109, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<110, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<111, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<112, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<113, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<114, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<115, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<116, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<117, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<118, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<119, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<120, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<121, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<122, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<123, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<124, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<125, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<126, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<127, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<>) when X < 16#d800 -> ensure_clean(Rest); -ensure_clean(<>) when X > 16#dfff, X < 16#fdd0 -> ensure_clean(Rest); -ensure_clean(<>) when X > 16#fdef, X < 16#fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#10000, X < 16#1fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#20000, X < 16#2fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#30000, X < 16#3fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#40000, X < 16#4fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#50000, X < 16#5fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#60000, X < 16#6fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#70000, X < 16#7fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#80000, X < 16#8fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#90000, X < 16#9fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#a0000, X < 16#afffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#b0000, X < 16#bfffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#c0000, X < 16#cfffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#d0000, X < 16#dfffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#e0000, X < 16#efffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#f0000, X < 16#ffffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#100000, X < 16#10fffe -> ensure_clean(Rest); -ensure_clean(Bin) -> erlang:error(badarg, [Bin]). - - -%% escape and/or replace bad codepoints if requested -clean(<<>>, Acc, _Config) -> unicode:characters_to_binary(lists:reverse(Acc)); -clean(<<0, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(0, Config) ++ Acc, Config); -clean(<<1, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(1, Config) ++ Acc, Config); -clean(<<2, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(2, Config) ++ Acc, Config); -clean(<<3, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(3, Config) ++ Acc, Config); -clean(<<4, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(4, Config) ++ Acc, Config); -clean(<<5, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(5, Config) ++ Acc, Config); -clean(<<6, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(6, Config) ++ Acc, Config); -clean(<<7, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(7, Config) ++ Acc, Config); -clean(<<8, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(8, Config) ++ Acc, Config); -clean(<<9, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(9, Config) ++ Acc, Config); -clean(<<10, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(10, Config) ++ Acc, Config); -clean(<<11, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(11, Config) ++ Acc, Config); -clean(<<12, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(12, Config) ++ Acc, Config); -clean(<<13, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(13, Config) ++ Acc, Config); -clean(<<14, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(14, Config) ++ Acc, Config); -clean(<<15, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(15, Config) ++ Acc, Config); -clean(<<16, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(16, Config) ++ Acc, Config); -clean(<<17, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(17, Config) ++ Acc, Config); -clean(<<18, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(18, Config) ++ Acc, Config); -clean(<<19, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(19, Config) ++ Acc, Config); -clean(<<20, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(20, Config) ++ Acc, Config); -clean(<<21, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(21, Config) ++ Acc, Config); -clean(<<22, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(22, Config) ++ Acc, Config); -clean(<<23, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(23, Config) ++ Acc, Config); -clean(<<24, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(24, Config) ++ Acc, Config); -clean(<<25, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(25, Config) ++ Acc, Config); -clean(<<26, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(26, Config) ++ Acc, Config); -clean(<<27, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(27, Config) ++ Acc, Config); -clean(<<28, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(28, Config) ++ Acc, Config); -clean(<<29, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(29, Config) ++ Acc, Config); -clean(<<30, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(30, Config) ++ Acc, Config); -clean(<<31, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(31, Config) ++ Acc, Config); -clean(<<32, Rest/binary>>, Acc, Config) -> clean(Rest, [32] ++ Acc, Config); -clean(<<33, Rest/binary>>, Acc, Config) -> clean(Rest, [33] ++ Acc, Config); -clean(<<34, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(34, Config) ++ Acc, Config); -clean(<<35, Rest/binary>>, Acc, Config) -> clean(Rest, [35] ++ Acc, Config); -clean(<<36, Rest/binary>>, Acc, Config) -> clean(Rest, [36] ++ Acc, Config); -clean(<<37, Rest/binary>>, Acc, Config) -> clean(Rest, [37] ++ Acc, Config); -clean(<<38, Rest/binary>>, Acc, Config) -> clean(Rest, [38] ++ Acc, Config); -clean(<<39, Rest/binary>>, Acc, Config) -> clean(Rest, [39] ++ Acc, Config); -clean(<<40, Rest/binary>>, Acc, Config) -> clean(Rest, [40] ++ Acc, Config); -clean(<<41, Rest/binary>>, Acc, Config) -> clean(Rest, [41] ++ Acc, Config); -clean(<<42, Rest/binary>>, Acc, Config) -> clean(Rest, [42] ++ Acc, Config); -clean(<<43, Rest/binary>>, Acc, Config) -> clean(Rest, [43] ++ Acc, Config); -clean(<<44, Rest/binary>>, Acc, Config) -> clean(Rest, [44] ++ Acc, Config); -clean(<<45, Rest/binary>>, Acc, Config) -> clean(Rest, [45] ++ Acc, Config); -clean(<<46, Rest/binary>>, Acc, Config) -> clean(Rest, [46] ++ Acc, Config); -clean(<<47, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(47, Config) ++ Acc, Config); -clean(<<48, Rest/binary>>, Acc, Config) -> clean(Rest, [48] ++ Acc, Config); -clean(<<49, Rest/binary>>, Acc, Config) -> clean(Rest, [49] ++ Acc, Config); -clean(<<50, Rest/binary>>, Acc, Config) -> clean(Rest, [50] ++ Acc, Config); -clean(<<51, Rest/binary>>, Acc, Config) -> clean(Rest, [51] ++ Acc, Config); -clean(<<52, Rest/binary>>, Acc, Config) -> clean(Rest, [52] ++ Acc, Config); -clean(<<53, Rest/binary>>, Acc, Config) -> clean(Rest, [53] ++ Acc, Config); -clean(<<54, Rest/binary>>, Acc, Config) -> clean(Rest, [54] ++ Acc, Config); -clean(<<55, Rest/binary>>, Acc, Config) -> clean(Rest, [55] ++ Acc, Config); -clean(<<56, Rest/binary>>, Acc, Config) -> clean(Rest, [56] ++ Acc, Config); -clean(<<57, Rest/binary>>, Acc, Config) -> clean(Rest, [57] ++ Acc, Config); -clean(<<58, Rest/binary>>, Acc, Config) -> clean(Rest, [58] ++ Acc, Config); -clean(<<59, Rest/binary>>, Acc, Config) -> clean(Rest, [59] ++ Acc, Config); -clean(<<60, Rest/binary>>, Acc, Config) -> clean(Rest, [60] ++ Acc, Config); -clean(<<61, Rest/binary>>, Acc, Config) -> clean(Rest, [61] ++ Acc, Config); -clean(<<62, Rest/binary>>, Acc, Config) -> clean(Rest, [62] ++ Acc, Config); -clean(<<63, Rest/binary>>, Acc, Config) -> clean(Rest, [63] ++ Acc, Config); -clean(<<64, Rest/binary>>, Acc, Config) -> clean(Rest, [64] ++ Acc, Config); -clean(<<65, Rest/binary>>, Acc, Config) -> clean(Rest, [65] ++ Acc, Config); -clean(<<66, Rest/binary>>, Acc, Config) -> clean(Rest, [66] ++ Acc, Config); -clean(<<67, Rest/binary>>, Acc, Config) -> clean(Rest, [67] ++ Acc, Config); -clean(<<68, Rest/binary>>, Acc, Config) -> clean(Rest, [68] ++ Acc, Config); -clean(<<69, Rest/binary>>, Acc, Config) -> clean(Rest, [69] ++ Acc, Config); -clean(<<70, Rest/binary>>, Acc, Config) -> clean(Rest, [70] ++ Acc, Config); -clean(<<71, Rest/binary>>, Acc, Config) -> clean(Rest, [71] ++ Acc, Config); -clean(<<72, Rest/binary>>, Acc, Config) -> clean(Rest, [72] ++ Acc, Config); -clean(<<73, Rest/binary>>, Acc, Config) -> clean(Rest, [73] ++ Acc, Config); -clean(<<74, Rest/binary>>, Acc, Config) -> clean(Rest, [74] ++ Acc, Config); -clean(<<75, Rest/binary>>, Acc, Config) -> clean(Rest, [75] ++ Acc, Config); -clean(<<76, Rest/binary>>, Acc, Config) -> clean(Rest, [76] ++ Acc, Config); -clean(<<77, Rest/binary>>, Acc, Config) -> clean(Rest, [77] ++ Acc, Config); -clean(<<78, Rest/binary>>, Acc, Config) -> clean(Rest, [78] ++ Acc, Config); -clean(<<79, Rest/binary>>, Acc, Config) -> clean(Rest, [79] ++ Acc, Config); -clean(<<80, Rest/binary>>, Acc, Config) -> clean(Rest, [80] ++ Acc, Config); -clean(<<81, Rest/binary>>, Acc, Config) -> clean(Rest, [81] ++ Acc, Config); -clean(<<82, Rest/binary>>, Acc, Config) -> clean(Rest, [82] ++ Acc, Config); -clean(<<83, Rest/binary>>, Acc, Config) -> clean(Rest, [83] ++ Acc, Config); -clean(<<84, Rest/binary>>, Acc, Config) -> clean(Rest, [84] ++ Acc, Config); -clean(<<85, Rest/binary>>, Acc, Config) -> clean(Rest, [85] ++ Acc, Config); -clean(<<86, Rest/binary>>, Acc, Config) -> clean(Rest, [86] ++ Acc, Config); -clean(<<87, Rest/binary>>, Acc, Config) -> clean(Rest, [87] ++ Acc, Config); -clean(<<88, Rest/binary>>, Acc, Config) -> clean(Rest, [88] ++ Acc, Config); -clean(<<89, Rest/binary>>, Acc, Config) -> clean(Rest, [89] ++ Acc, Config); -clean(<<90, Rest/binary>>, Acc, Config) -> clean(Rest, [90] ++ Acc, Config); -clean(<<91, Rest/binary>>, Acc, Config) -> clean(Rest, [91] ++ Acc, Config); -clean(<<92, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(92, Config) ++ Acc, Config); -clean(<<93, Rest/binary>>, Acc, Config) -> clean(Rest, [93] ++ Acc, Config); -clean(<<94, Rest/binary>>, Acc, Config) -> clean(Rest, [94] ++ Acc, Config); -clean(<<95, Rest/binary>>, Acc, Config) -> clean(Rest, [95] ++ Acc, Config); -clean(<<96, Rest/binary>>, Acc, Config) -> clean(Rest, [96] ++ Acc, Config); -clean(<<97, Rest/binary>>, Acc, Config) -> clean(Rest, [97] ++ Acc, Config); -clean(<<98, Rest/binary>>, Acc, Config) -> clean(Rest, [98] ++ Acc, Config); -clean(<<99, Rest/binary>>, Acc, Config) -> clean(Rest, [99] ++ Acc, Config); -clean(<<100, Rest/binary>>, Acc, Config) -> clean(Rest, [100] ++ Acc, Config); -clean(<<101, Rest/binary>>, Acc, Config) -> clean(Rest, [101] ++ Acc, Config); -clean(<<102, Rest/binary>>, Acc, Config) -> clean(Rest, [102] ++ Acc, Config); -clean(<<103, Rest/binary>>, Acc, Config) -> clean(Rest, [103] ++ Acc, Config); -clean(<<104, Rest/binary>>, Acc, Config) -> clean(Rest, [104] ++ Acc, Config); -clean(<<105, Rest/binary>>, Acc, Config) -> clean(Rest, [105] ++ Acc, Config); -clean(<<106, Rest/binary>>, Acc, Config) -> clean(Rest, [106] ++ Acc, Config); -clean(<<107, Rest/binary>>, Acc, Config) -> clean(Rest, [107] ++ Acc, Config); -clean(<<108, Rest/binary>>, Acc, Config) -> clean(Rest, [108] ++ Acc, Config); -clean(<<109, Rest/binary>>, Acc, Config) -> clean(Rest, [109] ++ Acc, Config); -clean(<<110, Rest/binary>>, Acc, Config) -> clean(Rest, [110] ++ Acc, Config); -clean(<<111, Rest/binary>>, Acc, Config) -> clean(Rest, [111] ++ Acc, Config); -clean(<<112, Rest/binary>>, Acc, Config) -> clean(Rest, [112] ++ Acc, Config); -clean(<<113, Rest/binary>>, Acc, Config) -> clean(Rest, [113] ++ Acc, Config); -clean(<<114, Rest/binary>>, Acc, Config) -> clean(Rest, [114] ++ Acc, Config); -clean(<<115, Rest/binary>>, Acc, Config) -> clean(Rest, [115] ++ Acc, Config); -clean(<<116, Rest/binary>>, Acc, Config) -> clean(Rest, [116] ++ Acc, Config); -clean(<<117, Rest/binary>>, Acc, Config) -> clean(Rest, [117] ++ Acc, Config); -clean(<<118, Rest/binary>>, Acc, Config) -> clean(Rest, [118] ++ Acc, Config); -clean(<<119, Rest/binary>>, Acc, Config) -> clean(Rest, [119] ++ Acc, Config); -clean(<<120, Rest/binary>>, Acc, Config) -> clean(Rest, [120] ++ Acc, Config); -clean(<<121, Rest/binary>>, Acc, Config) -> clean(Rest, [121] ++ Acc, Config); -clean(<<122, Rest/binary>>, Acc, Config) -> clean(Rest, [122] ++ Acc, Config); -clean(<<123, Rest/binary>>, Acc, Config) -> clean(Rest, [123] ++ Acc, Config); -clean(<<124, Rest/binary>>, Acc, Config) -> clean(Rest, [124] ++ Acc, Config); -clean(<<125, Rest/binary>>, Acc, Config) -> clean(Rest, [125] ++ Acc, Config); -clean(<<126, Rest/binary>>, Acc, Config) -> clean(Rest, [126] ++ Acc, Config); -clean(<<127, Rest/binary>>, Acc, Config) -> clean(Rest, [127] ++ Acc, Config); -clean(<>, Acc, Config) when X < 16#800 -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X == 16#2028; X == 16#2029 -> - clean(Rest, maybe_replace(X, Config) ++ Acc, Config); -clean(<>, Acc, Config) when X < 16#d800 -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X > 16#dfff, X < 16#fdd0 -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X > 16#fdef, X < 16#fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#10000, X < 16#1fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#20000, X < 16#2fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#30000, X < 16#3fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#40000, X < 16#4fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#50000, X < 16#5fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#60000, X < 16#6fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#70000, X < 16#7fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#80000, X < 16#8fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#90000, X < 16#9fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#a0000, X < 16#afffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#b0000, X < 16#bfffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#c0000, X < 16#cfffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#d0000, X < 16#dfffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#e0000, X < 16#efffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#f0000, X < 16#ffffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#100000, X < 16#10fffe -> - clean(Rest, [X] ++ Acc, Config); -%% noncharacters -clean(<<_/utf8, Rest/binary>>, Acc, Config) -> - clean(Rest, maybe_replace(noncharacter, Config) ++ Acc, Config); +cut(<<>>, N) -> {N, finished}; +cut(<>, N) when X < 32 -> {N, escape}; +cut(<<$\"/utf8, _/binary>>, N) -> {N, escape}; +cut(<<$//utf8, _/binary>>, N) -> {N, escape}; +cut(<<$\\/utf8, _/binary>>, N) -> {N, escape}; +cut(<>, N) when X < 128 -> cut(Rest, N + 1); +cut(<>, N) when X < 16#0800 -> cut(Rest, N + 2); +cut(<>, N) when X == 16#2028; X == 16#2029 -> {N, escape}; +cut(<>, N) when X < 16#d800 -> cut(Rest, N + 3); +cut(<>, N) when X > 16#dfff, X < 16#fdd0 -> cut(Rest, N + 3); +cut(<>, N) when X > 16#fdef, X < 16#fffe -> cut(Rest, N + 3); +cut(<>, N) when X >= 16#10000, X < 16#1fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#20000, X < 16#2fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#30000, X < 16#3fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#40000, X < 16#4fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#50000, X < 16#5fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#60000, X < 16#6fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#70000, X < 16#7fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#80000, X < 16#8fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#90000, X < 16#9fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#a0000, X < 16#afffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#b0000, X < 16#bfffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#c0000, X < 16#cfffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#d0000, X < 16#dfffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#e0000, X < 16#efffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#f0000, X < 16#ffffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#100000, X < 16#10fffe -> cut(Rest, N + 4); +%% noncharacters and reserved space +cut(<>, N) -> + {N, replace, case X of Y when Y < 16#10000 -> 3; _ -> 4 end}; %% surrogates -clean(<<237, X, _, Rest/binary>>, Acc, Config) when X >= 160 -> - clean(Rest, maybe_replace(surrogate, Config) ++ Acc, Config); +cut(<<237, X, _, _/binary>>, N) when X >= 160 -> {N, replace, 3}; %% u+fffe and u+ffff for R14BXX -clean(<<239, 191, X, Rest/binary>>, Acc, Config) when X == 190; X == 191 -> - clean(Rest, maybe_replace(noncharacter, Config) ++ Acc, Config); +cut(<<239, 191, X, _/binary>>, N) when X == 190; X == 191 -> {N, replace, 3}; %% overlong encodings and missing continuations of a 2 byte sequence -clean(<>, Acc, Config) when X >= 192, X =< 223 -> - clean(strip_continuations(Rest, 1), maybe_replace(badutf, Config) ++ Acc, Config); +cut(<>, N) when X >= 192, X =< 223 -> + {N, replace, 1 + count_continuations(Rest, 1)}; %% overlong encodings and missing continuations of a 3 byte sequence -clean(<>, Acc, Config) when X >= 224, X =< 239 -> - clean(strip_continuations(Rest, 2), maybe_replace(badutf, Config) ++ Acc, Config); +cut(<>, N) when X >= 224, X =< 239 -> + {N, replace, 1 + count_continuations(Rest, 2)}; %% overlong encodings and missing continuations of a 4 byte sequence -clean(<>, Acc, Config) when X >= 240, X =< 247 -> - clean(strip_continuations(Rest, 3), maybe_replace(badutf, Config) ++ Acc, Config); -clean(<<_, Rest/binary>>, Acc, Config) -> - clean(Rest, maybe_replace(badutf, Config) ++ Acc, Config). +cut(<>, N) when X >= 240, X =< 247 -> + {N, replace, 1 + count_continuations(Rest, 3)}; +cut(<<_, _/binary>>, N) -> {N, replace, 1}. -strip_continuations(Bin, 0) -> Bin; -strip_continuations(<>, N) when X >= 128, X =< 191 -> - strip_continuations(Rest, N - 1); +count_continuations(Bin, N) -> count_continuations(Bin, N, 0). + +count_continuations(_Bin, 0, Acc) -> Acc; +count_continuations(<>, N, Acc) when X >= 128, X =< 191 -> + count_continuations(Rest, N - 1, Acc + 1); %% not a continuation byte -strip_continuations(Bin, _) -> Bin. +count_continuations(_Bin, _, Acc) -> Acc. -maybe_escape(Escaped, #config{dirty_strings=true}) -> <>; maybe_escape(Escaped, #config{escaped_strings=true} = Config) -> escape(Escaped, Config); maybe_escape(Escaped, _Config) -> <>. @@ -527,29 +240,9 @@ escape(X, _) when X < 32 -> <<"\\u", (to_hex(A)), (to_hex(B)), (to_hex(C)), (to_hex(D))>>. -maybe_replace($\b, #config{escaped_strings=true}) -> [$b, $\\]; -maybe_replace($\t, #config{escaped_strings=true}) -> [$t, $\\]; -maybe_replace($\n, #config{escaped_strings=true}) -> [$n, $\\]; -maybe_replace($\f, #config{escaped_strings=true}) -> [$f, $\\]; -maybe_replace($\r, #config{escaped_strings=true}) -> [$r, $\\]; -maybe_replace($\", #config{escaped_strings=true}) -> [$\", $\\]; -maybe_replace($\\, #config{escaped_strings=true}) -> [$\\, $\\]; -maybe_replace($/, Config) -> - case Config#config.escaped_forward_slashes of - true -> [$/, $\\] - ; false -> [$/] - end; -maybe_replace(X, Config=#config{escaped_strings=true}) when X == 16#2028; X == 16#2029 -> - case Config#config.unescaped_jsonp of - true -> [X] - ; false -> lists:reverse(jsx_utils:json_escape_sequence(X)) - end; -maybe_replace(X, #config{escaped_strings=true}) when X < 32 -> - lists:reverse(jsx_utils:json_escape_sequence(X)); -maybe_replace(noncharacter, #config{replaced_bad_utf8=true}) -> [16#fffd]; -maybe_replace(surrogate, #config{replaced_bad_utf8=true}) -> [16#fffd]; -maybe_replace(badutf, #config{replaced_bad_utf8=true}) -> [16#fffd]; -maybe_replace(_, _) -> erlang:error(badarg). +maybe_replace(_, #config{replaced_bad_utf8=true}) -> <<16#fffd/utf8>>; +maybe_replace(_, _Config) -> {error, badarg}. + @@ -661,11 +354,6 @@ codepoints() -> ++ lists:seq(16#fdf0, 16#fffd) ). -escapables() -> - [ to_fake_utf8(N) || N <- - lists:seq(0, 31) ++ [34, 92, 16#2028, 16#2029] - ]. - extended_codepoints() -> unicode:characters_to_binary( lists:seq(16#10000, 16#1fffd) ++ [ @@ -675,6 +363,10 @@ extended_codepoints() -> ] ). +reserved_space() -> [ to_fake_utf8(N) || N <- lists:seq(16#fdd0, 16#fdef) ]. + +surrogates() -> [ to_fake_utf8(N) || N <- lists:seq(16#d800, 16#dfff) ]. + noncharacters() -> [ to_fake_utf8(N) || N <- lists:seq(16#fffe, 16#ffff) ]. extended_noncharacters() -> @@ -688,83 +380,81 @@ extended_noncharacters() -> ++ [16#ffffe, 16#fffff, 16#10fffe, 16#10ffff] ]. -surrogates() -> [ to_fake_utf8(N) || N <- lists:seq(16#d800, 16#dfff) ]. -reserved_space() -> [ to_fake_utf8(N) || N <- lists:seq(16#fdd0, 16#fdef) ]. - - -fail_ensure(Codepoints, Title) -> - {generator, - fun() -> case Codepoints of - [N|Rest] -> - [ {Title, ?_assertError(badarg, ensure_clean(N))} - | fail_ensure(Rest, Title) - ] - ; [] -> [] - end end - }. - -fail_clean(Codepoints, Title) -> - {generator, - fun() -> case Codepoints of - [N|Rest] -> - [ {Title, ?_assertError(badarg, clean(N, [], #config{}))} - | fail_clean(Rest, Title) - ] - ; [] -> [] - end end - }. - -fail_bad(Codepoints, Title) -> - {generator, - fun() -> case Codepoints of - [N|Rest] -> - [ {Title, ?_assertError(badarg, clean(N, [], #config{}))} - | fail_bad(Rest, Title) - ] - ; [] -> [] - end end - }. - -replace_bad(Codepoints, Title) -> - {generator, - fun() -> case Codepoints of - [N|Rest] -> - [ {Title, ?_assertEqual(<<16#fffd/utf8>>, clean(N, [], #config{replaced_bad_utf8=true}))} - | replace_bad(Rest, Title) - ] - ; [] -> [] - end end - }. - - -ensure_clean_test_() -> +clean_string_test_() -> [ - {"basic codepoints", ?_assertEqual(ok, ensure_clean(codepoints()))}, - {"escapables", ?_assertEqual(ok, ensure_clean(unicode:characters_to_binary(escapables())))}, - {"extended codepoints", ?_assertEqual(ok, ensure_clean(extended_codepoints()))}, - fail_ensure(noncharacters(), "noncharacters"), - fail_ensure(extended_noncharacters(), "extended noncharacters"), - fail_ensure(surrogates(), "surrogates"), - fail_ensure(reserved_space(), "reserved space") - ]. - - -clean_test_() -> - [ - {"basic codepoints", ?_assertEqual( + {"clean codepoints test", ?_assertEqual( codepoints(), - clean(codepoints(), [], #config{}) + clean_string(codepoints(), #config{}) )}, - fail_clean(escapables(), "escapables"), - {"extended codepoints", ?_assertEqual( + {"clean extended codepoints test", ?_assertEqual( extended_codepoints(), - clean(extended_codepoints(), [], #config{}) - )}, - fail_clean(noncharacters(), "noncharacters"), - fail_clean(extended_noncharacters(), "extended noncharacters"), - fail_clean(surrogates(), "surrogates"), - fail_clean(reserved_space(), "reserved space") + clean_string(extended_codepoints(), #config{}) + )} + ] ++ [ + { + "reserved character: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), + ?_assertEqual( + {error, badarg}, + clean_string(Codepoint, #config{}) + ) + } || Codepoint <- reserved_space() + ] ++ [ + { + "reserved character: " ++ lists:flatten(io_lib:format("~p", [Codepoint])) ++ " (replaced)", + ?_assertEqual( + <<16#fffd/utf8>>, + clean_string(Codepoint, #config{replaced_bad_utf8=true}) + ) + } || Codepoint <- reserved_space() + ] ++ [ + { + "surrogate: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), + ?_assertEqual( + {error, badarg}, + clean_string(Codepoint, #config{}) + ) + } || Codepoint <- surrogates() + ] ++ [ + { + "surrogate: " ++ lists:flatten(io_lib:format("~p", [Codepoint])) ++ " (replaced)", + ?_assertEqual( + <<16#fffd/utf8>>, + clean_string(Codepoint, #config{replaced_bad_utf8=true}) + ) + } || Codepoint <- surrogates() + ] ++ [ + { + "noncharacter: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), + ?_assertEqual( + {error, badarg}, + clean_string(Codepoint, #config{}) + ) + } || Codepoint <- noncharacters() + ] ++ [ + { + "noncharacter: " ++ lists:flatten(io_lib:format("~p", [Codepoint])) ++ " (replaced)", + ?_assertEqual( + <<16#fffd/utf8>>, + clean_string(Codepoint, #config{replaced_bad_utf8=true}) + ) + } || Codepoint <- noncharacters() + ] ++ [ + { + "extended noncharacter: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), + ?_assertEqual( + {error, badarg}, + clean_string(Codepoint, #config{}) + ) + } || Codepoint <- extended_noncharacters() + ] ++ [ + { + "extended noncharacter: " ++ lists:flatten(io_lib:format("~p", [Codepoint])) ++ " (replaced)", + ?_assertEqual( + <<16#fffd/utf8>>, + clean_string(Codepoint, #config{replaced_bad_utf8=true}) + ) + } || Codepoint <- extended_noncharacters() ]. @@ -933,50 +623,46 @@ escape_test_() -> {"maybe_escape u001f", ?_assertEqual( <<"\\u001f">>, maybe_escape(16#001f, #config{escaped_strings=true}) - )}, - {"dirty strings", ?_assertEqual( - <<0>>, - maybe_escape(16#0000, #config{escaped_strings=true, dirty_strings=true}) )} ]. bad_utf8_test_() -> [ - {"noncharacter u+fffe", ?_assertError(badarg, clean_string(to_fake_utf8(16#fffe), #config{}))}, + {"noncharacter u+fffe", ?_assertEqual( + {error, badarg}, + clean_string(to_fake_utf8(16#fffe), #config{}) + )}, {"noncharacter u+fffe replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(to_fake_utf8(16#fffe), #config{replaced_bad_utf8=true}) )}, - {"noncharacter u+ffff", ?_assertError(badarg, clean_string(to_fake_utf8(16#ffff), #config{}))}, + {"noncharacter u+ffff", ?_assertEqual( + {error, badarg}, + clean_string(to_fake_utf8(16#ffff), #config{}) + )}, {"noncharacter u+ffff replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(to_fake_utf8(16#ffff), #config{replaced_bad_utf8=true}) )}, - fail_bad(extended_noncharacters(), "extended noncharacters"), - replace_bad(extended_noncharacters(), "extended noncharacters replaced"), - fail_bad(surrogates(), "surrogates"), - replace_bad(surrogates(), "surrogates replaced"), - fail_bad(reserved_space(), "reserved space"), - replace_bad(reserved_space(), "reserved space replaced"), - {"orphan continuation byte u+0080", ?_assertError( - badarg, + {"orphan continuation byte u+0080", ?_assertEqual( + {error, badarg}, clean_string(<<16#0080>>, #config{}) )}, {"orphan continuation byte u+0080 replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(<<16#0080>>, #config{replaced_bad_utf8=true}) )}, - {"orphan continuation byte u+00bf", ?_assertError( - badarg, + {"orphan continuation byte u+00bf", ?_assertEqual( + {error, badarg}, clean_string(<<16#00bf>>, #config{}) )}, {"orphan continuation byte u+00bf replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(<<16#00bf>>, #config{replaced_bad_utf8=true}) )}, - {"2 continuation bytes", ?_assertError( - badarg, + {"2 continuation bytes", ?_assertEqual( + {error, badarg}, clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #config{}) )}, {"2 continuation bytes replaced", ?_assertEqual( @@ -984,35 +670,35 @@ bad_utf8_test_() -> clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #config{replaced_bad_utf8=true}) )}, {"3 continuation bytes", - ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{})) + ?_assertEqual({error, badarg}, clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{})) }, {"3 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 3), clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{replaced_bad_utf8=true}) )}, {"4 continuation bytes", - ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{})) + ?_assertEqual({error, badarg}, clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{})) }, {"4 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 4), clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{replaced_bad_utf8=true}) )}, {"5 continuation bytes", - ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{})) + ?_assertEqual({error, badarg}, clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{})) }, {"5 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 5), clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{replaced_bad_utf8=true}) )}, {"6 continuation bytes", - ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{})) + ?_assertEqual({error, badarg}, clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{})) }, {"6 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 6), clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{replaced_bad_utf8=true}) )}, - {"all continuation bytes", ?_assertError( - badarg, + {"all continuation bytes", ?_assertEqual( + {error, badarg}, clean_string(<<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, #config{}) )}, {"all continuation bytes replaced", ?_assertEqual( @@ -1022,101 +708,101 @@ bad_utf8_test_() -> #config{replaced_bad_utf8=true} ) )}, - {"lonely start byte", ?_assertError(badarg, clean_string(<<16#00c0>>, #config{}))}, + {"lonely start byte", ?_assertEqual({error, badarg}, clean_string(<<16#00c0>>, #config{}))}, {"lonely start byte replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(<<16#00c0>>, #config{replaced_bad_utf8=true}) )}, - {"lonely start bytes (2 byte)", ?_assertError( - badarg, + {"lonely start bytes (2 byte)", ?_assertEqual( + {error, badarg}, clean_string(<<16#00c0, 32, 16#00df>>, #config{}) )}, {"lonely start bytes (2 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, clean_string(<<16#00c0, 32, 16#00df>>, #config{replaced_bad_utf8=true}) )}, - {"lonely start bytes (3 byte)", ?_assertError( - badarg, + {"lonely start bytes (3 byte)", ?_assertEqual( + {error, badarg}, clean_string(<<16#00e0, 32, 16#00ef>>, #config{}) )}, {"lonely start bytes (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, clean_string(<<16#00e0, 32, 16#00ef>>, #config{replaced_bad_utf8=true}) )}, - {"lonely start bytes (4 byte)", ?_assertError( - badarg, + {"lonely start bytes (4 byte)", ?_assertEqual( + {error, badarg}, clean_string(<<16#00f0, 32, 16#00f7>>, #config{}) )}, {"lonely start bytes (4 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, clean_string(<<16#00f0, 32, 16#00f7>>, #config{replaced_bad_utf8=true}) )}, - {"missing continuation byte (3 byte)", ?_assertError( - badarg, + {"missing continuation byte (3 byte)", ?_assertEqual( + {error, badarg}, clean_string(<<224, 160, 32>>, #config{}) )}, {"missing continuation byte (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<224, 160, 32>>, #config{replaced_bad_utf8=true}) )}, - {"missing continuation byte (4 byte missing one)", ?_assertError( - badarg, + {"missing continuation byte (4 byte missing one)", ?_assertEqual( + {error, badarg}, clean_string(<<240, 144, 128, 32>>, #config{}) )}, {"missing continuation byte (4 byte missing one) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<240, 144, 128, 32>>, #config{replaced_bad_utf8=true}) )}, - {"missing continuation byte (4 byte missing two)", ?_assertError( - badarg, + {"missing continuation byte (4 byte missing two)", ?_assertEqual( + {error, badarg}, clean_string(<<240, 144, 32>>, #config{}) )}, {"missing continuation byte (4 byte missing two) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<240, 144, 32>>, #config{replaced_bad_utf8=true}) )}, - {"overlong encoding of u+002f (2 byte)", ?_assertError( - badarg, + {"overlong encoding of u+002f (2 byte)", ?_assertEqual( + {error, badarg}, clean_string(<<16#c0, 16#af, 32>>, #config{}) )}, {"overlong encoding of u+002f (2 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#c0, 16#af, 32>>, #config{replaced_bad_utf8=true}) )}, - {"overlong encoding of u+002f (3 byte)", ?_assertError( - badarg, + {"overlong encoding of u+002f (3 byte)", ?_assertEqual( + {error, badarg}, clean_string(<<16#e0, 16#80, 16#af, 32>>, #config{}) )}, {"overlong encoding of u+002f (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#e0, 16#80, 16#af, 32>>, #config{replaced_bad_utf8=true}) )}, - {"overlong encoding of u+002f (4 byte)", ?_assertError( - badarg, + {"overlong encoding of u+002f (4 byte)", ?_assertEqual( + {error, badarg}, clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #config{}) )}, {"overlong encoding of u+002f (4 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #config{replaced_bad_utf8=true}) )}, - {"highest overlong 2 byte sequence", ?_assertError( - badarg, + {"highest overlong 2 byte sequence", ?_assertEqual( + {error, badarg}, clean_string(<<16#c1, 16#bf, 32>>, #config{}) )}, {"highest overlong 2 byte sequence replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#c1, 16#bf, 32>>, #config{replaced_bad_utf8=true}) )}, - {"highest overlong 3 byte sequence", ?_assertError( - badarg, + {"highest overlong 3 byte sequence", ?_assertEqual( + {error, badarg}, clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #config{}) )}, {"highest overlong 3 byte sequence replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #config{replaced_bad_utf8=true}) )}, - {"highest overlong 4 byte sequence", ?_assertError( - badarg, + {"highest overlong 4 byte sequence", ?_assertEqual( + {error, badarg}, clean_string(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, #config{}) )}, {"highest overlong 4 byte sequence replaced", ?_assertEqual(