From 230be8ebc2e341fac05dd3a61167c844b5df567a Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sat, 23 Feb 2013 22:36:35 -0800 Subject: [PATCH] import better tests --- src/jsx_utils.erl | 658 ++++++++++++++++++++++++++++++---------------- 1 file changed, 425 insertions(+), 233 deletions(-) diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index bbfca93..505a3b1 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -145,134 +145,355 @@ to_hex(X) -> X + 48. %% ascii "1" is [49], "2" is [50], etc... clean_string(Bin, #config{dirty_strings=true}) -> Bin; -clean_string(Bin, Config) -> clean_string(Bin, <<>>, Config). - -clean_string(Bin, Acc, Config) -> - Length = cut(Bin), - <> = Bin, - case Rest of - <<>> -> <>; - _ -> maybe_escape(Rest, <>, Config) +clean_string(Bin, Config) -> + case Config#config.replaced_bad_utf8 orelse Config#config.escaped_strings of + true -> clean(Bin, [], Config) + ; false -> ensure_clean(Bin), Bin end. -cut(Bin) -> cut(Bin, 0). +%% fast path for no escaping and no correcting, throws error if string is 'bad' +ensure_clean(<<>>) -> ok; +ensure_clean(<<0, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<1, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<2, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<3, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<4, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<5, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<6, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<7, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<8, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<9, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<10, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<11, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<12, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<13, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<14, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<15, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<16, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<17, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<18, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<19, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<20, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<21, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<22, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<23, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<24, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<25, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<26, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<27, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<28, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<29, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<30, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<31, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<32, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<33, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<34, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<35, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<36, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<37, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<38, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<39, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<40, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<41, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<42, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<43, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<44, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<45, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<46, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<47, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<48, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<49, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<50, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<51, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<52, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<53, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<54, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<55, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<56, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<57, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<58, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<59, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<60, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<61, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<62, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<63, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<64, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<65, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<66, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<67, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<68, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<69, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<70, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<71, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<72, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<73, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<74, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<75, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<76, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<77, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<78, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<79, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<80, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<81, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<82, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<83, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<84, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<85, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<86, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<87, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<88, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<89, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<90, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<91, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<92, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<93, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<94, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<95, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<96, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<97, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<98, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<99, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<100, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<101, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<102, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<103, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<104, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<105, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<106, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<107, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<108, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<109, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<110, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<111, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<112, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<113, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<114, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<115, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<116, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<117, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<118, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<119, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<120, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<121, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<122, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<123, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<124, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<125, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<126, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<127, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<>) when X < 16#dcff -> ensure_clean(Rest); +ensure_clean(<>) when X > 16#dfff, X < 16#fdd0 -> ensure_clean(Rest); +ensure_clean(<>) when X > 16#fdef, X < 16#fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#10000, X < 16#1fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#20000, X < 16#2fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#30000, X < 16#3fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#40000, X < 16#4fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#50000, X < 16#5fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#60000, X < 16#6fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#70000, X < 16#7fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#80000, X < 16#8fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#90000, X < 16#9fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#a0000, X < 16#afffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#b0000, X < 16#bfffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#c0000, X < 16#cfffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#d0000, X < 16#dfffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#e0000, X < 16#efffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#f0000, X < 16#ffffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#100000, X < 16#10fffe -> ensure_clean(Rest); +ensure_clean(Bin) -> erlang:error(badarg, [Bin]). -cut(<<32, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<33, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<35, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<36, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<37, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<38, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<39, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<40, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<41, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<42, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<43, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<44, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<45, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<46, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<48, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<49, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<50, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<51, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<52, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<53, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<54, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<55, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<56, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<57, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<58, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<59, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<60, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<61, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<62, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<63, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<64, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<65, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<66, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<67, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<68, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<69, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<70, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<71, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<72, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<73, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<74, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<75, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<76, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<77, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<78, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<79, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<80, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<81, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<82, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<83, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<84, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<85, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<86, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<87, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<88, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<89, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<90, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<91, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<93, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<94, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<95, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<96, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<97, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<98, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<99, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<100, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<101, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<102, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<103, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<104, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<105, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<106, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<107, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<108, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<109, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<110, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<111, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<112, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<113, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<114, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<115, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<116, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<117, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<118, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<119, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<120, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<121, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<122, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<123, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<124, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<125, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<126, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<127, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<>, N) when X >= 128, X < 16#0800 -> cut(Rest, N + 2); -cut(<>, N) when X >= 16#0800, X < 16#2028 -> cut(Rest, N + 3); -cut(<>, N) when X >= 16#202a, X < 16#d800 -> cut(Rest, N + 3); -cut(<>, N) when X > 16#dfff, X < 16#fdd0 -> cut(Rest, N + 3); -cut(<>, N) when X > 16#fdef, X < 16#fffe -> cut(Rest, N + 3); -cut(<>, N) when X >= 16#10000, X < 16#1fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#20000, X < 16#2fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#30000, X < 16#3fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#40000, X < 16#4fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#50000, X < 16#5fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#60000, X < 16#6fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#70000, X < 16#7fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#80000, X < 16#8fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#90000, X < 16#9fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#a0000, X < 16#afffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#b0000, X < 16#bfffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#c0000, X < 16#cfffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#d0000, X < 16#dfffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#e0000, X < 16#efffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#f0000, X < 16#ffffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#100000, X < 16#10fffe -> cut(Rest, N + 4); -cut(_, N) -> N. + +%% escape and/or replace bad codepoints if requested +clean(<<>>, Acc, _Config) -> unicode:characters_to_binary(lists:reverse(Acc)); +clean(<<0, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(0, Config) ++ Acc, Config); +clean(<<1, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(1, Config) ++ Acc, Config); +clean(<<2, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(2, Config) ++ Acc, Config); +clean(<<3, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(3, Config) ++ Acc, Config); +clean(<<4, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(4, Config) ++ Acc, Config); +clean(<<5, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(5, Config) ++ Acc, Config); +clean(<<6, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(6, Config) ++ Acc, Config); +clean(<<7, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(7, Config) ++ Acc, Config); +clean(<<8, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(8, Config) ++ Acc, Config); +clean(<<9, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(9, Config) ++ Acc, Config); +clean(<<10, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(10, Config) ++ Acc, Config); +clean(<<11, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(11, Config) ++ Acc, Config); +clean(<<12, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(12, Config) ++ Acc, Config); +clean(<<13, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(13, Config) ++ Acc, Config); +clean(<<14, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(14, Config) ++ Acc, Config); +clean(<<15, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(15, Config) ++ Acc, Config); +clean(<<16, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(16, Config) ++ Acc, Config); +clean(<<17, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(17, Config) ++ Acc, Config); +clean(<<18, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(18, Config) ++ Acc, Config); +clean(<<19, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(19, Config) ++ Acc, Config); +clean(<<20, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(20, Config) ++ Acc, Config); +clean(<<21, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(21, Config) ++ Acc, Config); +clean(<<22, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(22, Config) ++ Acc, Config); +clean(<<23, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(23, Config) ++ Acc, Config); +clean(<<24, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(24, Config) ++ Acc, Config); +clean(<<25, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(25, Config) ++ Acc, Config); +clean(<<26, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(26, Config) ++ Acc, Config); +clean(<<27, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(27, Config) ++ Acc, Config); +clean(<<28, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(28, Config) ++ Acc, Config); +clean(<<29, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(29, Config) ++ Acc, Config); +clean(<<30, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(30, Config) ++ Acc, Config); +clean(<<31, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(31, Config) ++ Acc, Config); +clean(<<32, Rest/binary>>, Acc, Config) -> clean(Rest, [32] ++ Acc, Config); +clean(<<33, Rest/binary>>, Acc, Config) -> clean(Rest, [33] ++ Acc, Config); +clean(<<34, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(34, Config) ++ Acc, Config); +clean(<<35, Rest/binary>>, Acc, Config) -> clean(Rest, [35] ++ Acc, Config); +clean(<<36, Rest/binary>>, Acc, Config) -> clean(Rest, [36] ++ Acc, Config); +clean(<<37, Rest/binary>>, Acc, Config) -> clean(Rest, [37] ++ Acc, Config); +clean(<<38, Rest/binary>>, Acc, Config) -> clean(Rest, [38] ++ Acc, Config); +clean(<<39, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(39, Config) ++ Acc, Config); +clean(<<40, Rest/binary>>, Acc, Config) -> clean(Rest, [40] ++ Acc, Config); +clean(<<41, Rest/binary>>, Acc, Config) -> clean(Rest, [41] ++ Acc, Config); +clean(<<42, Rest/binary>>, Acc, Config) -> clean(Rest, [42] ++ Acc, Config); +clean(<<43, Rest/binary>>, Acc, Config) -> clean(Rest, [43] ++ Acc, Config); +clean(<<44, Rest/binary>>, Acc, Config) -> clean(Rest, [44] ++ Acc, Config); +clean(<<45, Rest/binary>>, Acc, Config) -> clean(Rest, [45] ++ Acc, Config); +clean(<<46, Rest/binary>>, Acc, Config) -> clean(Rest, [46] ++ Acc, Config); +clean(<<47, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(47, Config) ++ Acc, Config); +clean(<<48, Rest/binary>>, Acc, Config) -> clean(Rest, [48] ++ Acc, Config); +clean(<<49, Rest/binary>>, Acc, Config) -> clean(Rest, [49] ++ Acc, Config); +clean(<<50, Rest/binary>>, Acc, Config) -> clean(Rest, [50] ++ Acc, Config); +clean(<<51, Rest/binary>>, Acc, Config) -> clean(Rest, [51] ++ Acc, Config); +clean(<<52, Rest/binary>>, Acc, Config) -> clean(Rest, [52] ++ Acc, Config); +clean(<<53, Rest/binary>>, Acc, Config) -> clean(Rest, [53] ++ Acc, Config); +clean(<<54, Rest/binary>>, Acc, Config) -> clean(Rest, [54] ++ Acc, Config); +clean(<<55, Rest/binary>>, Acc, Config) -> clean(Rest, [55] ++ Acc, Config); +clean(<<56, Rest/binary>>, Acc, Config) -> clean(Rest, [56] ++ Acc, Config); +clean(<<57, Rest/binary>>, Acc, Config) -> clean(Rest, [57] ++ Acc, Config); +clean(<<58, Rest/binary>>, Acc, Config) -> clean(Rest, [58] ++ Acc, Config); +clean(<<59, Rest/binary>>, Acc, Config) -> clean(Rest, [59] ++ Acc, Config); +clean(<<60, Rest/binary>>, Acc, Config) -> clean(Rest, [60] ++ Acc, Config); +clean(<<61, Rest/binary>>, Acc, Config) -> clean(Rest, [61] ++ Acc, Config); +clean(<<62, Rest/binary>>, Acc, Config) -> clean(Rest, [62] ++ Acc, Config); +clean(<<63, Rest/binary>>, Acc, Config) -> clean(Rest, [63] ++ Acc, Config); +clean(<<64, Rest/binary>>, Acc, Config) -> clean(Rest, [64] ++ Acc, Config); +clean(<<65, Rest/binary>>, Acc, Config) -> clean(Rest, [65] ++ Acc, Config); +clean(<<66, Rest/binary>>, Acc, Config) -> clean(Rest, [66] ++ Acc, Config); +clean(<<67, Rest/binary>>, Acc, Config) -> clean(Rest, [67] ++ Acc, Config); +clean(<<68, Rest/binary>>, Acc, Config) -> clean(Rest, [68] ++ Acc, Config); +clean(<<69, Rest/binary>>, Acc, Config) -> clean(Rest, [69] ++ Acc, Config); +clean(<<70, Rest/binary>>, Acc, Config) -> clean(Rest, [70] ++ Acc, Config); +clean(<<71, Rest/binary>>, Acc, Config) -> clean(Rest, [71] ++ Acc, Config); +clean(<<72, Rest/binary>>, Acc, Config) -> clean(Rest, [72] ++ Acc, Config); +clean(<<73, Rest/binary>>, Acc, Config) -> clean(Rest, [73] ++ Acc, Config); +clean(<<74, Rest/binary>>, Acc, Config) -> clean(Rest, [74] ++ Acc, Config); +clean(<<75, Rest/binary>>, Acc, Config) -> clean(Rest, [75] ++ Acc, Config); +clean(<<76, Rest/binary>>, Acc, Config) -> clean(Rest, [76] ++ Acc, Config); +clean(<<77, Rest/binary>>, Acc, Config) -> clean(Rest, [77] ++ Acc, Config); +clean(<<78, Rest/binary>>, Acc, Config) -> clean(Rest, [78] ++ Acc, Config); +clean(<<79, Rest/binary>>, Acc, Config) -> clean(Rest, [79] ++ Acc, Config); +clean(<<80, Rest/binary>>, Acc, Config) -> clean(Rest, [80] ++ Acc, Config); +clean(<<81, Rest/binary>>, Acc, Config) -> clean(Rest, [81] ++ Acc, Config); +clean(<<82, Rest/binary>>, Acc, Config) -> clean(Rest, [82] ++ Acc, Config); +clean(<<83, Rest/binary>>, Acc, Config) -> clean(Rest, [83] ++ Acc, Config); +clean(<<84, Rest/binary>>, Acc, Config) -> clean(Rest, [84] ++ Acc, Config); +clean(<<85, Rest/binary>>, Acc, Config) -> clean(Rest, [85] ++ Acc, Config); +clean(<<86, Rest/binary>>, Acc, Config) -> clean(Rest, [86] ++ Acc, Config); +clean(<<87, Rest/binary>>, Acc, Config) -> clean(Rest, [87] ++ Acc, Config); +clean(<<88, Rest/binary>>, Acc, Config) -> clean(Rest, [88] ++ Acc, Config); +clean(<<89, Rest/binary>>, Acc, Config) -> clean(Rest, [89] ++ Acc, Config); +clean(<<90, Rest/binary>>, Acc, Config) -> clean(Rest, [90] ++ Acc, Config); +clean(<<91, Rest/binary>>, Acc, Config) -> clean(Rest, [91] ++ Acc, Config); +clean(<<92, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(92, Config) ++ Acc, Config); +clean(<<93, Rest/binary>>, Acc, Config) -> clean(Rest, [93] ++ Acc, Config); +clean(<<94, Rest/binary>>, Acc, Config) -> clean(Rest, [94] ++ Acc, Config); +clean(<<95, Rest/binary>>, Acc, Config) -> clean(Rest, [95] ++ Acc, Config); +clean(<<96, Rest/binary>>, Acc, Config) -> clean(Rest, [96] ++ Acc, Config); +clean(<<97, Rest/binary>>, Acc, Config) -> clean(Rest, [97] ++ Acc, Config); +clean(<<98, Rest/binary>>, Acc, Config) -> clean(Rest, [98] ++ Acc, Config); +clean(<<99, Rest/binary>>, Acc, Config) -> clean(Rest, [99] ++ Acc, Config); +clean(<<100, Rest/binary>>, Acc, Config) -> clean(Rest, [100] ++ Acc, Config); +clean(<<101, Rest/binary>>, Acc, Config) -> clean(Rest, [101] ++ Acc, Config); +clean(<<102, Rest/binary>>, Acc, Config) -> clean(Rest, [102] ++ Acc, Config); +clean(<<103, Rest/binary>>, Acc, Config) -> clean(Rest, [103] ++ Acc, Config); +clean(<<104, Rest/binary>>, Acc, Config) -> clean(Rest, [104] ++ Acc, Config); +clean(<<105, Rest/binary>>, Acc, Config) -> clean(Rest, [105] ++ Acc, Config); +clean(<<106, Rest/binary>>, Acc, Config) -> clean(Rest, [106] ++ Acc, Config); +clean(<<107, Rest/binary>>, Acc, Config) -> clean(Rest, [107] ++ Acc, Config); +clean(<<108, Rest/binary>>, Acc, Config) -> clean(Rest, [108] ++ Acc, Config); +clean(<<109, Rest/binary>>, Acc, Config) -> clean(Rest, [109] ++ Acc, Config); +clean(<<110, Rest/binary>>, Acc, Config) -> clean(Rest, [110] ++ Acc, Config); +clean(<<111, Rest/binary>>, Acc, Config) -> clean(Rest, [111] ++ Acc, Config); +clean(<<112, Rest/binary>>, Acc, Config) -> clean(Rest, [112] ++ Acc, Config); +clean(<<113, Rest/binary>>, Acc, Config) -> clean(Rest, [113] ++ Acc, Config); +clean(<<114, Rest/binary>>, Acc, Config) -> clean(Rest, [114] ++ Acc, Config); +clean(<<115, Rest/binary>>, Acc, Config) -> clean(Rest, [115] ++ Acc, Config); +clean(<<116, Rest/binary>>, Acc, Config) -> clean(Rest, [116] ++ Acc, Config); +clean(<<117, Rest/binary>>, Acc, Config) -> clean(Rest, [117] ++ Acc, Config); +clean(<<118, Rest/binary>>, Acc, Config) -> clean(Rest, [118] ++ Acc, Config); +clean(<<119, Rest/binary>>, Acc, Config) -> clean(Rest, [119] ++ Acc, Config); +clean(<<120, Rest/binary>>, Acc, Config) -> clean(Rest, [120] ++ Acc, Config); +clean(<<121, Rest/binary>>, Acc, Config) -> clean(Rest, [121] ++ Acc, Config); +clean(<<122, Rest/binary>>, Acc, Config) -> clean(Rest, [122] ++ Acc, Config); +clean(<<123, Rest/binary>>, Acc, Config) -> clean(Rest, [123] ++ Acc, Config); +clean(<<124, Rest/binary>>, Acc, Config) -> clean(Rest, [124] ++ Acc, Config); +clean(<<125, Rest/binary>>, Acc, Config) -> clean(Rest, [125] ++ Acc, Config); +clean(<<126, Rest/binary>>, Acc, Config) -> clean(Rest, [126] ++ Acc, Config); +clean(<<127, Rest/binary>>, Acc, Config) -> clean(Rest, [127] ++ Acc, Config); +clean(<>, Acc, Config) when X == 16#2028; X == 16#2029 -> + clean(Rest, maybe_replace(X, Config) ++ Acc, Config); +clean(<>, Acc, Config) when X < 16#dcff -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X > 16#dfff, X < 16#fdd0 -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X > 16#fdef, X < 16#fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#10000, X < 16#1fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#20000, X < 16#2fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#30000, X < 16#3fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#40000, X < 16#4fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#50000, X < 16#5fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#60000, X < 16#6fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#70000, X < 16#7fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#80000, X < 16#8fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#90000, X < 16#9fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#a0000, X < 16#afffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#b0000, X < 16#bfffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#c0000, X < 16#cfffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#d0000, X < 16#dfffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#e0000, X < 16#efffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#f0000, X < 16#ffffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#100000, X < 16#10fffe -> + clean(Rest, [X] ++ Acc, Config); +%% noncharacters +clean(<<_/utf8, Rest/binary>>, Acc, Config) -> + clean(Rest, maybe_replace(noncharacter, Config) ++ Acc, Config); +%% surrogates +clean(<<237, X, _, Rest/binary>>, Acc, Config) when X >= 160 -> + clean(Rest, maybe_replace(surrogate, Config) ++ Acc, Config); +%% u+fffe and u+ffff for R14BXX +clean(<<239, 191, X, Rest/binary>>, Acc, Config) when X == 190; X == 191 -> + clean(Rest, maybe_replace(noncharacter, Config) ++ Acc, Config); +%% overlong encodings and missing continuations of a 2 byte sequence +clean(<>, Acc, Config) when X >= 192, X =< 223 -> + clean(strip_continuations(Rest, 1), maybe_replace(badutf, Config) ++ Acc, Config); +%% overlong encodings and missing continuations of a 3 byte sequence +clean(<>, Acc, Config) when X >= 224, X =< 239 -> + clean(strip_continuations(Rest, 2), maybe_replace(badutf, Config) ++ Acc, Config); +%% overlong encodings and missing continuations of a 4 byte sequence +clean(<>, Acc, Config) when X >= 240, X =< 247 -> + clean(strip_continuations(Rest, 3), maybe_replace(badutf, Config) ++ Acc, Config); +clean(<<_, Rest/binary>>, Acc, Config) -> + clean(Rest, maybe_replace(badutf, Config) ++ Acc, Config). strip_continuations(Bin, 0) -> Bin; @@ -282,63 +503,34 @@ strip_continuations(<>, N) when X >= 128, X =< 191 -> strip_continuations(Bin, _) -> Bin. -maybe_escape(<>, Acc, #config{escaped_strings=true} = Config) -> - case escape(Codepoint, Config) of - inescapable -> noncharacter(<>, Acc, Config); - Escaped -> clean_string(Rest, <>, Config) +maybe_replace($\b, #config{escaped_strings=true}) -> [$b, $\\]; +maybe_replace($\t, #config{escaped_strings=true}) -> [$t, $\\]; +maybe_replace($\n, #config{escaped_strings=true}) -> [$n, $\\]; +maybe_replace($\f, #config{escaped_strings=true}) -> [$f, $\\]; +maybe_replace($\r, #config{escaped_strings=true}) -> [$r, $\\]; +maybe_replace($\", #config{escaped_strings=true}) -> [$\", $\\]; +maybe_replace($', Config=#config{escaped_strings=true}) -> + case Config#config.single_quoted_strings of + true -> [$', $\\] + ; false -> [$'] end; -maybe_escape(<>, Acc, Config) -> - case escape(Codepoint, Config) of - inescapable -> noncharacter(<>, Acc, Config); - _ -> clean_string(Rest, <>, Config) +maybe_replace($/, Config=#config{escaped_strings=true}) -> + case Config#config.escaped_forward_slashes of + true -> [$/, $\\] + ; false -> [$/] end; -maybe_escape(Bin, Acc, Config) -> noncharacter(Bin, Acc, Config). - - -escape($\b, _) -> <<"\\b">>; -escape($\t, _) -> <<"\\t">>; -escape($\n, _) -> <<"\\n">>; -escape($\f, _) -> <<"\\f">>; -escape($\r, _) -> <<"\\r">>; -escape($\", _) -> <<"\\\"">>; -escape($\\, _) -> <<"\\\\">>; -escape($/, #config{escaped_forward_slashes=true}) -> <<"\\/">>; -escape($/, _) -> <<"/">>; -escape(16#2028, #config{unescaped_jsonp=true}) -> <<16#2028/utf8>>; -escape(16#2028, _) -> <<"\\u2028">>; -escape(16#2029, #config{unescaped_jsonp=true}) -> <<16#2029/utf8>>; -escape(16#2029, _) -> <<"\\u2029">>; -escape(X, _) when X < 32 -> - <> = <>, - <<"\\u", (to_hex(A)), (to_hex(B)), (to_hex(C)), (to_hex(D))>>; -escape(_, _) -> inescapable. - - -%% noncharacters and reserved space -noncharacter(<<_/utf8, Rest/binary>>, Acc, Config) -> - maybe_replace(Rest, Acc, Config); -%% surrogates -noncharacter(<<237, X, _, Rest/binary>>, Acc, Config) when X >= 160 -> - maybe_replace(Rest, Acc, Config); -%% u+fffe and u+ffff for R14BXX -noncharacter(<<239, 191, X, Rest/binary>>, Acc, Config) when X == 190; X == 191 -> - maybe_replace(Rest, Acc, Config); -%% overlong encodings and missing continuations of a 2 byte sequence -noncharacter(<>, Acc, Config) when X >= 192, X =< 223 -> - maybe_replace(strip_continuations(Rest, 1), Acc, Config); -%% overlong encodings and missing continuations of a 3 byte sequence -noncharacter(<>, Acc, Config) when X >= 224, X =< 239 -> - maybe_replace(strip_continuations(Rest, 2), Acc, Config); -%% overlong encodings and missing continuations of a 4 byte sequence -noncharacter(<>, Acc, Config) when X >= 240, X =< 247 -> - maybe_replace(strip_continuations(Rest, 3), Acc, Config); -noncharacter(<<_, Rest/binary>>, Acc, Config) -> - maybe_replace(Rest, Acc, Config). - - -maybe_replace(Bin, Acc, #config{replaced_bad_utf8=true} = Config) -> - clean_string(Bin, <>, Config); -maybe_replace(_, _, _) -> {error, badarg}. +maybe_replace($\\, #config{escaped_strings=true}) -> [$\\, $\\]; +maybe_replace(X, Config=#config{escaped_strings=true}) when X == 16#2028; X == 16#2029 -> + case Config#config.unescaped_jsonp of + true -> [X] + ; false -> lists:reverse(jsx_utils:json_escape_sequence(X)) + end; +maybe_replace(X, #config{escaped_strings=true}) when X < 32 -> + lists:reverse(jsx_utils:json_escape_sequence(X)); +maybe_replace(noncharacter, #config{replaced_bad_utf8=true}) -> [16#fffd]; +maybe_replace(surrogate, #config{replaced_bad_utf8=true}) -> [16#fffd]; +maybe_replace(badutf, #config{replaced_bad_utf8=true}) -> [16#fffd]; +maybe_replace(_, _) -> erlang:error(badarg). @@ -491,8 +683,8 @@ clean_string_test_() -> ] ++ [ { "reserved character: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), - ?_assertEqual( - {error, badarg}, + ?_assertError( + badarg, clean_string(Codepoint, #config{}) ) } || Codepoint <- reserved_space() @@ -507,8 +699,8 @@ clean_string_test_() -> ] ++ [ { "surrogate: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), - ?_assertEqual( - {error, badarg}, + ?_assertError( + badarg, clean_string(Codepoint, #config{}) ) } || Codepoint <- surrogates() @@ -523,8 +715,8 @@ clean_string_test_() -> ] ++ [ { "noncharacter: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), - ?_assertEqual( - {error, badarg}, + ?_assertError( + badarg, clean_string(Codepoint, #config{}) ) } || Codepoint <- noncharacters() @@ -539,8 +731,8 @@ clean_string_test_() -> ] ++ [ { "extended noncharacter: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), - ?_assertEqual( - {error, badarg}, + ?_assertError( + badarg, clean_string(Codepoint, #config{}) ) } || Codepoint <- extended_noncharacters() @@ -728,80 +920,80 @@ escape_test_() -> bad_utf8_test_() -> [ - {"noncharacter u+fffe", ?_assertEqual( - {error, badarg}, + {"noncharacter u+fffe", ?_assertError( + badarg, clean_string(to_fake_utf8(16#fffe), #config{}) )}, {"noncharacter u+fffe replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(to_fake_utf8(16#fffe), #config{replaced_bad_utf8=true}) )}, - {"noncharacter u+ffff", ?_assertEqual( - {error, badarg}, + {"noncharacter u+ffff", ?_assertError( + badarg, clean_string(to_fake_utf8(16#ffff), #config{}) )}, {"noncharacter u+ffff replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(to_fake_utf8(16#ffff), #config{replaced_bad_utf8=true}) )}, - {"orphan continuation byte u+0080", ?_assertEqual( - {error, badarg}, + {"orphan continuation byte u+0080", ?_assertError( + badarg, clean_string(<<16#0080>>, #config{}) )}, {"orphan continuation byte u+0080 replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(<<16#0080>>, #config{replaced_bad_utf8=true}) )}, - {"orphan continuation byte u+00bf", ?_assertEqual( - {error, badarg}, + {"orphan continuation byte u+00bf", ?_assertError( + badarg, clean_string(<<16#00bf>>, #config{}) )}, {"orphan continuation byte u+00bf replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(<<16#00bf>>, #config{replaced_bad_utf8=true}) )}, - {"2 continuation bytes", ?_assertEqual( - {error, badarg}, + {"2 continuation bytes", ?_assertError( + badarg, clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #config{}) )}, {"2 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 2), clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #config{replaced_bad_utf8=true}) )}, - {"3 continuation bytes", ?_assertEqual( - {error, badarg}, + {"3 continuation bytes", ?_assertError( + badarg, clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{}) )}, {"3 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 3), clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{replaced_bad_utf8=true}) )}, - {"4 continuation bytes", ?_assertEqual( - {error, badarg}, + {"4 continuation bytes", ?_assertError( + badarg, clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{}) )}, {"4 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 4), clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{replaced_bad_utf8=true}) )}, - {"5 continuation bytes", ?_assertEqual( - {error, badarg}, + {"5 continuation bytes", ?_assertError( + badarg, clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{}) )}, {"5 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 5), clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{replaced_bad_utf8=true}) )}, - {"6 continuation bytes", ?_assertEqual( - {error, badarg}, + {"6 continuation bytes", ?_assertError( + badarg, clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{}) )}, {"6 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 6), clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{replaced_bad_utf8=true}) )}, - {"all continuation bytes", ?_assertEqual( - {error, badarg}, + {"all continuation bytes", ?_assertError( + badarg, clean_string(<<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, #config{}) )}, {"all continuation bytes replaced", ?_assertEqual( @@ -811,104 +1003,104 @@ bad_utf8_test_() -> #config{replaced_bad_utf8=true} ) )}, - {"lonely start byte", ?_assertEqual( - {error, badarg}, + {"lonely start byte", ?_assertError( + badarg, clean_string(<<16#00c0>>, #config{}) )}, {"lonely start byte replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(<<16#00c0>>, #config{replaced_bad_utf8=true}) )}, - {"lonely start bytes (2 byte)", ?_assertEqual( - {error, badarg}, + {"lonely start bytes (2 byte)", ?_assertError( + badarg, clean_string(<<16#00c0, 32, 16#00df>>, #config{}) )}, {"lonely start bytes (2 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, clean_string(<<16#00c0, 32, 16#00df>>, #config{replaced_bad_utf8=true}) )}, - {"lonely start bytes (3 byte)", ?_assertEqual( - {error, badarg}, + {"lonely start bytes (3 byte)", ?_assertError( + badarg, clean_string(<<16#00e0, 32, 16#00ef>>, #config{}) )}, {"lonely start bytes (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, clean_string(<<16#00e0, 32, 16#00ef>>, #config{replaced_bad_utf8=true}) )}, - {"lonely start bytes (4 byte)", ?_assertEqual( - {error, badarg}, + {"lonely start bytes (4 byte)", ?_assertError( + badarg, clean_string(<<16#00f0, 32, 16#00f7>>, #config{}) )}, {"lonely start bytes (4 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, clean_string(<<16#00f0, 32, 16#00f7>>, #config{replaced_bad_utf8=true}) )}, - {"missing continuation byte (3 byte)", ?_assertEqual( - {error, badarg}, + {"missing continuation byte (3 byte)", ?_assertError( + badarg, clean_string(<<224, 160, 32>>, #config{}) )}, {"missing continuation byte (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<224, 160, 32>>, #config{replaced_bad_utf8=true}) )}, - {"missing continuation byte (4 byte missing one)", ?_assertEqual( - {error, badarg}, + {"missing continuation byte (4 byte missing one)", ?_assertError( + badarg, clean_string(<<240, 144, 128, 32>>, #config{}) )}, {"missing continuation byte (4 byte missing one) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<240, 144, 128, 32>>, #config{replaced_bad_utf8=true}) )}, - {"missing continuation byte (4 byte missing two)", ?_assertEqual( - {error, badarg}, + {"missing continuation byte (4 byte missing two)", ?_assertError( + badarg, clean_string(<<240, 144, 32>>, #config{}) )}, {"missing continuation byte (4 byte missing two) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<240, 144, 32>>, #config{replaced_bad_utf8=true}) )}, - {"overlong encoding of u+002f (2 byte)", ?_assertEqual( - {error, badarg}, + {"overlong encoding of u+002f (2 byte)", ?_assertError( + badarg, clean_string(<<16#c0, 16#af, 32>>, #config{}) )}, {"overlong encoding of u+002f (2 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#c0, 16#af, 32>>, #config{replaced_bad_utf8=true}) )}, - {"overlong encoding of u+002f (3 byte)", ?_assertEqual( - {error, badarg}, + {"overlong encoding of u+002f (3 byte)", ?_assertError( + badarg, clean_string(<<16#e0, 16#80, 16#af, 32>>, #config{}) )}, {"overlong encoding of u+002f (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#e0, 16#80, 16#af, 32>>, #config{replaced_bad_utf8=true}) )}, - {"overlong encoding of u+002f (4 byte)", ?_assertEqual( - {error, badarg}, + {"overlong encoding of u+002f (4 byte)", ?_assertError( + badarg, clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #config{}) )}, {"overlong encoding of u+002f (4 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #config{replaced_bad_utf8=true}) )}, - {"highest overlong 2 byte sequence", ?_assertEqual( - {error, badarg}, + {"highest overlong 2 byte sequence", ?_assertError( + badarg, clean_string(<<16#c1, 16#bf, 32>>, #config{}) )}, {"highest overlong 2 byte sequence replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#c1, 16#bf, 32>>, #config{replaced_bad_utf8=true}) )}, - {"highest overlong 3 byte sequence", ?_assertEqual( - {error, badarg}, + {"highest overlong 3 byte sequence", ?_assertError( + badarg, clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #config{}) )}, {"highest overlong 3 byte sequence replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #config{replaced_bad_utf8=true}) )}, - {"highest overlong 4 byte sequence", ?_assertEqual( - {error, badarg}, + {"highest overlong 4 byte sequence", ?_assertError( + badarg, clean_string(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, #config{}) )}, {"highest overlong 4 byte sequence replaced", ?_assertEqual(