From bd3a64f457c58b794817272b9a9c6d2c0e3e532f Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Thu, 31 Jan 2013 06:18:29 -0800 Subject: [PATCH 01/61] remove integration tests from jsx module --- src/jsx.erl | 223 +--------------------------------------------------- 1 file changed, 1 insertion(+), 222 deletions(-) diff --git a/src/jsx.erl b/src/jsx.erl index b4eb526..2534540 100644 --- a/src/jsx.erl +++ b/src/jsx.erl @@ -159,232 +159,11 @@ parser(Handler, State, Opts) -> jsx_parser:parser(Handler, State, Opts). -include_lib("eunit/include/eunit.hrl"). -jsx_decoder_test_() -> - jsx_decoder_gen(load_tests(code:lib_dir(jsx, priv) ++ "/test_cases/")). - - -encoder_decoder_equiv_test_() -> - [ - {"encoder/decoder equivalency", - ?_assert((jsx_decoder:decoder(?MODULE, [], []))( - <<"[\"a\", 17, 3.14, true, {\"k\":false}, []]">> - ) =:= (jsx_encoder:encoder(?MODULE, [], []))([<<"a">>, 17, 3.14, true, [{<<"k">>, false}], []]) - ) - } - ]. - - -partial_numbers_test_() -> - [ - {"partial integer", - ?_assert(begin - {incomplete, F} = jsx:decode(<<"{\"integer\": 12345">>), - F(<<"67890}">>) - end =:= [{<<"integer">>, 1234567890}] - ) - }, - {"partial integer", - ?_assert(begin - {incomplete, F} = jsx:decode(<<"{\"integer\": 1234567890">>), - F(<<"}">>) - end =:= [{<<"integer">>, 1234567890}] - ) - }, - {"partial float", - ?_assert(begin - {incomplete, F} = jsx:decode(<<"{\"float\": 1.">>), - F(<<"23}">>) - end =:= [{<<"float">>, 1.23}] - ) - }, - {"partial float", - ?_assert(begin - {incomplete, F} = jsx:decode(<<"{\"float\": 1.2">>), - F(<<"3}">>) - end =:= [{<<"float">>, 1.23}] - ) - }, - {"partial float", - ?_assert(begin - {incomplete, F} = jsx:decode(<<"{\"float\": 1.23">>), - F(<<"}">>) - end =:= [{<<"float">>, 1.23}] - ) - }, - {"partial exp", - ?_assert(begin - {incomplete, F} = jsx:decode(<<"{\"exp\": 1.0e">>), - F(<<"1}">>) - end =:= [{<<"exp">>, 10.0}] - ) - }, - {"partial exp", - ?_assert(begin - {incomplete, F} = jsx:decode(<<"{\"exp\": 1.0e1">>), - F(<<"2}">>) - end =:= [{<<"exp">>, 1.0e12}] - ) - }, - {"partial exp", - ?_assert(begin - {incomplete, F} = jsx:decode(<<"{\"exp\": 1.0e1">>), - F(<<"}">>) - end =:= [{<<"exp">>, 10.0}] - ) - }, - {"partial zero", - ?_assert(begin - {incomplete, F} = jsx:decode(<<"{\"zero\": 0">>), - F(<<".0}">>) - end =:= [{<<"zero">>, 0.0}] - ) - }, - {"partial zero", - ?_assert(begin - {incomplete, F} = jsx:decode(<<"{\"zero\": 0">>), - F(<<"}">>) - end =:= [{<<"zero">>, 0}] - ) - } - ]. - - -single_quoted_strings_test_() -> - [ - {"single quoted keys", - ?_assertEqual( - to_term(<<"{'key':true}">>, [single_quoted_strings]), - [{<<"key">>, true}] - ) - }, - {"multiple single quoted keys", - ?_assertEqual( - to_term(<<"{'key':true, 'another key':true}">>, [single_quoted_strings]), - [{<<"key">>, true}, {<<"another key">>, true}] - ) - }, - {"nested single quoted keys", - ?_assertEqual( - to_term(<<"{'key': {'key':true, 'another key':true}}">>, [single_quoted_strings]), - [{<<"key">>, [{<<"key">>, true}, {<<"another key">>, true}]}] - ) - }, - {"single quoted string", - ?_assertEqual( - to_term(<<"['string']">>, [single_quoted_strings]), - [<<"string">>] - ) - }, - {"single quote in double quoted string", - ?_assertEqual( - to_term(<<"[\"a single quote: '\"]">>, [single_quoted_strings]), - [<<"a single quote: '">>] - ) - }, - {"escaped single quote in single quoted string", - ?_assertEqual( - to_term(<<"['a single quote: \\'']">>, [single_quoted_strings]), - [<<"a single quote: '">>] - ) - }, - {"escaped single quote when single quotes are disallowed", - ?_assertError( - badarg, - to_term(<<"[\"a single quote: \\'\"]">>) - ) - }, - {"mismatched quotes", - ?_assertError( - badarg, - to_term(<<"['mismatched\"]">>, [single_quoted_strings]) - ) - } - ]. - - -%% test handler +%% stub test handler init([]) -> []. handle_event(end_json, State) -> lists:reverse([end_json] ++ State); handle_event(Event, State) -> [Event] ++ State. - -jsx_decoder_gen([]) -> []; -jsx_decoder_gen([Test|Rest]) -> - Name = proplists:get_value(name, Test), - JSON = proplists:get_value(json, Test), - JSX = proplists:get_value(jsx, Test), - Flags = proplists:get_value(jsx_flags, Test, []), - {generator, fun() -> - [{Name, ?_assertEqual(test_decode(JSON, Flags), JSX)}, - {Name ++ " (incremental)", - ?_assertEqual(incremental_decode(JSON, Flags), JSX) - } - | jsx_decoder_gen(Rest) - ] - end}. - - -load_tests(Path) -> - %% search the specified directory for any files with the .test ending - TestSpecs = filelib:wildcard("*.test", Path), - load_tests(TestSpecs, Path, []). - -load_tests([], _Dir, Acc) -> - lists:reverse(Acc); -load_tests([Test|Rest], Dir, Acc) -> - case file:consult(Dir ++ "/" ++ Test) of - {ok, TestSpec} -> - ParsedTest = parse_tests(TestSpec, Dir), - load_tests(Rest, Dir, [ParsedTest] ++ Acc) - ; {error, _Reason} -> - erlang:error(badarg, [Test|Rest], Dir, Acc) - end. - - -parse_tests(TestSpec, Dir) -> - parse_tests(TestSpec, Dir, []). - -parse_tests([{json, Path}|Rest], Dir, Acc) when is_list(Path) -> - case file:read_file(Dir ++ "/" ++ Path) of - {ok, Bin} -> parse_tests(Rest, Dir, [{json, Bin}] ++ Acc) - ; _ -> erlang:error(badarg, [[{json, Path}|Rest], Dir, Acc]) - end; -parse_tests([KV|Rest], Dir, Acc) -> - parse_tests(Rest, Dir, [KV] ++ Acc); -parse_tests([], _Dir, Acc) -> - Acc. - - -test_decode(JSON, Flags) -> - try - case (jsx_decoder:decoder(?MODULE, [], Flags))(JSON) of - {incomplete, More} -> - case More(<<" ">>) of - {incomplete, _} -> {error, badarg} - ; Events -> Events - end - ; Events -> Events - end - catch - error:badarg -> {error, badarg} - end. - - -incremental_decode(<>, Flags) -> - P = jsx_decoder:decoder(?MODULE, [], Flags ++ [explicit_end]), - try incremental_decode_loop(P(C), Rest) - catch error:badarg -> {error, badarg} - end. - -incremental_decode_loop({incomplete, More}, <<>>) -> - case More(end_stream) of - {incomplete, _} -> {error, badarg} - ; X -> X - end; -incremental_decode_loop({incomplete, More}, <>) -> - incremental_decode_loop(More(C), Rest). - - -endif. From 231a55c7e19187827889777154bb5c439db58c2e Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Mon, 4 Feb 2013 20:55:40 -0800 Subject: [PATCH 02/61] add unit tests for jsx_to_term, remove integration tests --- src/jsx_to_term.erl | 222 +++++++++++++++++++++++++------------------- 1 file changed, 126 insertions(+), 96 deletions(-) diff --git a/src/jsx_to_term.erl b/src/jsx_to_term.erl index 754391c..c5ed81c 100644 --- a/src/jsx_to_term.erl +++ b/src/jsx_to_term.erl @@ -91,8 +91,8 @@ handle_event(end_object, {[Object, Last|Terms], Opts}) -> handle_event(start_array, {Terms, Opts}) -> {[[]|Terms], Opts}; handle_event(end_array, {[List, {key, Key}, Last|Terms], Opts}) -> {[[{Key, post_decode(lists:reverse(List), Opts)}] ++ Last] ++ Terms, Opts}; -handle_event(end_array, {[Current, Last|Terms], Opts}) -> - {[[post_decode(lists:reverse(Current), Opts)] ++ Last] ++ Terms, Opts}; +handle_event(end_array, {[List, Last|Terms], Opts}) -> + {[[post_decode(lists:reverse(List), Opts)] ++ Last] ++ Terms, Opts}; handle_event({key, Key}, {Terms, Opts}) -> {[{key, format_key(Key, Opts)}] ++ Terms, Opts}; @@ -119,117 +119,147 @@ post_decode(Value, Opts) -> (Opts#opts.post_decode)(Value). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). -basic_test_() -> + +opts_test_() -> + %% for post_decode tests + F = fun(X) -> X end, + G = fun(X, Y) -> {X, Y} end, [ - {"empty object", ?_assertEqual(to_term(<<"{}">>, []), [{}])}, - {"simple object", ?_assertEqual(to_term(<<"{\"key\": true}">>, []), [{<<"key">>, true}])}, - {"less simple object", ?_assertEqual( - to_term(<<"{\"a\": 1, \"b\": 2}">>, []), - [{<<"a">>, 1}, {<<"b">>, 2}] + {"empty opts", ?_assertEqual(#opts{}, parse_opts([]))}, + {"implicit binary labels", ?_assertEqual(#opts{}, parse_opts([labels]))}, + {"binary labels", ?_assertEqual(#opts{}, parse_opts([{labels, binary}]))}, + {"atom labels", ?_assertEqual(#opts{labels=atom}, parse_opts([{labels, atom}]))}, + {"existing atom labels", ?_assertEqual( + #opts{labels=existing_atom}, + parse_opts([{labels, existing_atom}]) )}, - {"nested object", ?_assertEqual( - to_term(<<"{\"key\": {\"key\": true}}">>, []), - [{<<"key">>, [{<<"key">>, true}]}] + {"post decode", ?_assertEqual( + #opts{post_decode=F}, + parse_opts([{post_decode, F}]) )}, - {"empty array", ?_assert(to_term(<<"[]">>, []) =:= [])}, - {"list of lists", ?_assertEqual(to_term(<<"[[],[],[]]">>, []), [[], [], []])}, - {"list of strings", ?_assertEqual(to_term(<<"[\"hi\", \"there\"]">>, []), [<<"hi">>, <<"there">>])}, - {"list of numbers", ?_assertEqual(to_term(<<"[1, 2.0, 3e4, -5]">>, []), [1, 2.0, 3.0e4, -5])}, - {"list of literals", ?_assertEqual(to_term(<<"[true,false,null]">>, []), [true,false,null])}, - {"list of objects", ?_assertEqual( - to_term(<<"[{}, {\"a\":1, \"b\":2}, {\"key\":[true,false]}]">>, []), - [[{}], [{<<"a">>,1},{<<"b">>,2}], [{<<"key">>,[true,false]}]] + {"post decode wrong arity", ?_assertError(badarg, parse_opts([{post_decode, G}]))} + ]. + + +format_key_test_() -> + [ + {"binary key", ?_assertEqual(<<"key">>, format_key(<<"key">>, #opts{labels=binary}))}, + {"atom key", ?_assertEqual(key, format_key(<<"key">>, #opts{labels=atom}))}, + {"existing atom key", ?_assertEqual( + key, + format_key(<<"key">>, #opts{labels=existing_atom}) + )}, + {"nonexisting atom key", ?_assertError( + badarg, + format_key(<<"nonexistentatom">>, #opts{labels=existing_atom}) )} ]. -comprehensive_test_() -> - {"comprehensive test", ?_assertEqual(to_term(comp_json(), []), comp_term())}. - -comp_json() -> - <<"[ - {\"a key\": {\"a key\": -17.346, \"another key\": 3e152, \"last key\": 14}}, - [0,1,2,3,4,5], - [{\"a\": \"a\", \"b\": \"b\"}, {\"c\": \"c\", \"d\": \"d\"}], - [true, false, null], - {}, - [], - [{},{}], - {\"key\": [], \"another key\": {}} - ]">>. - -comp_term() -> - [ - [{<<"a key">>, [{<<"a key">>, -17.346}, {<<"another key">>, 3.0e152}, {<<"last key">>, 14}]}], - [0,1,2,3,4,5], - [[{<<"a">>, <<"a">>}, {<<"b">>, <<"b">>}], [{<<"c">>, <<"c">>}, {<<"d">>, <<"d">>}]], - [true, false, null], - [{}], - [], - [[{}], [{}]], - [{<<"key">>, []}, {<<"another key">>, [{}]}] - ]. - -atom_labels_test_() -> - {"atom labels test", ?_assertEqual(to_term(comp_json(), [{labels, atom}]), atom_term())}. - -atom_term() -> - [ - [{'a key', [{'a key', -17.346}, {'another key', 3.0e152}, {'last key', 14}]}], - [0,1,2,3,4,5], - [[{a, <<"a">>}, {b, <<"b">>}], [{'c', <<"c">>}, {'d', <<"d">>}]], - [true, false, null], - [{}], - [], - [[{}], [{}]], - [{key, []}, {'another key', [{}]}] - ]. - -naked_test_() -> - [ - {"naked integer", ?_assertEqual(to_term(<<"123">>, []), 123)}, - {"naked float", ?_assertEqual(to_term(<<"-4.32e-17">>, []), -4.32e-17)}, - {"naked literal", ?_assertEqual(to_term(<<"true">>, []), true)}, - {"naked string", ?_assertEqual(to_term(<<"\"string\"">>, []), <<"string">>)} - ]. post_decoders_test_() -> - JSON = <<"{\"object\": { - \"literals\": [true, false, null], - \"strings\": [\"foo\", \"bar\", \"baz\"], - \"numbers\": [1, 1.0, 1e0] - }}">>, + Events = [ + [{}], + [{<<"key">>, <<"value">>}], + [{<<"true">>, true}, {<<"false">>, false}, {<<"null">>, null}], + [], + [<<"string">>], + [true, false, null], + true, + false, + null, + <<"hello">>, + <<"world">>, + 1, + 1.0 + ], [ {"no post_decode", ?_assertEqual( - to_term(JSON, []), - [{<<"object">>, [ - {<<"literals">>, [true, false, null]}, - {<<"strings">>, [<<"foo">>, <<"bar">>, <<"baz">>]}, - {<<"numbers">>, [1, 1.0, 1.0]} - ]}] + Events, + [ post_decode(Event, #opts{}) || Event <- Events ] )}, {"replace arrays with empty arrays", ?_assertEqual( - to_term(JSON, [{post_decode, fun([T|_] = V) when is_tuple(T) -> V; (V) when is_list(V) -> []; (V) -> V end}]), - [{<<"object">>, [{<<"literals">>, []}, {<<"strings">>, []}, {<<"numbers">>, []}]}] + [ + [{}], + [{<<"key">>, <<"value">>}], + [{<<"true">>, true}, {<<"false">>, false}, {<<"null">>, null}], + [], + [], + [], + true, + false, + null, + <<"hello">>, + <<"world">>, + 1, + 1.0 + ], + [ post_decode(Event, #opts{ + post_decode=fun([T|_] = V) when is_tuple(T) -> V; (V) when is_list(V) -> []; (V) -> V end + }) || Event <- Events + ] )}, {"replace objects with empty objects", ?_assertEqual( - to_term(JSON, [{post_decode, fun(V) when is_list(V) -> [{}]; (V) -> V end}]), - [{}] + [ + [{}], + [{}], + [{}], + [], + [<<"string">>], + [true, false, null], + true, + false, + null, + <<"hello">>, + <<"world">>, + 1, + 1.0 + ], + [ post_decode(Event, #opts{ + post_decode=fun([T|_]) when is_tuple(T) -> [{}]; (V) -> V end + }) || Event <- Events + ] )}, - {"replace all non-list values with false", ?_assertEqual( - to_term(JSON, [{post_decode, fun(V) when is_list(V) -> V; (_) -> false end}]), - [{<<"object">>, [ - {<<"literals">>, [false, false, false]}, - {<<"strings">>, [false, false, false]}, - {<<"numbers">>, [false, false, false]} - ]}] + {"replace all non-array/non-object values with false", ?_assertEqual( + [ + [{}], + [{<<"key">>, <<"value">>}], + [{<<"true">>, true}, {<<"false">>, false}, {<<"null">>, null}], + [], + [<<"string">>], + [true, false, null], + false, + false, + false, + false, + false, + false, + false + ], + [ post_decode(Event, #opts{ + post_decode=fun(V) when is_list(V) -> V; (_) -> false end + }) || Event <- Events + ] )}, {"atoms_to_strings", ?_assertEqual( - to_term(JSON, [{post_decode, fun(V) when is_atom(V) -> unicode:characters_to_binary(atom_to_list(V)); (V) -> V end}]), - [{<<"object">>, [ - {<<"literals">>, [<<"true">>, <<"false">>, <<"null">>]}, - {<<"strings">>, [<<"foo">>, <<"bar">>, <<"baz">>]}, - {<<"numbers">>, [1, 1.0, 1.0]} - ]}] + [ + [{}], + [{<<"key">>, <<"value">>}], + [{<<"true">>, true}, {<<"false">>, false}, {<<"null">>, null}], + [], + [<<"string">>], + [true, false, null], + <<"true">>, + <<"false">>, + <<"null">>, + <<"hello">>, + <<"world">>, + 1, + 1.0 + ], + [ post_decode(Event, #opts{ + post_decode=fun(V) when is_atom(V) -> unicode:characters_to_binary(atom_to_list(V)); (V) -> V end + }) || Event <- Events + ] )} ]. From 81f0321e32211037a19739e52e2c53eb575e1a19 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Mon, 4 Feb 2013 23:15:50 -0800 Subject: [PATCH 03/61] add tests for stricter parsing of opts --- src/jsx_to_term.erl | 4 +++- src/jsx_verify.erl | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/jsx_to_term.erl b/src/jsx_to_term.erl index c5ed81c..d8fb0bd 100644 --- a/src/jsx_to_term.erl +++ b/src/jsx_to_term.erl @@ -137,7 +137,9 @@ opts_test_() -> #opts{post_decode=F}, parse_opts([{post_decode, F}]) )}, - {"post decode wrong arity", ?_assertError(badarg, parse_opts([{post_decode, G}]))} + {"post decode wrong arity", ?_assertError(badarg, parse_opts([{post_decode, G}]))}, + {"invalid opt flag", ?_assertError(badarg, parse_opts([error]))}, + {"invalid opt tuple", ?_assertError(badarg, parse_opts([{error, true}]))} ]. diff --git a/src/jsx_verify.erl b/src/jsx_verify.erl index ff13950..93388b1 100644 --- a/src/jsx_verify.erl +++ b/src/jsx_verify.erl @@ -108,7 +108,9 @@ opts_test_() -> {"repeated keys false", ?_assertEqual( #opts{repeated_keys=false}, parse_opts([{repeated_keys, false}]) - )} + )}, + {"invalid opt flag", ?_assertError(badarg, parse_opts([error]))}, + {"invalid opt tuple", ?_assertError(badarg, parse_opts([{error, true}]))} ]. From 822a668f8e0c4c6df81d5773f6e0759b4fc7d357 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Mon, 4 Feb 2013 23:28:17 -0800 Subject: [PATCH 04/61] remove integration tests and add unit tests to jsx_to_json.erl --- src/jsx_to_json.erl | 138 +++++--------------------------------------- 1 file changed, 14 insertions(+), 124 deletions(-) diff --git a/src/jsx_to_json.erl b/src/jsx_to_json.erl index 7ddc13c..085836f 100644 --- a/src/jsx_to_json.erl +++ b/src/jsx_to_json.erl @@ -184,138 +184,28 @@ indent_or_space(Opts) -> -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). -basic_format_test_() -> - [ - {"empty object", ?_assertEqual(format(<<"{}">>, []), <<"{}">>)}, - {"empty array", ?_assertEqual(format(<<"[]">>, []), <<"[]">>)}, - {"naked integer", ?_assertEqual(format(<<"123">>, []), <<"123">>)}, - {"naked float", ?_assertEqual(format(<<"1.23">>, []), <<"1.23">>)}, - {"naked string", ?_assertEqual(format(<<"\"hi\"">>, []), <<"\"hi\"">>)}, - {"naked string with control character", ?_assertEqual( - format(<<"\"hi\\n\"">>, []), <<"\"hi\\n\"">> - )}, - {"naked literal", ?_assertEqual(format(<<"true">>, []), <<"true">>)}, - {"simple object", ?_assertEqual( - format(<<" { \"key\" :\n\t \"value\"\r\r\r\n } ">>, []), - <<"{\"key\":\"value\"}">> - )}, - {"really simple object", ?_assertEqual(format(<<"{\"k\":\"v\"}">>, []) , <<"{\"k\":\"v\"}">>)}, - {"nested object", ?_assertEqual( - format(<<"{\"k\":{\"k\":\"v\"}, \"j\":{}}">>, []), - <<"{\"k\":{\"k\":\"v\"},\"j\":{}}">> - )}, - {"simple array", ?_assertEqual( - format(<<" [\n\ttrue,\n\tfalse , \n \tnull\n] ">>, []), - <<"[true,false,null]">> - )}, - {"really simple array", ?_assertEqual(format(<<"[1]">>, []), <<"[1]">>)}, - {"nested array", ?_assertEqual(format(<<"[[[]]]">>, []), <<"[[[]]]">>)}, - {"nested structures", ?_assertEqual( - format(<<"[ - { - \"key\":\"value\", - \"another key\": \"another value\", - \"a list\": [true, false] - }, - [[{}]] - ]">>, []), - <<"[{\"key\":\"value\",\"another key\":\"another value\",\"a list\":[true,false]},[[{}]]]">> - )}, - {"simple nested structure", - ?_assertEqual( - format(<<"[[],{\"k\":[[],{}],\"j\":{}},[]]">>, []), - <<"[[],{\"k\":[[],{}],\"j\":{}},[]]">> - ) - } - ]. - -basic_to_json_test_() -> - [ - {"empty object", ?_assertEqual(to_json([{}], []), <<"{}">>)}, - {"empty array", ?_assertEqual(to_json([], []), <<"[]">>)}, - {"naked integer", ?_assertEqual(to_json(123, []), <<"123">>)}, - {"naked float", ?_assertEqual(to_json(1.23, []) , <<"1.23">>)}, - {"naked string", ?_assertEqual(to_json(<<"hi">>, []), <<"\"hi\"">>)}, - {"naked string with control character", ?_assertEqual( - to_json(<<"hi\n">>, []), <<"\"hi\\n\"">> - )}, - {"naked literal", ?_assertEqual(to_json(true, []), <<"true">>)}, - {"simple object", ?_assertEqual( - to_json( - [{<<"key">>, <<"value">>}], - [] - ), - <<"{\"key\":\"value\"}">> - )}, - {"nested object", ?_assertEqual( - to_json( - [{<<"k">>,[{<<"k">>,<<"v">>}]},{<<"j">>,[{}]}], - [] - ), - <<"{\"k\":{\"k\":\"v\"},\"j\":{}}">> - )}, - {"simple array", ?_assertEqual(to_json([true, false, null], []), <<"[true,false,null]">>)}, - {"really simple array", ?_assertEqual(to_json([1], []), <<"[1]">>)}, - {"nested array", ?_assertEqual(to_json([[[]]], []), <<"[[[]]]">>)}, - {"nested structures", ?_assertEqual( - to_json( - [ - [ - {<<"key">>, <<"value">>}, - {<<"another key">>, <<"another value">>}, - {<<"a list">>, [true, false]} - ], - [[[{}]]] - ], - [] - ), - <<"[{\"key\":\"value\",\"another key\":\"another value\",\"a list\":[true,false]},[[{}]]]">> - )}, - {"simple nested structure", ?_assertEqual( - to_json( - [[], [{<<"k">>, [[], [{}]]}, {<<"j">>, [{}]}], []], - [] - ), - <<"[[],{\"k\":[[],{}],\"j\":{}},[]]">> - )} - ]. opts_test_() -> [ + {"empty opts", ?_assertEqual(#opts{}, parse_opts([]))}, {"unspecified indent/space", ?_assertEqual( - format(<<" [\n\ttrue,\n\tfalse,\n\tnull\n] ">>, [space, indent]), - <<"[\n true,\n false,\n null\n]">> + #opts{space=1, indent=1}, + parse_opts([space, indent]) )}, - {"specific indent/space", ?_assertEqual( - format( - <<"\n{\n\"key\" : [],\n\"another key\" : true\n}\n">>, - [{space, 2}, {indent, 3}] - ), - <<"{\n \"key\": [],\n \"another key\": true\n}">> + {"specific indent", ?_assertEqual( + #opts{indent=4}, + parse_opts([{indent, 4}]) )}, - {"nested structures", ?_assertEqual( - format( - <<"[{\"key\":\"value\", \"another key\": \"another value\"}, [[true, false, null]]]">>, - [{space, 2}, {indent, 2}] - ), - <<"[\n {\n \"key\": \"value\",\n \"another key\": \"another value\"\n },\n [\n [\n true,\n false,\n null\n ]\n ]\n]">> + {"specific space", ?_assertEqual( + #opts{space=2}, + parse_opts([{space, 2}]) )}, - {"array spaces", ?_assertEqual( - format(<<"[1,2,3]">>, [{space, 2}]), - <<"[1, 2, 3]">> + {"specific space and indent", ?_assertEqual( + #opts{space=2, indent=2}, + parse_opts([{space, 2}, {indent, 2}]) )}, - {"object spaces", ?_assertEqual( - format(<<"{\"a\":true,\"b\":true,\"c\":true}">>, [{space, 2}]), - <<"{\"a\": true, \"b\": true, \"c\": true}">> - )}, - {"array indent", ?_assertEqual( - format(<<"[1.23, 1.23, 1.23]">>, [{indent, 2}]), - <<"[\n 1.23,\n 1.23,\n 1.23\n]">> - )}, - {"object indent", ?_assertEqual( - format(<<"{\"a\":true,\"b\":true,\"c\":true}">>, [{indent, 2}]), - <<"{\n \"a\":true,\n \"b\":true,\n \"c\":true\n}">> - )} + {"invalid opt flag", ?_assertError(badarg, parse_opts([error]))}, + {"invalid opt tuple", ?_assertError(badarg, parse_opts([{error, true}]))} ]. From d8cde35e4565128b564443f805d78957b2230af9 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Mon, 4 Feb 2013 23:51:02 -0800 Subject: [PATCH 05/61] add formatting tests to jsx_to_json --- src/jsx_to_json.erl | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/jsx_to_json.erl b/src/jsx_to_json.erl index 085836f..bea2088 100644 --- a/src/jsx_to_json.erl +++ b/src/jsx_to_json.erl @@ -209,4 +209,47 @@ opts_test_() -> ]. +space_test_() -> + [ + {"no space", ?_assertEqual([], space(#opts{space=0}))}, + {"one space", ?_assertEqual(<<" ">>, space(#opts{space=1}))}, + {"four spaces", ?_assertEqual(<<" ">>, space(#opts{space=4}))} + ]. + + +indent_test_() -> + [ + {"no indent", ?_assertEqual([], indent(#opts{indent=0, depth=1}))}, + {"indent 1 depth 1", ?_assertEqual( + [[?newline], ?space], + indent(#opts{indent=1, depth=1}) + )}, + {"indent 1 depth 2", ?_assertEqual( + [[[?newline], ?space], ?space], + indent(#opts{indent=1, depth=2}) + )}, + {"indent 4 depth 1", ?_assertEqual( + [[?newline], <<" ">>], + indent(#opts{indent=4, depth=1}) + )}, + {"indent 4 depth 2", ?_assertEqual( + [[[?newline], <<" ">>], <<" ">>], + indent(#opts{indent=4, depth=2}) + )} + ]. + + +indent_or_space_test_() -> + [ + {"no indent so space", ?_assertEqual( + <<" ">>, + indent_or_space(#opts{space=1, indent=0, depth=1}) + )}, + {"indent so no space", ?_assertEqual( + [[?newline], ?space], + indent_or_space(#opts{space=1, indent=1, depth=1}) + )} + ]. + + -endif. \ No newline at end of file From 052a92d3253722e10b6086da7e457d05dff56700 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 5 Feb 2013 12:24:56 -0800 Subject: [PATCH 06/61] bad utf8 tests for clean_string/2 --- src/jsx_utils.erl | 317 ++++++++++++++++++++-------------------------- 1 file changed, 140 insertions(+), 177 deletions(-) diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index 19be8f9..2be2c7a 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -589,204 +589,167 @@ opts_test_() -> ]. -xcode(Bin) -> xcode(Bin, #opts{}). - -xcode(Bin, [replaced_bad_utf8]) -> xcode(Bin, #opts{replaced_bad_utf8=true}); -xcode(Bin, Opts) -> - try clean_string(Bin, Opts) - catch error:badarg -> {error, badarg} - end. - - -is_bad({error, badarg}) -> true; -is_bad(_) -> false. - - bad_utf8_test_() -> [ {"orphan continuation byte u+0080", - ?_assert(is_bad(xcode(<<16#0080>>))) + ?_assertError(badarg, clean_string(<<16#0080>>, #opts{})) }, {"orphan continuation byte u+0080 replaced", - ?_assertEqual(xcode(<<16#0080>>, [replaced_bad_utf8]), <<16#fffd/utf8>>) + ?_assertEqual(<<16#fffd/utf8>>, clean_string(<<16#0080>>, #opts{replaced_bad_utf8=true})) }, {"orphan continuation byte u+00bf", - ?_assert(is_bad(xcode(<<16#00bf>>))) + ?_assertError(badarg, clean_string(<<16#00bf>>, #opts{})) }, {"orphan continuation byte u+00bf replaced", - ?_assertEqual(xcode(<<16#00bf>>, [replaced_bad_utf8]), <<16#fffd/utf8>>) + ?_assertEqual(<<16#fffd/utf8>>, clean_string(<<16#00bf>>, #opts{replaced_bad_utf8=true})) }, {"2 continuation bytes", - ?_assert(is_bad(xcode(<<(binary:copy(<<16#0080>>, 2))/binary>>))) - }, - {"2 continuation bytes replaced", - ?_assertEqual( - xcode(<<(binary:copy(<<16#0080>>, 2))/binary>>, [replaced_bad_utf8]), - binary:copy(<<16#fffd/utf8>>, 2) - ) + ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #opts{})) }, + {"2 continuation bytes replaced", ?_assertEqual( + binary:copy(<<16#fffd/utf8>>, 2), + clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #opts{replaced_bad_utf8=true}) + )}, {"3 continuation bytes", - ?_assert(is_bad(xcode(<<(binary:copy(<<16#0080>>, 3))/binary>>))) - }, - {"3 continuation bytes replaced", - ?_assertEqual( - xcode(<<(binary:copy(<<16#0080>>, 3))/binary>>, [replaced_bad_utf8]), - binary:copy(<<16#fffd/utf8>>, 3) - ) + ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #opts{})) }, + {"3 continuation bytes replaced", ?_assertEqual( + binary:copy(<<16#fffd/utf8>>, 3), + clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #opts{replaced_bad_utf8=true}) + )}, {"4 continuation bytes", - ?_assert(is_bad(xcode(<<(binary:copy(<<16#0080>>, 4))/binary>>))) - }, - {"4 continuation bytes replaced", - ?_assertEqual( - xcode(<<(binary:copy(<<16#0080>>, 4))/binary>>, [replaced_bad_utf8]), - binary:copy(<<16#fffd/utf8>>, 4) - ) + ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #opts{})) }, + {"4 continuation bytes replaced", ?_assertEqual( + binary:copy(<<16#fffd/utf8>>, 4), + clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #opts{replaced_bad_utf8=true}) + )}, {"5 continuation bytes", - ?_assert(is_bad(xcode(<<(binary:copy(<<16#0080>>, 5))/binary>>))) - }, - {"5 continuation bytes replaced", - ?_assertEqual( - xcode(<<(binary:copy(<<16#0080>>, 5))/binary>>, [replaced_bad_utf8]), - binary:copy(<<16#fffd/utf8>>, 5) - ) + ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #opts{})) }, + {"5 continuation bytes replaced", ?_assertEqual( + binary:copy(<<16#fffd/utf8>>, 5), + clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #opts{replaced_bad_utf8=true}) + )}, {"6 continuation bytes", - ?_assert(is_bad(xcode(<<(binary:copy(<<16#0080>>, 6))/binary>>))) + ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #opts{})) }, - {"6 continuation bytes replaced", - ?_assertEqual( - xcode(<<(binary:copy(<<16#0080>>, 6))/binary>>, [replaced_bad_utf8]), - binary:copy(<<16#fffd/utf8>>, 6) + {"6 continuation bytes replaced", ?_assertEqual( + binary:copy(<<16#fffd/utf8>>, 6), + clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #opts{replaced_bad_utf8=true}) + )}, + {"all continuation bytes", ?_assertError( + badarg, + clean_string(<<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, #opts{}) + )}, + {"all continuation bytes replaced", ?_assertEqual( + binary:copy(<<16#fffd/utf8>>, length(lists:seq(16#0080, 16#00bf))), + clean_string( + <<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, + #opts{replaced_bad_utf8=true} ) - }, - {"all continuation bytes", - ?_assert(is_bad(xcode(<<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>))) - }, - {"all continuation bytes replaced", - ?_assertEqual( - xcode(<<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, [replaced_bad_utf8]), - binary:copy(<<16#fffd/utf8>>, length(lists:seq(16#0080, 16#00bf))) - ) - }, - {"lonely start byte", - ?_assert(is_bad(xcode(<<16#00c0>>))) - }, - {"lonely start byte replaced", - ?_assertEqual( - xcode(<<16#00c0>>, [replaced_bad_utf8]), - <<16#fffd/utf8>> - ) - }, - {"lonely start bytes (2 byte)", - ?_assert(is_bad(xcode(<<16#00c0, 32, 16#00df>>))) - }, - {"lonely start bytes (2 byte) replaced", - ?_assertEqual( - xcode(<<16#00c0, 32, 16#00df>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32, 16#fffd/utf8>> - ) - }, - {"lonely start bytes (3 byte)", - ?_assert(is_bad(xcode(<<16#00e0, 32, 16#00ef>>))) - }, - {"lonely start bytes (3 byte) replaced", - ?_assertEqual( - xcode(<<16#00e0, 32, 16#00ef>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32, 16#fffd/utf8>> - ) - }, - {"lonely start bytes (4 byte)", - ?_assert(is_bad(xcode(<<16#00f0, 32, 16#00f7>>))) - }, - {"lonely start bytes (4 byte) replaced", - ?_assertEqual( - xcode(<<16#00f0, 32, 16#00f7>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32, 16#fffd/utf8>> - ) - }, - {"missing continuation byte (3 byte)", - ?_assert(is_bad(xcode(<<224, 160, 32>>))) - }, - {"missing continuation byte (3 byte) replaced", - ?_assertEqual( - xcode(<<224, 160, 32>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32>> - ) - }, - {"missing continuation byte (4 byte missing one)", - ?_assert(is_bad(xcode(<<240, 144, 128, 32>>))) - }, - {"missing continuation byte (4 byte missing one) replaced", - ?_assertEqual( - xcode(<<240, 144, 128, 32>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32>> - ) - }, - {"missing continuation byte (4 byte missing two)", - ?_assert(is_bad(xcode(<<240, 144, 32>>))) - }, - {"missing continuation byte (4 byte missing two) replaced", - ?_assertEqual( - xcode(<<240, 144, 32>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32>> - ) - }, - {"overlong encoding of u+002f (2 byte)", - ?_assert(is_bad(xcode(<<16#c0, 16#af, 32>>))) - }, - {"overlong encoding of u+002f (2 byte) replaced", - ?_assertEqual( - xcode(<<16#c0, 16#af, 32>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32>> - ) - }, - {"overlong encoding of u+002f (3 byte)", - ?_assert(is_bad(xcode(<<16#e0, 16#80, 16#af, 32>>))) - }, - {"overlong encoding of u+002f (3 byte) replaced", - ?_assertEqual( - xcode(<<16#e0, 16#80, 16#af, 32>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32>> - ) - }, - {"overlong encoding of u+002f (4 byte)", - ?_assert(is_bad(xcode(<<16#f0, 16#80, 16#80, 16#af, 32>>))) - }, - {"overlong encoding of u+002f (4 byte) replaced", - ?_assertEqual( - xcode(<<16#f0, 16#80, 16#80, 16#af, 32>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32>> - ) - }, - {"highest overlong 2 byte sequence", - ?_assert(is_bad(xcode(<<16#c1, 16#bf, 32>>))) - }, - {"highest overlong 2 byte sequence replaced", - ?_assertEqual( - xcode(<<16#c1, 16#bf, 32>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32>> - ) - }, - {"highest overlong 3 byte sequence", - ?_assert(is_bad(xcode(<<16#e0, 16#9f, 16#bf, 32>>))) - }, - {"highest overlong 3 byte sequence replaced", - ?_assertEqual( - xcode(<<16#e0, 16#9f, 16#bf, 32>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32>> - ) - }, - {"highest overlong 4 byte sequence", - ?_assert(is_bad(xcode(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>))) - }, - {"highest overlong 4 byte sequence replaced", - ?_assertEqual( - xcode(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32>> - ) - } + )}, + {"lonely start byte", ?_assertError(badarg, clean_string(<<16#00c0>>, #opts{}))}, + {"lonely start byte replaced", ?_assertEqual( + <<16#fffd/utf8>>, + clean_string(<<16#00c0>>, #opts{replaced_bad_utf8=true}) + )}, + {"lonely start bytes (2 byte)", ?_assertError( + badarg, + clean_string(<<16#00c0, 32, 16#00df>>, #opts{}) + )}, + {"lonely start bytes (2 byte) replaced", ?_assertEqual( + <<16#fffd/utf8, 32, 16#fffd/utf8>>, + clean_string(<<16#00c0, 32, 16#00df>>, #opts{replaced_bad_utf8=true}) + )}, + {"lonely start bytes (3 byte)", ?_assertError( + badarg, + clean_string(<<16#00e0, 32, 16#00ef>>, #opts{}) + )}, + {"lonely start bytes (3 byte) replaced", ?_assertEqual( + <<16#fffd/utf8, 32, 16#fffd/utf8>>, + clean_string(<<16#00e0, 32, 16#00ef>>, #opts{replaced_bad_utf8=true}) + )}, + {"lonely start bytes (4 byte)", ?_assertError( + badarg, + clean_string(<<16#00f0, 32, 16#00f7>>, #opts{}) + )}, + {"lonely start bytes (4 byte) replaced", ?_assertEqual( + <<16#fffd/utf8, 32, 16#fffd/utf8>>, + clean_string(<<16#00f0, 32, 16#00f7>>, #opts{replaced_bad_utf8=true}) + )}, + {"missing continuation byte (3 byte)", ?_assertError( + badarg, + clean_string(<<224, 160, 32>>, #opts{}) + )}, + {"missing continuation byte (3 byte) replaced", ?_assertEqual( + <<16#fffd/utf8, 32>>, + clean_string(<<224, 160, 32>>, #opts{replaced_bad_utf8=true}) + )}, + {"missing continuation byte (4 byte missing one)", ?_assertError( + badarg, + clean_string(<<240, 144, 128, 32>>, #opts{}) + )}, + {"missing continuation byte (4 byte missing one) replaced", ?_assertEqual( + <<16#fffd/utf8, 32>>, + clean_string(<<240, 144, 128, 32>>, #opts{replaced_bad_utf8=true}) + )}, + {"missing continuation byte (4 byte missing two)", ?_assertError( + badarg, + clean_string(<<240, 144, 32>>, #opts{}) + )}, + {"missing continuation byte (4 byte missing two) replaced", ?_assertEqual( + <<16#fffd/utf8, 32>>, + clean_string(<<240, 144, 32>>, #opts{replaced_bad_utf8=true}) + )}, + {"overlong encoding of u+002f (2 byte)", ?_assertError( + badarg, + clean_string(<<16#c0, 16#af, 32>>, #opts{}) + )}, + {"overlong encoding of u+002f (2 byte) replaced", ?_assertEqual( + <<16#fffd/utf8, 32>>, + clean_string(<<16#c0, 16#af, 32>>, #opts{replaced_bad_utf8=true}) + )}, + {"overlong encoding of u+002f (3 byte)", ?_assertError( + badarg, + clean_string(<<16#e0, 16#80, 16#af, 32>>, #opts{}) + )}, + {"overlong encoding of u+002f (3 byte) replaced", ?_assertEqual( + <<16#fffd/utf8, 32>>, + clean_string(<<16#e0, 16#80, 16#af, 32>>, #opts{replaced_bad_utf8=true}) + )}, + {"overlong encoding of u+002f (4 byte)", ?_assertError( + badarg, + clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #opts{}) + )}, + {"overlong encoding of u+002f (4 byte) replaced", ?_assertEqual( + <<16#fffd/utf8, 32>>, + clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #opts{replaced_bad_utf8=true}) + )}, + {"highest overlong 2 byte sequence", ?_assertError( + badarg, + clean_string(<<16#c1, 16#bf, 32>>, #opts{}) + )}, + {"highest overlong 2 byte sequence replaced", ?_assertEqual( + <<16#fffd/utf8, 32>>, + clean_string(<<16#c1, 16#bf, 32>>, #opts{replaced_bad_utf8=true}) + )}, + {"highest overlong 3 byte sequence", ?_assertError( + badarg, + clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #opts{}) + )}, + {"highest overlong 3 byte sequence replaced", ?_assertEqual( + <<16#fffd/utf8, 32>>, + clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #opts{replaced_bad_utf8=true}) + )}, + {"highest overlong 4 byte sequence", ?_assertError( + badarg, + clean_string(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, #opts{}) + )}, + {"highest overlong 4 byte sequence replaced", ?_assertEqual( + <<16#fffd/utf8, 32>>, + clean_string(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, #opts{replaced_bad_utf8=true}) + )} ]. From c99cbd9d12451c6849434e04446a87475a2d6d8a Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 5 Feb 2013 16:40:15 -0800 Subject: [PATCH 07/61] move all tests relating to cleaning of binary strings to jsx_utils --- src/jsx_encoder.erl | 136 ----------------- src/jsx_parser.erl | 135 ----------------- src/jsx_utils.erl | 347 ++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 321 insertions(+), 297 deletions(-) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 86a894e..a06c270 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -351,8 +351,6 @@ escapes_test_() -> {"carriage return escape", ?_assertEqual(encode(<<"\r">>, [escaped_strings]), [{string, <<"\\r">>}, end_json])}, {"tab escape", ?_assertEqual(encode(<<"\t">>, [escaped_strings]), [{string, <<"\\t">>}, end_json])}, {"quote escape", ?_assertEqual(encode(<<"\"">>, [escaped_strings]), [{string, <<"\\\"">>}, end_json])}, - {"single quote escape", ?_assertEqual(encode(<<"'">>, [escaped_strings, single_quoted_strings]), [{string, <<"\\'">>}, end_json])}, - {"no single quote escape", ?_assertEqual(encode(<<"'">>, [escaped_strings]), [{string, <<"'">>}, end_json])}, {"forward slash escape", ?_assertEqual(encode(<<"/">>, [escaped_strings, escaped_forward_slashes]), [{string, <<"\\/">>}, end_json])}, {"no forward slash escape", ?_assertEqual(encode(<<"/">>, [escaped_strings]), [{string, <<"/">>}, end_json])}, {"back slash escape", ?_assertEqual(encode(<<"\\">>, [escaped_strings]), [{string, <<"\\\\">>}, end_json])}, @@ -370,138 +368,4 @@ escapes_test_() -> ]. -surrogates_test_() -> - [ - {"surrogates - badarg", - ?_assert(check_bad(surrogates())) - }, - {"surrogates - replaced", - ?_assert(check_replaced(surrogates())) - } - ]. - - -good_characters_test_() -> - [ - {"acceptable codepoints", - ?_assert(check_good(good())) - }, - {"acceptable codepoints - escaped_strings", - ?_assert(check_good(good(), [escaped_strings])) - }, - {"acceptable codepoints - replaced_bad_utf8", - ?_assert(check_good(good(), [escaped_strings])) - }, - {"acceptable codepoints - escaped_strings + replaced_bad_utf8", - ?_assert(check_good(good(), [escaped_strings, replaced_bad_utf8])) - }, - {"acceptable extended", - ?_assert(check_good(good_extended())) - }, - {"acceptable extended - escaped_strings", - ?_assert(check_good(good_extended(), [escaped_strings])) - }, - {"acceptable extended - escaped_strings", - ?_assert(check_good(good_extended(), [replaced_bad_utf8])) - } - ]. - - -reserved_test_() -> - [ - {"reserved noncharacters - badarg", - ?_assert(check_bad(reserved_space())) - }, - {"reserved noncharacters - replaced", - ?_assert(check_replaced(reserved_space())) - } - ]. - - -noncharacters_test_() -> - [ - {"noncharacters - badarg", - ?_assert(check_bad(noncharacters())) - }, - {"noncharacters - replaced", - ?_assert(check_replaced(noncharacters())) - } - ]. - - -extended_noncharacters_test_() -> - [ - {"extended noncharacters - badarg", - ?_assert(check_bad(extended_noncharacters())) - }, - {"extended noncharacters - replaced", - ?_assert(check_replaced(extended_noncharacters())) - } - ]. - - -check_bad(List) -> - [] == lists:dropwhile(fun({_, {error, badarg}}) -> true ; (_) -> false end, - check(List, [], []) - ). - - -check_replaced(List) -> - [] == lists:dropwhile(fun({_, [{string, <<16#fffd/utf8>>}|_]}) -> true ; (_) -> false - end, - check(List, [replaced_bad_utf8], []) - ). - - -check_good(List) -> check_good(List, []). - -check_good(List, Opts) -> - [] == lists:dropwhile(fun({_, [{string, _}|_]}) -> true ; (_) -> false end, - check(List, Opts, []) - ). - - -check([], _Opts, Acc) -> Acc; -check([H|T], Opts, Acc) -> - R = encode(to_fake_utf(H, utf8), Opts), - check(T, Opts, [{H, R}] ++ Acc). - - -noncharacters() -> lists:seq(16#fffe, 16#ffff). - -extended_noncharacters() -> - [16#1fffe, 16#1ffff, 16#2fffe, 16#2ffff] - ++ [16#3fffe, 16#3ffff, 16#4fffe, 16#4ffff] - ++ [16#5fffe, 16#5ffff, 16#6fffe, 16#6ffff] - ++ [16#7fffe, 16#7ffff, 16#8fffe, 16#8ffff] - ++ [16#9fffe, 16#9ffff, 16#afffe, 16#affff] - ++ [16#bfffe, 16#bffff, 16#cfffe, 16#cffff] - ++ [16#dfffe, 16#dffff, 16#efffe, 16#effff] - ++ [16#ffffe, 16#fffff, 16#10fffe, 16#10ffff]. - -surrogates() -> lists:seq(16#d800, 16#dfff). - -reserved_space() -> lists:seq(16#fdd0, 16#fdef). - -good() -> lists:seq(16#0000, 16#d7ff) ++ lists:seq(16#e000, 16#fdcf) ++ lists:seq(16#fdf0, 16#fffd). - -good_extended() -> [16#10000, 16#20000, 16#30000, 16#40000, 16#50000, - 16#60000, 16#70000, 16#80000, 16#90000, 16#a0000, - 16#b0000, 16#c0000, 16#d0000, 16#e0000, 16#f0000 - ] ++ lists:seq(16#100000, 16#10fffd). - - -%% erlang refuses to encode certain codepoints, so fake them all -to_fake_utf(N, utf8) when N < 16#0080 -> <>; -to_fake_utf(N, utf8) when N < 16#0800 -> - <<0:5, Y:5, X:6>> = <>, - <<2#110:3, Y:5, 2#10:2, X:6>>; -to_fake_utf(N, utf8) when N < 16#10000 -> - <> = <>, - <<2#1110:4, Z:4, 2#10:2, Y:6, 2#10:2, X:6>>; -to_fake_utf(N, utf8) -> - <<0:3, W:3, Z:6, Y:6, X:6>> = <>, - <<2#11110:5, W:3, 2#10:2, Z:6, 2#10:2, Y:6, 2#10:2, X:6>>. - - -endif. \ No newline at end of file diff --git a/src/jsx_parser.erl b/src/jsx_parser.erl index dc496ac..d622b5d 100644 --- a/src/jsx_parser.erl +++ b/src/jsx_parser.erl @@ -352,8 +352,6 @@ escapes_test_() -> {"carriage return escape", ?_assertEqual(encode(<<"\r">>, [escaped_strings]), [{string, <<"\\r">>}, end_json])}, {"tab escape", ?_assertEqual(encode(<<"\t">>, [escaped_strings]), [{string, <<"\\t">>}, end_json])}, {"quote escape", ?_assertEqual(encode(<<"\"">>, [escaped_strings]), [{string, <<"\\\"">>}, end_json])}, - {"single quote escape", ?_assertEqual(encode(<<"'">>, [escaped_strings, single_quoted_strings]), [{string, <<"\\'">>}, end_json])}, - {"no single quote escape", ?_assertEqual(encode(<<"'">>, [escaped_strings]), [{string, <<"'">>}, end_json])}, {"forward slash escape", ?_assertEqual(encode(<<"/">>, [escaped_strings, escaped_forward_slashes]), [{string, <<"\\/">>}, end_json])}, {"no forward slash escape", ?_assertEqual(encode(<<"/">>, [escaped_strings]), [{string, <<"/">>}, end_json])}, {"back slash escape", ?_assertEqual(encode(<<"\\">>, [escaped_strings]), [{string, <<"\\\\">>}, end_json])}, @@ -371,137 +369,4 @@ escapes_test_() -> ]. -surrogates_test_() -> - [ - {"surrogates - badarg", - ?_assert(check_bad(surrogates())) - }, - {"surrogates - replaced", - ?_assert(check_replaced(surrogates())) - } - ]. - - -good_characters_test_() -> - [ - {"acceptable codepoints", - ?_assert(check_good(good())) - }, - {"acceptable codepoints - escaped_strings", - ?_assert(check_good(good(), [escaped_strings])) - }, - {"acceptable codepoints - replaced_bad_utf8", - ?_assert(check_good(good(), [escaped_strings])) - }, - {"acceptable codepoints - escaped_strings + replaced_bad_utf8", - ?_assert(check_good(good(), [escaped_strings, replaced_bad_utf8])) - }, - {"acceptable extended", - ?_assert(check_good(good_extended())) - }, - {"acceptable extended - escaped_strings", - ?_assert(check_good(good_extended(), [escaped_strings])) - }, - {"acceptable extended - escaped_strings", - ?_assert(check_good(good_extended(), [replaced_bad_utf8])) - } - ]. - - -reserved_test_() -> - [ - {"reserved noncharacters - badarg", - ?_assert(check_bad(reserved_space())) - }, - {"reserved noncharacters - replaced", - ?_assert(check_replaced(reserved_space())) - } - ]. - - -noncharacters_test_() -> - [ - {"noncharacters - badarg", - ?_assert(check_bad(noncharacters())) - }, - {"noncharacters - replaced", - ?_assert(check_replaced(noncharacters())) - } - ]. - - -extended_noncharacters_test_() -> - [ - {"extended noncharacters - badarg", - ?_assert(check_bad(extended_noncharacters())) - }, - {"extended noncharacters - replaced", - ?_assert(check_replaced(extended_noncharacters())) - } - ]. - - -check_bad(List) -> - [] == lists:dropwhile(fun({_, {error, badarg}}) -> true ; (_) -> false end, - check(List, [], []) - ). - - -check_replaced(List) -> - [] == lists:dropwhile(fun({_, [{string, <<16#fffd/utf8>>}|_]}) -> true ; (_) -> false - end, - check(List, [replaced_bad_utf8], []) - ). - - -check_good(List) -> check_good(List, []). - -check_good(List, Opts) -> - [] == lists:dropwhile(fun({_, [{string, _}|_]}) -> true ; (_) -> false end, - check(List, Opts, []) - ). - - -check([], _Opts, Acc) -> Acc; -check([H|T], Opts, Acc) -> - R = encode(to_fake_utf(H, utf8), Opts), - check(T, Opts, [{H, R}] ++ Acc). - - -noncharacters() -> lists:seq(16#fffe, 16#ffff). - -extended_noncharacters() -> - [16#1fffe, 16#1ffff, 16#2fffe, 16#2ffff] - ++ [16#3fffe, 16#3ffff, 16#4fffe, 16#4ffff] - ++ [16#5fffe, 16#5ffff, 16#6fffe, 16#6ffff] - ++ [16#7fffe, 16#7ffff, 16#8fffe, 16#8ffff] - ++ [16#9fffe, 16#9ffff, 16#afffe, 16#affff] - ++ [16#bfffe, 16#bffff, 16#cfffe, 16#cffff] - ++ [16#dfffe, 16#dffff, 16#efffe, 16#effff] - ++ [16#ffffe, 16#fffff, 16#10fffe, 16#10ffff]. - -surrogates() -> lists:seq(16#d800, 16#dfff). - -reserved_space() -> lists:seq(16#fdd0, 16#fdef). - -good() -> lists:seq(16#0000, 16#d7ff) ++ lists:seq(16#e000, 16#fdcf) ++ lists:seq(16#fdf0, 16#fffd). - -good_extended() -> [16#10000, 16#20000, 16#30000, 16#40000, 16#50000, - 16#60000, 16#70000, 16#80000, 16#90000, 16#a0000, - 16#b0000, 16#c0000, 16#d0000, 16#e0000, 16#f0000 - ] ++ lists:seq(16#100000, 16#10fffd). - - -%% erlang refuses to encode certain codepoints, so fake them all -to_fake_utf(N, utf8) when N < 16#0080 -> <>; -to_fake_utf(N, utf8) when N < 16#0800 -> - <<0:5, Y:5, X:6>> = <>, - <<2#110:3, Y:5, 2#10:2, X:6>>; -to_fake_utf(N, utf8) when N < 16#10000 -> - <> = <>, - <<2#1110:4, Z:4, 2#10:2, Y:6, 2#10:2, X:6>>; -to_fake_utf(N, utf8) -> - <<0:3, W:3, Z:6, Y:6, X:6>> = <>, - <<2#11110:5, W:3, 2#10:2, Z:6, 2#10:2, Y:6, 2#10:2, X:6>>. - -endif. \ No newline at end of file diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index 2be2c7a..a6c4bf4 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -282,8 +282,7 @@ ensure_clean(<<124, Rest/binary>>) -> ensure_clean(Rest); ensure_clean(<<125, Rest/binary>>) -> ensure_clean(Rest); ensure_clean(<<126, Rest/binary>>) -> ensure_clean(Rest); ensure_clean(<<127, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<>) when X < 16#800 -> ensure_clean(Rest); -ensure_clean(<>) when X < 16#dcff -> ensure_clean(Rest); +ensure_clean(<>) when X < 16#d800 -> ensure_clean(Rest); ensure_clean(<>) when X > 16#dfff, X < 16#fdd0 -> ensure_clean(Rest); ensure_clean(<>) when X > 16#fdef, X < 16#fffe -> ensure_clean(Rest); ensure_clean(<>) when X >= 16#10000, X < 16#1fffe -> ensure_clean(Rest); @@ -346,7 +345,7 @@ clean(<<35, Rest/binary>>, Acc, Opts) -> clean(Rest, [35] ++ Acc, Opts); clean(<<36, Rest/binary>>, Acc, Opts) -> clean(Rest, [36] ++ Acc, Opts); clean(<<37, Rest/binary>>, Acc, Opts) -> clean(Rest, [37] ++ Acc, Opts); clean(<<38, Rest/binary>>, Acc, Opts) -> clean(Rest, [38] ++ Acc, Opts); -clean(<<39, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(39, Opts) ++ Acc, Opts); +clean(<<39, Rest/binary>>, Acc, Opts) -> clean(Rest, [39] ++ Acc, Opts); clean(<<40, Rest/binary>>, Acc, Opts) -> clean(Rest, [40] ++ Acc, Opts); clean(<<41, Rest/binary>>, Acc, Opts) -> clean(Rest, [41] ++ Acc, Opts); clean(<<42, Rest/binary>>, Acc, Opts) -> clean(Rest, [42] ++ Acc, Opts); @@ -439,7 +438,7 @@ clean(<>, Acc, Opts) when X < 16#800 -> clean(Rest, [X] ++ Acc, Opts); clean(<>, Acc, Opts) when X == 16#2028; X == 16#2029 -> clean(Rest, maybe_replace(X, Opts) ++ Acc, Opts); -clean(<>, Acc, Opts) when X < 16#dcff -> +clean(<>, Acc, Opts) when X < 16#d800 -> clean(Rest, [X] ++ Acc, Opts); clean(<>, Acc, Opts) when X > 16#dfff, X < 16#fdd0 -> clean(Rest, [X] ++ Acc, Opts); @@ -512,17 +511,12 @@ maybe_replace($\n, #opts{escaped_strings=true}) -> [$n, $\\]; maybe_replace($\f, #opts{escaped_strings=true}) -> [$f, $\\]; maybe_replace($\r, #opts{escaped_strings=true}) -> [$r, $\\]; maybe_replace($\", #opts{escaped_strings=true}) -> [$\", $\\]; -maybe_replace($', Opts=#opts{escaped_strings=true}) -> - case Opts#opts.single_quoted_strings of - true -> [$', $\\] - ; false -> [$'] - end; -maybe_replace($/, Opts=#opts{escaped_strings=true}) -> +maybe_replace($\\, #opts{escaped_strings=true}) -> [$\\, $\\]; +maybe_replace($/, Opts) -> case Opts#opts.escaped_forward_slashes of true -> [$/, $\\] ; false -> [$/] end; -maybe_replace($\\, #opts{escaped_strings=true}) -> [$\\, $\\]; maybe_replace(X, Opts=#opts{escaped_strings=true}) when X == 16#2028; X == 16#2029 -> case Opts#opts.unescaped_jsonp of true -> [X] @@ -549,6 +543,7 @@ json_escape_sequence_test_() -> {"json escape sequence test - 16#def", ?_assertEqual(json_escape_sequence(16#def), "\\u0def")} ]. + opts_test_() -> [ {"all flags", @@ -589,23 +584,323 @@ opts_test_() -> ]. +%% erlang refuses to encode certain codepoints, so fake them +to_fake_utf8(N) when N < 16#0080 -> <>; +to_fake_utf8(N) when N < 16#0800 -> + <<0:5, Y:5, X:6>> = <>, + <<2#110:3, Y:5, 2#10:2, X:6>>; +to_fake_utf8(N) when N < 16#10000 -> + <> = <>, + <<2#1110:4, Z:4, 2#10:2, Y:6, 2#10:2, X:6>>; +to_fake_utf8(N) -> + <<0:3, W:3, Z:6, Y:6, X:6>> = <>, + <<2#11110:5, W:3, 2#10:2, Z:6, 2#10:2, Y:6, 2#10:2, X:6>>. + + +codepoints() -> + unicode:characters_to_binary( + [32, 33] + ++ lists:seq(35, 91) + ++ lists:seq(93, 16#2027) + ++ lists:seq(16#202a, 16#d7ff) + ++ lists:seq(16#e000, 16#fdcf) + ++ lists:seq(16#fdf0, 16#fffd) + ). + +escapables() -> + [ to_fake_utf8(N) || N <- + lists:seq(0, 31) ++ [34, 92, 16#2028, 16#2029] + ]. + +extended_codepoints() -> + unicode:characters_to_binary( + lists:seq(16#10000, 16#1fffd) ++ [ + 16#20000, 16#30000, 16#40000, 16#50000, 16#60000, + 16#70000, 16#80000, 16#90000, 16#a0000, 16#b0000, + 16#c0000, 16#d0000, 16#e0000, 16#f0000 + ] ++ lists:seq(16#100000, 16#10fffd) + ). + +noncharacters() -> [ to_fake_utf8(N) || N <- lists:seq(16#fffe, 16#ffff) ]. + +extended_noncharacters() -> + [ to_fake_utf8(N) || N <- [16#1fffe, 16#1ffff, 16#2fffe, 16#2ffff] + ++ [16#3fffe, 16#3ffff, 16#4fffe, 16#4ffff] + ++ [16#5fffe, 16#5ffff, 16#6fffe, 16#6ffff] + ++ [16#7fffe, 16#7ffff, 16#8fffe, 16#8ffff] + ++ [16#9fffe, 16#9ffff, 16#afffe, 16#affff] + ++ [16#bfffe, 16#bffff, 16#cfffe, 16#cffff] + ++ [16#dfffe, 16#dffff, 16#efffe, 16#effff] + ++ [16#ffffe, 16#fffff, 16#10fffe, 16#10ffff] + ]. + +surrogates() -> [ to_fake_utf8(N) || N <- lists:seq(16#d800, 16#dfff) ]. + +reserved_space() -> [ to_fake_utf8(N) || N <- lists:seq(16#fdd0, 16#fdef) ]. + + +fail_ensure(Codepoints) -> + {generator, + fun() -> case Codepoints of + [N|Rest] -> + [ ?_assertError(badarg, ensure_clean(N)) | fail_clean(Rest) ] + ; [] -> [] + end end + }. + +fail_clean(Codepoints) -> + {generator, + fun() -> case Codepoints of + [N|Rest] -> + [ ?_assertError(badarg, clean(N, [], #opts{})) | fail_clean(Rest) ] + ; [] -> [] + end end + }. + +fail_bad(Codepoints) -> + {generator, + fun() -> case Codepoints of + [N|Rest] -> + [ ?_assertError(badarg, clean(N, [], #opts{})) | fail_bad(Rest) ] + ; [] -> [] + end end + }. + +replace_bad(Codepoints) -> + {generator, + fun() -> case Codepoints of + [N|Rest] -> + [ ?_assertEqual( + <<16#fffd/utf8>>, + clean(N, [], #opts{replaced_bad_utf8=true})) + | fail_bad(Rest) + ] + ; [] -> [] + end end + }. + + +ensure_clean_test_() -> + [ + {"basic codepoints", ?_assertEqual(ok, ensure_clean(codepoints()))}, + {"escapables", ?_assertEqual(ok, ensure_clean(unicode:characters_to_binary(escapables())))}, + {"extended codepoints", ?_assertEqual(ok, ensure_clean(extended_codepoints()))}, + {"noncharacters", fail_ensure(noncharacters())}, + {"extended noncharacter", fail_ensure(extended_noncharacters())}, + {"surrogates", fail_ensure(surrogates())}, + {"reserved_space", fail_ensure(reserved_space())} + ]. + + +clean_test_() -> + [ + {"basic codepoints", ?_assertEqual( + codepoints(), + clean(codepoints(), [], #opts{}) + )}, + {"escapables", fail_clean(escapables())}, + {"extended codepoints", ?_assertEqual( + extended_codepoints(), + clean(extended_codepoints(), [], #opts{}) + )}, + {"noncharacters", fail_clean(noncharacters())}, + {"extended noncharacter", fail_clean(extended_noncharacters())}, + {"surrogates", fail_clean(surrogates())}, + {"reserved_space", fail_clean(reserved_space())} + ]. + + +escape_test_() -> + [ + {"escape u0000", ?_assertEqual( + <<"\\u0000">>, + clean_string(to_fake_utf8(16#0000), #opts{escaped_strings=true}) + )}, + {"escape u0001", ?_assertEqual( + <<"\\u0001">>, + clean_string(to_fake_utf8(16#0001), #opts{escaped_strings=true}) + )}, + {"escape u0002", ?_assertEqual( + <<"\\u0002">>, + clean_string(to_fake_utf8(16#0002), #opts{escaped_strings=true}) + )}, + {"escape u0003", ?_assertEqual( + <<"\\u0003">>, + clean_string(to_fake_utf8(16#0003), #opts{escaped_strings=true}) + )}, + {"escape u0004", ?_assertEqual( + <<"\\u0004">>, + clean_string(to_fake_utf8(16#0004), #opts{escaped_strings=true}) + )}, + {"escape u0005", ?_assertEqual( + <<"\\u0005">>, + clean_string(to_fake_utf8(16#0005), #opts{escaped_strings=true}) + )}, + {"escape u0006", ?_assertEqual( + <<"\\u0006">>, + clean_string(to_fake_utf8(16#0006), #opts{escaped_strings=true}) + )}, + {"escape u0007", ?_assertEqual( + <<"\\u0007">>, + clean_string(to_fake_utf8(16#0007), #opts{escaped_strings=true}) + )}, + {"escape u0008", ?_assertEqual( + <<"\\b">>, + clean_string(to_fake_utf8(16#0008), #opts{escaped_strings=true}) + )}, + {"escape u0009", ?_assertEqual( + <<"\\t">>, + clean_string(to_fake_utf8(16#0009), #opts{escaped_strings=true}) + )}, + {"escape u000a", ?_assertEqual( + <<"\\n">>, + clean_string(to_fake_utf8(16#000a), #opts{escaped_strings=true}) + )}, + {"escape u000b", ?_assertEqual( + <<"\\u000b">>, + clean_string(to_fake_utf8(16#000b), #opts{escaped_strings=true}) + )}, + {"escape u000c", ?_assertEqual( + <<"\\f">>, + clean_string(to_fake_utf8(16#000c), #opts{escaped_strings=true}) + )}, + {"escape u000d", ?_assertEqual( + <<"\\r">>, + clean_string(to_fake_utf8(16#000d), #opts{escaped_strings=true}) + )}, + {"escape u000e", ?_assertEqual( + <<"\\u000e">>, + clean_string(to_fake_utf8(16#000e), #opts{escaped_strings=true}) + )}, + {"escape u000f", ?_assertEqual( + <<"\\u000f">>, + clean_string(to_fake_utf8(16#000f), #opts{escaped_strings=true}) + )}, + {"escape u0010", ?_assertEqual( + <<"\\u0010">>, + clean_string(to_fake_utf8(16#0010), #opts{escaped_strings=true}) + )}, + {"escape u0011", ?_assertEqual( + <<"\\u0011">>, + clean_string(to_fake_utf8(16#0011), #opts{escaped_strings=true}) + )}, + {"escape u0012", ?_assertEqual( + <<"\\u0012">>, + clean_string(to_fake_utf8(16#0012), #opts{escaped_strings=true}) + )}, + {"escape u0013", ?_assertEqual( + <<"\\u0013">>, + clean_string(to_fake_utf8(16#0013), #opts{escaped_strings=true}) + )}, + {"escape u0014", ?_assertEqual( + <<"\\u0014">>, + clean_string(to_fake_utf8(16#0014), #opts{escaped_strings=true}) + )}, + {"escape u0015", ?_assertEqual( + <<"\\u0015">>, + clean_string(to_fake_utf8(16#0015), #opts{escaped_strings=true}) + )}, + {"escape u0016", ?_assertEqual( + <<"\\u0016">>, + clean_string(to_fake_utf8(16#0016), #opts{escaped_strings=true}) + )}, + {"escape u0017", ?_assertEqual( + <<"\\u0017">>, + clean_string(to_fake_utf8(16#0017), #opts{escaped_strings=true}) + )}, + {"escape u0018", ?_assertEqual( + <<"\\u0018">>, + clean_string(to_fake_utf8(16#0018), #opts{escaped_strings=true}) + )}, + {"escape u0019", ?_assertEqual( + <<"\\u0019">>, + clean_string(to_fake_utf8(16#0019), #opts{escaped_strings=true}) + )}, + {"escape u001a", ?_assertEqual( + <<"\\u001a">>, + clean_string(to_fake_utf8(16#001a), #opts{escaped_strings=true}) + )}, + {"escape u001b", ?_assertEqual( + <<"\\u001b">>, + clean_string(to_fake_utf8(16#001b), #opts{escaped_strings=true}) + )}, + {"escape u001c", ?_assertEqual( + <<"\\u001c">>, + clean_string(to_fake_utf8(16#001c), #opts{escaped_strings=true}) + )}, + {"escape u001d", ?_assertEqual( + <<"\\u001d">>, + clean_string(to_fake_utf8(16#001d), #opts{escaped_strings=true}) + )}, + {"escape u001e", ?_assertEqual( + <<"\\u001e">>, + clean_string(to_fake_utf8(16#001e), #opts{escaped_strings=true}) + )}, + {"escape u001f", ?_assertEqual( + <<"\\u001f">>, + clean_string(to_fake_utf8(16#001f), #opts{escaped_strings=true}) + )}, + {"escape u0022", ?_assertEqual( + <<"\\\"">>, + clean_string(to_fake_utf8(16#0022), #opts{escaped_strings=true}) + )}, + {"escape u002f", ?_assertEqual( + <<"\\/">>, + clean_string(to_fake_utf8(16#002f), #opts{escaped_strings=true, escaped_forward_slashes=true}) + )}, + {"escape u005c", ?_assertEqual( + <<"\\\\">>, + clean_string(to_fake_utf8(16#005c), #opts{escaped_strings=true}) + )}, + {"escape u2028", ?_assertEqual( + <<"\\u2028">>, + clean_string(to_fake_utf8(16#2028), #opts{escaped_strings=true}) + )}, + {"escape u2029", ?_assertEqual( + <<"\\u2029">>, + clean_string(to_fake_utf8(16#2029), #opts{escaped_strings=true}) + )} + ]. + + bad_utf8_test_() -> [ - {"orphan continuation byte u+0080", - ?_assertError(badarg, clean_string(<<16#0080>>, #opts{})) - }, - {"orphan continuation byte u+0080 replaced", - ?_assertEqual(<<16#fffd/utf8>>, clean_string(<<16#0080>>, #opts{replaced_bad_utf8=true})) - }, - {"orphan continuation byte u+00bf", - ?_assertError(badarg, clean_string(<<16#00bf>>, #opts{})) - }, - {"orphan continuation byte u+00bf replaced", - ?_assertEqual(<<16#fffd/utf8>>, clean_string(<<16#00bf>>, #opts{replaced_bad_utf8=true})) - }, - {"2 continuation bytes", - ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #opts{})) - }, + {"noncharacter u+fffe", ?_assertError(badarg, clean_string(to_fake_utf8(16#fffe), #opts{}))}, + {"noncharacter u+fffe replaced", ?_assertEqual( + <<16#fffd/utf8>>, + clean_string(to_fake_utf8(16#fffe), #opts{replaced_bad_utf8=true}) + )}, + {"noncharacter u+ffff", ?_assertError(badarg, clean_string(to_fake_utf8(16#ffff), #opts{}))}, + {"noncharacter u+ffff replaced", ?_assertEqual( + <<16#fffd/utf8>>, + clean_string(to_fake_utf8(16#ffff), #opts{replaced_bad_utf8=true}) + )}, + {"extended noncharacters", fail_bad(extended_noncharacters())}, + {"extended noncharacters replaced", replace_bad(extended_noncharacters())}, + {"surrogates", fail_bad(surrogates())}, + {"surrogates replaced", replace_bad(surrogates())}, + {"reserved_space", fail_bad(reserved_space())}, + {"reserved_space replaced", replace_bad(reserved_space())}, + {"orphan continuation byte u+0080", ?_assertError( + badarg, + clean_string(<<16#0080>>, #opts{}) + )}, + {"orphan continuation byte u+0080 replaced", ?_assertEqual( + <<16#fffd/utf8>>, + clean_string(<<16#0080>>, #opts{replaced_bad_utf8=true}) + )}, + {"orphan continuation byte u+00bf", ?_assertError( + badarg, + clean_string(<<16#00bf>>, #opts{}) + )}, + {"orphan continuation byte u+00bf replaced", ?_assertEqual( + <<16#fffd/utf8>>, + clean_string(<<16#00bf>>, #opts{replaced_bad_utf8=true}) + )}, + {"2 continuation bytes", ?_assertError( + badarg, + clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #opts{}) + )}, {"2 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 2), clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #opts{replaced_bad_utf8=true}) From 1bf196b9e30591e1cae21fd5e6d1a795f9221e33 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 5 Feb 2013 16:52:28 -0800 Subject: [PATCH 08/61] move all encoder escape tests to jsx_utils --- src/jsx_encoder.erl | 25 ------------ src/jsx_parser.erl | 28 ------------- src/jsx_utils.erl | 96 ++++++++++++++++++++++++++------------------- 3 files changed, 56 insertions(+), 93 deletions(-) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index a06c270..2ca2a0c 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -343,29 +343,4 @@ pre_encoders_test_() -> ]. -escapes_test_() -> - [ - {"backspace escape", ?_assertEqual(encode(<<"\b">>, [escaped_strings]), [{string, <<"\\b">>}, end_json])}, - {"formfeed escape", ?_assertEqual(encode(<<"\f">>, [escaped_strings]), [{string, <<"\\f">>}, end_json])}, - {"newline escape", ?_assertEqual(encode(<<"\n">>, [escaped_strings]), [{string, <<"\\n">>}, end_json])}, - {"carriage return escape", ?_assertEqual(encode(<<"\r">>, [escaped_strings]), [{string, <<"\\r">>}, end_json])}, - {"tab escape", ?_assertEqual(encode(<<"\t">>, [escaped_strings]), [{string, <<"\\t">>}, end_json])}, - {"quote escape", ?_assertEqual(encode(<<"\"">>, [escaped_strings]), [{string, <<"\\\"">>}, end_json])}, - {"forward slash escape", ?_assertEqual(encode(<<"/">>, [escaped_strings, escaped_forward_slashes]), [{string, <<"\\/">>}, end_json])}, - {"no forward slash escape", ?_assertEqual(encode(<<"/">>, [escaped_strings]), [{string, <<"/">>}, end_json])}, - {"back slash escape", ?_assertEqual(encode(<<"\\">>, [escaped_strings]), [{string, <<"\\\\">>}, end_json])}, - {"jsonp escape", ?_assertEqual( - encode(<<16#2028/utf8, 16#2029/utf8>>, [escaped_strings]), - [{string, <<"\\u2028\\u2029">>}, end_json] - )}, - {"no jsonp escape", ?_assertEqual( - encode(<<16#2028/utf8, 16#2029/utf8>>, [escaped_strings, unescaped_jsonp]), - [{string, <<16#2028/utf8, 16#2029/utf8>>}, end_json] - )}, - {"control escape", ?_assertEqual(encode(<<0>>, [escaped_strings]), [{string, <<"\\u0000">>}, end_json])}, - {"dirty strings", ?_assertEqual(encode(<<"\n">>, [escaped_strings, dirty_strings]), [{string, <<"\n">>}, end_json])}, - {"ignore bad escapes", ?_assertEqual(encode(<<"\\x25">>, [escaped_strings, ignored_bad_escapes]), [{string, <<"\\\\x25">>}, end_json])} - ]. - - -endif. \ No newline at end of file diff --git a/src/jsx_parser.erl b/src/jsx_parser.erl index d622b5d..e00bc2c 100644 --- a/src/jsx_parser.erl +++ b/src/jsx_parser.erl @@ -341,32 +341,4 @@ encode_failures_test_() -> ]. - - - -escapes_test_() -> - [ - {"backspace escape", ?_assertEqual(encode(<<"\b">>, [escaped_strings]), [{string, <<"\\b">>}, end_json])}, - {"formfeed escape", ?_assertEqual(encode(<<"\f">>, [escaped_strings]), [{string, <<"\\f">>}, end_json])}, - {"newline escape", ?_assertEqual(encode(<<"\n">>, [escaped_strings]), [{string, <<"\\n">>}, end_json])}, - {"carriage return escape", ?_assertEqual(encode(<<"\r">>, [escaped_strings]), [{string, <<"\\r">>}, end_json])}, - {"tab escape", ?_assertEqual(encode(<<"\t">>, [escaped_strings]), [{string, <<"\\t">>}, end_json])}, - {"quote escape", ?_assertEqual(encode(<<"\"">>, [escaped_strings]), [{string, <<"\\\"">>}, end_json])}, - {"forward slash escape", ?_assertEqual(encode(<<"/">>, [escaped_strings, escaped_forward_slashes]), [{string, <<"\\/">>}, end_json])}, - {"no forward slash escape", ?_assertEqual(encode(<<"/">>, [escaped_strings]), [{string, <<"/">>}, end_json])}, - {"back slash escape", ?_assertEqual(encode(<<"\\">>, [escaped_strings]), [{string, <<"\\\\">>}, end_json])}, - {"jsonp escape", ?_assertEqual( - encode(<<16#2028/utf8, 16#2029/utf8>>, [escaped_strings]), - [{string, <<"\\u2028\\u2029">>}, end_json] - )}, - {"no jsonp escape", ?_assertEqual( - encode(<<16#2028/utf8, 16#2029/utf8>>, [escaped_strings, unescaped_jsonp]), - [{string, <<16#2028/utf8, 16#2029/utf8>>}, end_json] - )}, - {"control escape", ?_assertEqual(encode(<<0>>, [escaped_strings]), [{string, <<"\\u0000">>}, end_json])}, - {"dirty strings", ?_assertEqual(encode(<<"\n">>, [escaped_strings, dirty_strings]), [{string, <<"\n">>}, end_json])}, - {"ignore bad escapes", ?_assertEqual(encode(<<"\\x25">>, [escaped_strings, ignored_bad_escapes]), [{string, <<"\\\\x25">>}, end_json])} - ]. - - -endif. \ No newline at end of file diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index a6c4bf4..31f376c 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -712,6 +712,62 @@ clean_test_() -> escape_test_() -> [ + {"escape backspace", ?_assertEqual( + <<"\\b">>, + clean_string(to_fake_utf8(16#0008), #opts{escaped_strings=true}) + )}, + {"escape tab", ?_assertEqual( + <<"\\t">>, + clean_string(to_fake_utf8(16#0009), #opts{escaped_strings=true}) + )}, + {"escape newline", ?_assertEqual( + <<"\\n">>, + clean_string(to_fake_utf8(16#000a), #opts{escaped_strings=true}) + )}, + {"escape formfeed", ?_assertEqual( + <<"\\f">>, + clean_string(to_fake_utf8(16#000c), #opts{escaped_strings=true}) + )}, + {"escape carriage return", ?_assertEqual( + <<"\\r">>, + clean_string(to_fake_utf8(16#000d), #opts{escaped_strings=true}) + )}, + {"escape quote", ?_assertEqual( + <<"\\\"">>, + clean_string(to_fake_utf8(16#0022), #opts{escaped_strings=true}) + )}, + {"escape forward slash", ?_assertEqual( + <<"\\/">>, + clean_string(to_fake_utf8(16#002f), #opts{escaped_strings=true, escaped_forward_slashes=true}) + )}, + {"do not escape forward slash", ?_assertEqual( + <<"/">>, + clean_string(to_fake_utf8(16#002f), #opts{escaped_strings=true}) + )}, + {"escape backslash", ?_assertEqual( + <<"\\\\">>, + clean_string(to_fake_utf8(16#005c), #opts{escaped_strings=true}) + )}, + {"escape jsonp (u2028)", ?_assertEqual( + <<"\\u2028">>, + clean_string(to_fake_utf8(16#2028), #opts{escaped_strings=true}) + )}, + {"do not escape jsonp (u2028)", ?_assertEqual( + <<16#2028/utf8>>, + clean_string(to_fake_utf8(16#2028), #opts{escaped_strings=true, unescaped_jsonp=true}) + )}, + {"escape jsonp (u2029)", ?_assertEqual( + <<"\\u2029">>, + clean_string(to_fake_utf8(16#2029), #opts{escaped_strings=true}) + )}, + {"do not escape jsonp (u2029)", ?_assertEqual( + <<16#2029/utf8>>, + clean_string(to_fake_utf8(16#2029), #opts{escaped_strings=true, unescaped_jsonp=true}) + )}, + {"dirty string", ?_assertEqual( + <<"\n">>, + clean_string(to_fake_utf8(16#000a), #opts{escaped_strings=true, dirty_strings=true}) + )}, {"escape u0000", ?_assertEqual( <<"\\u0000">>, clean_string(to_fake_utf8(16#0000), #opts{escaped_strings=true}) @@ -744,30 +800,10 @@ escape_test_() -> <<"\\u0007">>, clean_string(to_fake_utf8(16#0007), #opts{escaped_strings=true}) )}, - {"escape u0008", ?_assertEqual( - <<"\\b">>, - clean_string(to_fake_utf8(16#0008), #opts{escaped_strings=true}) - )}, - {"escape u0009", ?_assertEqual( - <<"\\t">>, - clean_string(to_fake_utf8(16#0009), #opts{escaped_strings=true}) - )}, - {"escape u000a", ?_assertEqual( - <<"\\n">>, - clean_string(to_fake_utf8(16#000a), #opts{escaped_strings=true}) - )}, {"escape u000b", ?_assertEqual( <<"\\u000b">>, clean_string(to_fake_utf8(16#000b), #opts{escaped_strings=true}) )}, - {"escape u000c", ?_assertEqual( - <<"\\f">>, - clean_string(to_fake_utf8(16#000c), #opts{escaped_strings=true}) - )}, - {"escape u000d", ?_assertEqual( - <<"\\r">>, - clean_string(to_fake_utf8(16#000d), #opts{escaped_strings=true}) - )}, {"escape u000e", ?_assertEqual( <<"\\u000e">>, clean_string(to_fake_utf8(16#000e), #opts{escaped_strings=true}) @@ -839,26 +875,6 @@ escape_test_() -> {"escape u001f", ?_assertEqual( <<"\\u001f">>, clean_string(to_fake_utf8(16#001f), #opts{escaped_strings=true}) - )}, - {"escape u0022", ?_assertEqual( - <<"\\\"">>, - clean_string(to_fake_utf8(16#0022), #opts{escaped_strings=true}) - )}, - {"escape u002f", ?_assertEqual( - <<"\\/">>, - clean_string(to_fake_utf8(16#002f), #opts{escaped_strings=true, escaped_forward_slashes=true}) - )}, - {"escape u005c", ?_assertEqual( - <<"\\\\">>, - clean_string(to_fake_utf8(16#005c), #opts{escaped_strings=true}) - )}, - {"escape u2028", ?_assertEqual( - <<"\\u2028">>, - clean_string(to_fake_utf8(16#2028), #opts{escaped_strings=true}) - )}, - {"escape u2029", ?_assertEqual( - <<"\\u2029">>, - clean_string(to_fake_utf8(16#2029), #opts{escaped_strings=true}) )} ]. From a325d266bc657934e57cf664d4dfca1447920928 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 5 Feb 2013 16:54:59 -0800 Subject: [PATCH 09/61] whitespace --- src/jsx_encoder.erl | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 2ca2a0c..92a2281 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -73,23 +73,23 @@ value(Term, Handler, Opts) -> ?error([Term, Handler, Opts]). list_or_object([Term|Rest], {Handler, State}, Opts) -> - case pre_encode(Term, Opts) of - {K, V} -> - object([{K, V}|Rest], {Handler, Handler:handle_event(start_object, State)}, Opts) - ; T -> - list([T|Rest], {Handler, Handler:handle_event(start_array, State)}, Opts) - end. + case pre_encode(Term, Opts) of + {K, V} -> + object([{K, V}|Rest], {Handler, Handler:handle_event(start_object, State)}, Opts) + ; T -> + list([T|Rest], {Handler, Handler:handle_event(start_array, State)}, Opts) + end. object([{Key, Value}, Next|Rest], {Handler, State}, Opts) when is_atom(Key); is_binary(Key) -> V = pre_encode(Value, Opts), - object( + object( [pre_encode(Next, Opts)|Rest], { Handler, value( - V, - {Handler, Handler:handle_event({key, clean_string(fix_key(Key), Opts)}, State)}, + V, + {Handler, Handler:handle_event({key, clean_string(fix_key(Key), Opts)}, State)}, Opts ) }, @@ -331,15 +331,15 @@ pre_encoders_test_() -> end_json ] )}, - {"pre_encode list", ?_assertEqual( - encode([1,2,3], [{pre_encode, fun(X) when is_integer(X) -> X + 1; (V) -> V end}]), - [ - start_array, - {integer, 2}, {integer, 3}, {integer, 4}, - end_array, - end_json - ] - )} + {"pre_encode list", ?_assertEqual( + encode([1,2,3], [{pre_encode, fun(X) when is_integer(X) -> X + 1; (V) -> V end}]), + [ + start_array, + {integer, 2}, {integer, 3}, {integer, 4}, + end_array, + end_json + ] + )} ]. From c36bc7babb58559cbb46dfe460995dced099c0e2 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 5 Feb 2013 18:08:19 -0800 Subject: [PATCH 10/61] fix incorrect tail call in jsx_utils test generator --- src/jsx_utils.erl | 56 +++++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index 31f376c..be4fa6a 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -639,41 +639,45 @@ surrogates() -> [ to_fake_utf8(N) || N <- lists:seq(16#d800, 16#dfff) ]. reserved_space() -> [ to_fake_utf8(N) || N <- lists:seq(16#fdd0, 16#fdef) ]. -fail_ensure(Codepoints) -> +fail_ensure(Codepoints, Title) -> {generator, fun() -> case Codepoints of [N|Rest] -> - [ ?_assertError(badarg, ensure_clean(N)) | fail_clean(Rest) ] + [ {Title, ?_assertError(badarg, ensure_clean(N))} + | fail_ensure(Rest, Title) + ] ; [] -> [] end end }. -fail_clean(Codepoints) -> +fail_clean(Codepoints, Title) -> {generator, fun() -> case Codepoints of [N|Rest] -> - [ ?_assertError(badarg, clean(N, [], #opts{})) | fail_clean(Rest) ] + [ {Title, ?_assertError(badarg, clean(N, [], #opts{}))} + | fail_clean(Rest, Title) + ] ; [] -> [] end end }. -fail_bad(Codepoints) -> +fail_bad(Codepoints, Title) -> {generator, fun() -> case Codepoints of [N|Rest] -> - [ ?_assertError(badarg, clean(N, [], #opts{})) | fail_bad(Rest) ] + [ {Title, ?_assertError(badarg, clean(N, [], #opts{}))} + | fail_bad(Rest, Title) + ] ; [] -> [] end end }. -replace_bad(Codepoints) -> +replace_bad(Codepoints, Title) -> {generator, fun() -> case Codepoints of [N|Rest] -> - [ ?_assertEqual( - <<16#fffd/utf8>>, - clean(N, [], #opts{replaced_bad_utf8=true})) - | fail_bad(Rest) + [ {Title, ?_assertEqual(<<16#fffd/utf8>>, clean(N, [], #opts{replaced_bad_utf8=true}))} + | replace_bad(Rest, Title) ] ; [] -> [] end end @@ -685,10 +689,10 @@ ensure_clean_test_() -> {"basic codepoints", ?_assertEqual(ok, ensure_clean(codepoints()))}, {"escapables", ?_assertEqual(ok, ensure_clean(unicode:characters_to_binary(escapables())))}, {"extended codepoints", ?_assertEqual(ok, ensure_clean(extended_codepoints()))}, - {"noncharacters", fail_ensure(noncharacters())}, - {"extended noncharacter", fail_ensure(extended_noncharacters())}, - {"surrogates", fail_ensure(surrogates())}, - {"reserved_space", fail_ensure(reserved_space())} + fail_ensure(noncharacters(), "noncharacters"), + fail_ensure(extended_noncharacters(), "extended noncharacters"), + fail_ensure(surrogates(), "surrogates"), + fail_ensure(reserved_space(), "reserved space") ]. @@ -698,15 +702,15 @@ clean_test_() -> codepoints(), clean(codepoints(), [], #opts{}) )}, - {"escapables", fail_clean(escapables())}, + fail_clean(escapables(), "escapables"), {"extended codepoints", ?_assertEqual( extended_codepoints(), clean(extended_codepoints(), [], #opts{}) )}, - {"noncharacters", fail_clean(noncharacters())}, - {"extended noncharacter", fail_clean(extended_noncharacters())}, - {"surrogates", fail_clean(surrogates())}, - {"reserved_space", fail_clean(reserved_space())} + fail_clean(noncharacters(), "noncharacters"), + fail_clean(extended_noncharacters(), "extended noncharacters"), + fail_clean(surrogates(), "surrogates"), + fail_clean(reserved_space(), "reserved space") ]. @@ -891,12 +895,12 @@ bad_utf8_test_() -> <<16#fffd/utf8>>, clean_string(to_fake_utf8(16#ffff), #opts{replaced_bad_utf8=true}) )}, - {"extended noncharacters", fail_bad(extended_noncharacters())}, - {"extended noncharacters replaced", replace_bad(extended_noncharacters())}, - {"surrogates", fail_bad(surrogates())}, - {"surrogates replaced", replace_bad(surrogates())}, - {"reserved_space", fail_bad(reserved_space())}, - {"reserved_space replaced", replace_bad(reserved_space())}, + fail_bad(extended_noncharacters(), "extended noncharacters"), + replace_bad(extended_noncharacters(), "extended noncharacters replaced"), + fail_bad(surrogates(), "surrogates"), + replace_bad(surrogates(), "surrogates replaced"), + fail_bad(reserved_space(), "reserved space"), + replace_bad(reserved_space(), "reserved space replaced"), {"orphan continuation byte u+0080", ?_assertError( badarg, clean_string(<<16#0080>>, #opts{}) From f45cb3978614330963aedc8ec21a907bb25fa855 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 5 Feb 2013 20:12:29 -0800 Subject: [PATCH 11/61] add test to jsx_utils for multiple defined pre_encoders --- src/jsx_utils.erl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index be4fa6a..479dd76 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -580,7 +580,14 @@ opts_test_() -> ignored_bad_escapes=true } ) - } + }, + {"two pre-encoders defined", ?_assertError( + badarg, + parse_opts([ + {pre_encode, fun(_) -> true end}, + {pre_encode, fun(_) -> false end} + ]) + )} ]. From 77548df033ab677d13725744faac6ac2d2966520 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 5 Feb 2013 20:25:41 -0800 Subject: [PATCH 12/61] add a few more tests for opt parsing --- src/jsx_utils.erl | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index 479dd76..a05486f 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -106,7 +106,7 @@ valid_flags() -> pre_encoder, %% pre_encode loose_unicode, %% replaced_bad_utf8 escape_forward_slash, %% escaped_forward_slashes - single_quotes, %% single_quotes_strings + single_quotes, %% single_quoted_strings no_jsonp_escapes, %% unescaped_jsonp json_escape, %% escaped_strings ignore_bad_escapes %% ignored_bad_escapes @@ -581,7 +581,31 @@ opts_test_() -> } ) }, - {"two pre-encoders defined", ?_assertError( + {"deprecated flags", ?_assertEqual( + parse_opts([ + {pre_encoder, fun lists:length/1}, + loose_unicode, + escape_forward_slash, + single_quotes, + no_jsonp_escapes, + json_escape, + ignore_bad_escapes + ]), + #opts{ + pre_encode=fun lists:length/1, + replaced_bad_utf8=true, + escaped_forward_slashes=true, + single_quoted_strings=true, + unescaped_jsonp=true, + escaped_strings=true, + ignored_bad_escapes=true + } + )}, + {"pre_encode flag", ?_assertEqual( + parse_opts([{pre_encode, fun lists:length/1}]), + #opts{pre_encode=fun lists:length/1} + )}, + {"two pre_encoders defined", ?_assertError( badarg, parse_opts([ {pre_encode, fun(_) -> true end}, From dfbda5a9d3e5b007b11474fd27a7fdba1af76f02 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 5 Feb 2013 20:29:38 -0800 Subject: [PATCH 13/61] fix eunit test ordering in jsx_utils --- src/jsx_utils.erl | 53 ++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index a05486f..78bd7bc 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -548,16 +548,6 @@ opts_test_() -> [ {"all flags", ?_assertEqual( - parse_opts([ - replaced_bad_utf8, - escaped_forward_slashes, - explicit_end, - single_quoted_strings, - unescaped_jsonp, - comments, - dirty_strings, - ignored_bad_escapes - ]), #opts{ replaced_bad_utf8=true, escaped_forward_slashes=true, @@ -567,30 +557,31 @@ opts_test_() -> comments=true, dirty_strings=true, ignored_bad_escapes=true - } + }, + parse_opts([ + replaced_bad_utf8, + escaped_forward_slashes, + explicit_end, + single_quoted_strings, + unescaped_jsonp, + comments, + dirty_strings, + ignored_bad_escapes + ]) ) }, {"relax flag", ?_assertEqual( - parse_opts([relax]), #opts{ replaced_bad_utf8=true, single_quoted_strings=true, comments=true, ignored_bad_escapes=true - } + }, + parse_opts([relax]) ) }, {"deprecated flags", ?_assertEqual( - parse_opts([ - {pre_encoder, fun lists:length/1}, - loose_unicode, - escape_forward_slash, - single_quotes, - no_jsonp_escapes, - json_escape, - ignore_bad_escapes - ]), #opts{ pre_encode=fun lists:length/1, replaced_bad_utf8=true, @@ -599,11 +590,20 @@ opts_test_() -> unescaped_jsonp=true, escaped_strings=true, ignored_bad_escapes=true - } + }, + parse_opts([ + {pre_encoder, fun lists:length/1}, + loose_unicode, + escape_forward_slash, + single_quotes, + no_jsonp_escapes, + json_escape, + ignore_bad_escapes + ]) )}, {"pre_encode flag", ?_assertEqual( - parse_opts([{pre_encode, fun lists:length/1}]), - #opts{pre_encode=fun lists:length/1} + #opts{pre_encode=fun lists:length/1}, + parse_opts([{pre_encode, fun lists:length/1}]) )}, {"two pre_encoders defined", ?_assertError( badarg, @@ -611,7 +611,8 @@ opts_test_() -> {pre_encode, fun(_) -> true end}, {pre_encode, fun(_) -> false end} ]) - )} + )}, + {"bad option flag", ?_assertError(badarg, parse_opts([error]))} ]. From 50133ffefdd309d9501a195de041a027f2847078 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 5 Feb 2013 21:07:38 -0800 Subject: [PATCH 14/61] remove various tests from encoder and parser --- src/jsx_encoder.erl | 211 +------------------------------------------- src/jsx_parser.erl | 172 ------------------------------------ 2 files changed, 2 insertions(+), 381 deletions(-) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 92a2281..fa722ae 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -23,7 +23,7 @@ -module(jsx_encoder). --export([encoder/3]). +-export([encoder/3, pre_encode/2]). -spec encoder(Handler::module(), State::any(), Opts::jsx:opts()) -> jsx:encoder(). @@ -120,7 +120,7 @@ list([], {Handler, State}, _Opts) -> Handler:handle_event(end_array, State); list(Term, Handler, Opts) -> ?error([Term, Handler, Opts]). -pre_encode(Value, #opts{pre_encode=false}) -> Value; +pre_encode(Value, #opts{pre_encode=false}) -> io:format("~p~n", [Value]), Value; pre_encode(Value, Opts) -> (Opts#opts.pre_encode)(Value). @@ -136,211 +136,4 @@ clean_string(Bin, Opts) -> jsx_utils:clean_string(Bin, Opts). -include_lib("eunit/include/eunit.hrl"). -encode(Term) -> encode(Term, []). - -encode(Term, Opts) -> - try (encoder(jsx, [], Opts))(Term) - catch _:_ -> {error, badarg} - end. - - -encode_test_() -> - [ - {"naked string", ?_assertEqual(encode(<<"a string\n">>), [{string, <<"a string\n">>}, end_json])}, - {"escaped naked string", ?_assertEqual(encode(<<"a string\n">>, [escaped_strings]), [{string, <<"a string\\n">>}, end_json])}, - {"naked integer", ?_assertEqual(encode(123), [{integer, 123}, end_json])}, - {"naked float", ?_assertEqual(encode(1.23), [{float, 1.23}, end_json])}, - {"naked literal", ?_assertEqual(encode(null), [{literal, null}, end_json])}, - {"empty object", ?_assertEqual(encode([{}]), [start_object, end_object, end_json])}, - {"empty list", ?_assertEqual(encode([]), [start_array, end_array, end_json])}, - {"simple list", ?_assertEqual( - encode([1,2,3,true,false]), - [ - start_array, - {integer, 1}, - {integer, 2}, - {integer, 3}, - {literal, true}, - {literal, false}, - end_array, - end_json - ] - ) - }, - {"simple object", ?_assertEqual( - encode([{<<"a">>, true}, {<<"b">>, false}]), - [ - start_object, - {key, <<"a">>}, - {literal, true}, - {key, <<"b">>}, - {literal, false}, - end_object, - end_json - ] - ) - }, - {"complex term", ?_assertEqual( - encode([ - {<<"a">>, true}, - {<<"b">>, false}, - {<<"c">>, [1,2,3]}, - {<<"d">>, [{<<"key">>, <<"value">>}]} - ]), - [ - start_object, - {key, <<"a">>}, - {literal, true}, - {key, <<"b">>}, - {literal, false}, - {key, <<"c">>}, - start_array, - {integer, 1}, - {integer, 2}, - {integer, 3}, - end_array, - {key, <<"d">>}, - start_object, - {key, <<"key">>}, - {string, <<"value">>}, - end_object, - end_object, - end_json - ] - ) - }, - {"atom keys", ?_assertEqual( - encode([{key, <<"value">>}]), - [start_object, {key, <<"key">>}, {string, <<"value">>}, end_object, end_json] - ) - } - ]. - - -pre_encoders_test_() -> - Term = [ - {<<"object">>, [ - {<<"literals">>, [true, false, null]}, - {<<"strings">>, [<<"foo">>, <<"bar">>, <<"baz">>]}, - {<<"numbers">>, [1, 1.0, 1.0e0]} - ]} - ], - [ - {"no pre encode", ?_assertEqual( - encode(Term, []), - [ - start_object, - {key, <<"object">>}, start_object, - {key, <<"literals">>}, start_array, - {literal, true}, {literal, false}, {literal, null}, - end_array, - {key, <<"strings">>}, start_array, - {string, <<"foo">>}, {string, <<"bar">>}, {string, <<"baz">>}, - end_array, - {key, <<"numbers">>}, start_array, - {integer, 1}, {float, 1.0}, {float, 1.0}, - end_array, - end_object, - end_object, - end_json - ] - )}, - {"replace lists with empty lists", ?_assertEqual( - encode(Term, [{pre_encode, fun(V) -> case V of [{_,_}|_] -> V; [{}] -> V; V when is_list(V) -> []; _ -> V end end}]), - [ - start_object, - {key, <<"object">>}, start_object, - {key, <<"literals">>}, start_array, end_array, - {key, <<"strings">>}, start_array, end_array, - {key, <<"numbers">>}, start_array, end_array, - end_object, - end_object, - end_json - ] - )}, - {"replace objects with empty objects", ?_assertEqual( - encode(Term, [{pre_encode, fun(V) -> case V of [{_,_}|_] -> [{}]; _ -> V end end}]), - [ - start_object, - end_object, - end_json - ] - )}, - {"replace all non-list and non_tuple values with false", ?_assertEqual( - encode(Term, [{pre_encode, fun(V) when is_list(V); is_tuple(V) -> V; (_) -> false end}]), - [ - start_object, - {key, <<"object">>}, start_object, - {key, <<"literals">>}, start_array, - {literal, false}, {literal, false}, {literal, false}, - end_array, - {key, <<"strings">>}, start_array, - {literal, false}, {literal, false}, {literal, false}, - end_array, - {key, <<"numbers">>}, start_array, - {literal, false}, {literal, false}, {literal, false}, - end_array, - end_object, - end_object, - end_json - ] - )}, - {"replace all atoms with atom_to_list", ?_assertEqual( - encode(Term, [{pre_encode, fun(V) when is_atom(V) -> unicode:characters_to_binary(atom_to_list(V)); (V) -> V end}]), - [ - start_object, - {key, <<"object">>}, start_object, - {key, <<"literals">>}, start_array, - {string, <<"true">>}, {string, <<"false">>}, {string, <<"null">>}, - end_array, - {key, <<"strings">>}, start_array, - {string, <<"foo">>}, {string, <<"bar">>}, {string, <<"baz">>}, - end_array, - {key, <<"numbers">>}, start_array, - {integer, 1}, {float, 1.0}, {float, 1.0}, - end_array, - end_object, - end_object, - end_json - ] - )}, - {"pre_encode tuple", ?_assertEqual( - encode({1, 2, 3}, [{pre_encode, fun(Tuple) when is_tuple(Tuple) -> tuple_to_list(Tuple); (V) -> V end}]), - [ - start_array, - {integer, 1}, {integer, 2}, {integer, 3}, - end_array, - end_json - ] - )}, - {"pre_encode 2-tuples", ?_assertEqual( - encode([{two, 1}, {three, 2}], [{pre_encode, fun({K, V}) -> {K, V + 1}; (V) -> V end}]), - [ - start_object, - {key, <<"two">>}, {integer, 2}, {key, <<"three">>}, {integer, 3}, - end_object, - end_json - ] - )}, - {"pre_encode one field record", ?_assertEqual( - encode([{foo, bar}], [{pre_encode, fun({foo, V}) -> {V, undefined}; (undefined) -> false; (V) -> V end}]), - [ - start_object, - {key, <<"bar">>}, {literal, false}, - end_object, - end_json - ] - )}, - {"pre_encode list", ?_assertEqual( - encode([1,2,3], [{pre_encode, fun(X) when is_integer(X) -> X + 1; (V) -> V end}]), - [ - start_array, - {integer, 2}, {integer, 3}, {integer, 4}, - end_array, - end_json - ] - )} - ]. - - -endif. \ No newline at end of file diff --git a/src/jsx_parser.erl b/src/jsx_parser.erl index e00bc2c..adf8126 100644 --- a/src/jsx_parser.erl +++ b/src/jsx_parser.erl @@ -169,176 +169,4 @@ clean_string(Bin, Opts) -> jsx_utils:clean_string(Bin, Opts). -include_lib("eunit/include/eunit.hrl"). -incomplete_test_() -> - F = parser(jsx, [], []), - [ - {"incomplete test", ?_assertEqual( - begin - {incomplete, A} = F(start_object), - {incomplete, B} = A(key), - {incomplete, C} = B(true), - {incomplete, D} = C(end_object), - D(end_json) - end, - [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json] - )} - ]. - -encode(Term) -> encode(Term, []). - -encode(Term, Opts) -> - try (parser(jsx, [], Opts))(Term) - catch error:badarg -> {error, badarg} - end. - - -encode_test_() -> - [ - {"naked string", ?_assertEqual( - encode([{string, <<"a string\n">>}, end_json]), [{string, <<"a string\n">>}, end_json] - )}, - {"naked integer - simple rep", ?_assertEqual( - encode([123, end_json]), [{integer, 123}, end_json] - )}, - {"naked integer - alt rep", ?_assertEqual( - encode([{number, 123}, end_json]), [{integer, 123}, end_json] - )}, - {"naked integer - full rep", ?_assertEqual( - encode([{integer, 123}, end_json]), [{integer, 123}, end_json] - )}, - {"naked float - simple rep", ?_assertEqual( - encode([1.23, end_json]), [{float, 1.23}, end_json] - )}, - {"naked float - alt rep", ?_assertEqual( - encode([{number, 1.23}, end_json]), [{float, 1.23}, end_json] - )}, - {"naked float - full rep", ?_assertEqual( - encode([{float, 1.23}, end_json]), [{float, 1.23}, end_json] - )}, - {"naked literal - simple rep", ?_assertEqual( - encode([null, end_json]), [{literal, null}, end_json] - )}, - {"naked literal - full rep", ?_assertEqual( - encode([{literal, null}, end_json]), [{literal, null}, end_json] - )}, - {"empty object", ?_assertEqual( - encode([start_object, end_object, end_json]), [start_object, end_object, end_json] - )}, - {"empty list", ?_assertEqual( - encode([start_array, end_array, end_json]), [start_array, end_array, end_json] - )}, - {"simple list", ?_assertEqual( - encode([ - start_array, - {integer, 1}, - {integer, 2}, - {integer, 3}, - {literal, true}, - {literal, false}, - end_array, - end_json - ]), - [ - start_array, - {integer, 1}, - {integer, 2}, - {integer, 3}, - {literal, true}, - {literal, false}, - end_array, - end_json - ] - ) - }, - {"simple object", ?_assertEqual( - encode([ - start_object, - {key, <<"a">>}, - {literal, true}, - {key, <<"b">>}, - {literal, false}, - end_object, - end_json - ]), - [ - start_object, - {key, <<"a">>}, - {literal, true}, - {key, <<"b">>}, - {literal, false}, - end_object, - end_json - ] - ) - }, - {"complex term", ?_assertEqual( - encode([ - start_object, - {key, <<"a">>}, - {literal, true}, - {key, <<"b">>}, - {literal, false}, - {key, <<"c">>}, - start_array, - {integer, 1}, - {integer, 2}, - {integer, 3}, - end_array, - {key, <<"d">>}, - start_object, - {key, <<"key">>}, - {string, <<"value">>}, - end_object, - end_object, - end_json - ]), - [ - start_object, - {key, <<"a">>}, - {literal, true}, - {key, <<"b">>}, - {literal, false}, - {key, <<"c">>}, - start_array, - {integer, 1}, - {integer, 2}, - {integer, 3}, - end_array, - {key, <<"d">>}, - start_object, - {key, <<"key">>}, - {string, <<"value">>}, - end_object, - end_object, - end_json - ] - ) - }, - {"atom keys", ?_assertEqual( - encode([start_object, {key, key}, {string, <<"value">>}, end_object, end_json]), - [start_object, {key, <<"key">>}, {string, <<"value">>}, end_object, end_json] - )} - ]. - -encode_failures_test_() -> - [ - {"unwrapped values", ?_assertEqual( - {error, badarg}, - encode([{string, <<"a string\n">>}, {string, <<"a string\n">>}, end_json]) - )}, - {"unbalanced array", ?_assertEqual( - {error, badarg}, - encode([start_array, end_array, end_array, end_json]) - )}, - {"premature finish", ?_assertEqual( - {error, badarg}, - encode([start_object, {key, <<"key">>, start_array, end_json}]) - )}, - {"really premature finish", ?_assertEqual( - {error, badarg}, - encode([end_json]) - )} - ]. - - -endif. \ No newline at end of file From 12b319eccc93a0e7cf4993bce538d8ee411fe3ec Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 5 Feb 2013 21:39:19 -0800 Subject: [PATCH 15/61] remove all old external test cases --- priv/test_cases/absurdly_deep_array.json | 1 - priv/test_cases/absurdly_deep_array.test | 1550 ----------------- priv/test_cases/array.json | 1 - priv/test_cases/array.test | 29 - priv/test_cases/bad_low_surrogate.json | 1 - priv/test_cases/bad_low_surrogate.test | 3 - .../bad_low_surrogate_replaced.json | 1 - .../bad_low_surrogate_replaced.test | 4 - priv/test_cases/bad_naked_number.json | 1 - priv/test_cases/bad_naked_number.test | 3 - priv/test_cases/bom.json | 1 - priv/test_cases/bom.test | 3 - priv/test_cases/comment_style_a.json | 10 - priv/test_cases/comment_style_a.test | 4 - priv/test_cases/comment_style_b.json | 1 - priv/test_cases/comment_style_b.test | 4 - priv/test_cases/deep_array.json | 1 - priv/test_cases/deep_array.test | 3 - priv/test_cases/empty_array.json | 1 - priv/test_cases/empty_array.test | 3 - priv/test_cases/empty_object.json | 1 - priv/test_cases/empty_object.test | 3 - priv/test_cases/empty_object_in_array.json | 1 - priv/test_cases/empty_object_in_array.test | 3 - priv/test_cases/empty_string.json | 1 - priv/test_cases/empty_string.test | 3 - priv/test_cases/encoded_surrogates.json | 1 - priv/test_cases/encoded_surrogates.test | 3 - priv/test_cases/escaped_control.json | 1 - priv/test_cases/escaped_control.test | 3 - priv/test_cases/escapes.json | 1 - priv/test_cases/escapes.test | 5 - priv/test_cases/exp.json | 1 - priv/test_cases/exp.test | 14 - priv/test_cases/false.json | 1 - priv/test_cases/false.test | 3 - priv/test_cases/fraction.json | 1 - priv/test_cases/fraction.test | 19 - priv/test_cases/integer.json | 1 - priv/test_cases/integer.test | 14 - priv/test_cases/multibyte_utf.json | 1 - priv/test_cases/multibyte_utf.test | 3 - priv/test_cases/naked_number_a.json | 1 - priv/test_cases/naked_number_a.test | 3 - priv/test_cases/naked_number_b.json | 1 - priv/test_cases/naked_number_b.test | 3 - priv/test_cases/naked_number_c.json | 1 - priv/test_cases/naked_number_c.test | 3 - priv/test_cases/naked_number_d.json | 1 - priv/test_cases/naked_number_d.test | 3 - priv/test_cases/naked_number_e.json | 1 - priv/test_cases/naked_number_e.test | 3 - priv/test_cases/naked_number_f.json | 1 - priv/test_cases/naked_number_f.test | 3 - priv/test_cases/naked_number_g.json | 1 - priv/test_cases/naked_number_g.test | 3 - priv/test_cases/naked_string.json | 1 - priv/test_cases/naked_string.test | 3 - priv/test_cases/negative_zero.json | 1 - priv/test_cases/negative_zero.test | 14 - priv/test_cases/null.json | 1 - priv/test_cases/null.test | 3 - priv/test_cases/numbers.json | 1 - priv/test_cases/numbers.test | 23 - priv/test_cases/object.json | 1 - priv/test_cases/object.test | 22 - priv/test_cases/string.json | 1 - priv/test_cases/string.test | 5 - priv/test_cases/string_escapes.json | 1 - priv/test_cases/string_escapes.test | 11 - priv/test_cases/true.json | 1 - priv/test_cases/true.test | 3 - priv/test_cases/unbalanced_array.json | 1 - priv/test_cases/unbalanced_array.test | 3 - priv/test_cases/unicode_to_codepoint.json | 1 - priv/test_cases/unicode_to_codepoint.test | 6 - priv/test_cases/unpaired_surrogate.json | 1 - priv/test_cases/unpaired_surrogate.test | 3 - .../unpaired_surrogate_replaced.json | 1 - .../unpaired_surrogate_replaced.test | 4 - priv/test_cases/whitespace.json | 4 - priv/test_cases/whitespace.test | 3 - priv/test_cases/zero.json | 1 - priv/test_cases/zero.test | 14 - 84 files changed, 1871 deletions(-) delete mode 100644 priv/test_cases/absurdly_deep_array.json delete mode 100644 priv/test_cases/absurdly_deep_array.test delete mode 100644 priv/test_cases/array.json delete mode 100644 priv/test_cases/array.test delete mode 100644 priv/test_cases/bad_low_surrogate.json delete mode 100644 priv/test_cases/bad_low_surrogate.test delete mode 100644 priv/test_cases/bad_low_surrogate_replaced.json delete mode 100644 priv/test_cases/bad_low_surrogate_replaced.test delete mode 100644 priv/test_cases/bad_naked_number.json delete mode 100644 priv/test_cases/bad_naked_number.test delete mode 100644 priv/test_cases/bom.json delete mode 100644 priv/test_cases/bom.test delete mode 100644 priv/test_cases/comment_style_a.json delete mode 100644 priv/test_cases/comment_style_a.test delete mode 100644 priv/test_cases/comment_style_b.json delete mode 100644 priv/test_cases/comment_style_b.test delete mode 100644 priv/test_cases/deep_array.json delete mode 100644 priv/test_cases/deep_array.test delete mode 100644 priv/test_cases/empty_array.json delete mode 100644 priv/test_cases/empty_array.test delete mode 100644 priv/test_cases/empty_object.json delete mode 100644 priv/test_cases/empty_object.test delete mode 100644 priv/test_cases/empty_object_in_array.json delete mode 100644 priv/test_cases/empty_object_in_array.test delete mode 100644 priv/test_cases/empty_string.json delete mode 100644 priv/test_cases/empty_string.test delete mode 100644 priv/test_cases/encoded_surrogates.json delete mode 100644 priv/test_cases/encoded_surrogates.test delete mode 100644 priv/test_cases/escaped_control.json delete mode 100644 priv/test_cases/escaped_control.test delete mode 100644 priv/test_cases/escapes.json delete mode 100644 priv/test_cases/escapes.test delete mode 100644 priv/test_cases/exp.json delete mode 100644 priv/test_cases/exp.test delete mode 100644 priv/test_cases/false.json delete mode 100644 priv/test_cases/false.test delete mode 100644 priv/test_cases/fraction.json delete mode 100644 priv/test_cases/fraction.test delete mode 100644 priv/test_cases/integer.json delete mode 100644 priv/test_cases/integer.test delete mode 100644 priv/test_cases/multibyte_utf.json delete mode 100644 priv/test_cases/multibyte_utf.test delete mode 100644 priv/test_cases/naked_number_a.json delete mode 100644 priv/test_cases/naked_number_a.test delete mode 100644 priv/test_cases/naked_number_b.json delete mode 100644 priv/test_cases/naked_number_b.test delete mode 100644 priv/test_cases/naked_number_c.json delete mode 100644 priv/test_cases/naked_number_c.test delete mode 100644 priv/test_cases/naked_number_d.json delete mode 100644 priv/test_cases/naked_number_d.test delete mode 100644 priv/test_cases/naked_number_e.json delete mode 100644 priv/test_cases/naked_number_e.test delete mode 100644 priv/test_cases/naked_number_f.json delete mode 100644 priv/test_cases/naked_number_f.test delete mode 100644 priv/test_cases/naked_number_g.json delete mode 100644 priv/test_cases/naked_number_g.test delete mode 100644 priv/test_cases/naked_string.json delete mode 100644 priv/test_cases/naked_string.test delete mode 100644 priv/test_cases/negative_zero.json delete mode 100644 priv/test_cases/negative_zero.test delete mode 100644 priv/test_cases/null.json delete mode 100644 priv/test_cases/null.test delete mode 100644 priv/test_cases/numbers.json delete mode 100644 priv/test_cases/numbers.test delete mode 100644 priv/test_cases/object.json delete mode 100644 priv/test_cases/object.test delete mode 100644 priv/test_cases/string.json delete mode 100644 priv/test_cases/string.test delete mode 100644 priv/test_cases/string_escapes.json delete mode 100644 priv/test_cases/string_escapes.test delete mode 100644 priv/test_cases/true.json delete mode 100644 priv/test_cases/true.test delete mode 100644 priv/test_cases/unbalanced_array.json delete mode 100644 priv/test_cases/unbalanced_array.test delete mode 100644 priv/test_cases/unicode_to_codepoint.json delete mode 100644 priv/test_cases/unicode_to_codepoint.test delete mode 100644 priv/test_cases/unpaired_surrogate.json delete mode 100644 priv/test_cases/unpaired_surrogate.test delete mode 100644 priv/test_cases/unpaired_surrogate_replaced.json delete mode 100644 priv/test_cases/unpaired_surrogate_replaced.test delete mode 100644 priv/test_cases/whitespace.json delete mode 100644 priv/test_cases/whitespace.test delete mode 100644 priv/test_cases/zero.json delete mode 100644 priv/test_cases/zero.test diff --git a/priv/test_cases/absurdly_deep_array.json b/priv/test_cases/absurdly_deep_array.json deleted file mode 100644 index b3ba00e..0000000 --- a/priv/test_cases/absurdly_deep_array.json +++ /dev/null @@ -1 +0,0 @@ -[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] \ No newline at end of file diff --git a/priv/test_cases/absurdly_deep_array.test b/priv/test_cases/absurdly_deep_array.test deleted file mode 100644 index 8d51dcb..0000000 --- a/priv/test_cases/absurdly_deep_array.test +++ /dev/null @@ -1,1550 +0,0 @@ -{name, "absurdly_deep_array"}. -{jsx, [start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,start_array,start_array,start_array,start_array, - start_array,start_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_array,end_array,end_array, - end_array,end_array,end_array,end_array,end_json]}. -{json, "absurdly_deep_array.json"}. diff --git a/priv/test_cases/array.json b/priv/test_cases/array.json deleted file mode 100644 index ee1041c..0000000 --- a/priv/test_cases/array.json +++ /dev/null @@ -1 +0,0 @@ -["foo","bar", "baz",[true],[false],[null],true, false, null, 0.7, {"key":"value"},[{}, null,null,null,[]],"\n\r\\", [-1]] diff --git a/priv/test_cases/array.test b/priv/test_cases/array.test deleted file mode 100644 index 7150207..0000000 --- a/priv/test_cases/array.test +++ /dev/null @@ -1,29 +0,0 @@ -{name, "array"}. -{jsx, [start_array, - {string,<<"foo">>}, - {string,<<"bar">>}, - {string,<<"baz">>}, - start_array, - {literal,true}, - end_array,start_array, - {literal,false}, - end_array,start_array, - {literal,null}, - end_array, - {literal,true}, - {literal,false}, - {literal,null}, - {float,0.7}, - start_object, - {key,<<"key">>}, - {string,<<"value">>}, - end_object,start_array,start_object,end_object, - {literal,null}, - {literal,null}, - {literal,null}, - start_array,end_array,end_array, - {string,<<"\n\r\\">>}, - start_array, - {integer,-1}, - end_array,end_array,end_json]}. -{json, "array.json"}. diff --git a/priv/test_cases/bad_low_surrogate.json b/priv/test_cases/bad_low_surrogate.json deleted file mode 100644 index 4b22b57..0000000 --- a/priv/test_cases/bad_low_surrogate.json +++ /dev/null @@ -1 +0,0 @@ -["\ud801\u0032"] \ No newline at end of file diff --git a/priv/test_cases/bad_low_surrogate.test b/priv/test_cases/bad_low_surrogate.test deleted file mode 100644 index 7d85b62..0000000 --- a/priv/test_cases/bad_low_surrogate.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "bad_low_surrogate"}. -{jsx, {error, badarg}}. -{json, "bad_low_surrogate.json"}. diff --git a/priv/test_cases/bad_low_surrogate_replaced.json b/priv/test_cases/bad_low_surrogate_replaced.json deleted file mode 100644 index 4b22b57..0000000 --- a/priv/test_cases/bad_low_surrogate_replaced.json +++ /dev/null @@ -1 +0,0 @@ -["\ud801\u0032"] \ No newline at end of file diff --git a/priv/test_cases/bad_low_surrogate_replaced.test b/priv/test_cases/bad_low_surrogate_replaced.test deleted file mode 100644 index 5762c33..0000000 --- a/priv/test_cases/bad_low_surrogate_replaced.test +++ /dev/null @@ -1,4 +0,0 @@ -{name, "bad_low_surrogate_replaced"}. -{jsx, [start_array,{string, <<16#fffd/utf8, 16#fffd/utf8>>},end_array,end_json]}. -{json, "bad_low_surrogate_replaced.json"}. -{jsx_flags, [replaced_bad_utf8]}. diff --git a/priv/test_cases/bad_naked_number.json b/priv/test_cases/bad_naked_number.json deleted file mode 100644 index 92880af..0000000 --- a/priv/test_cases/bad_naked_number.json +++ /dev/null @@ -1 +0,0 @@ -1 1 \ No newline at end of file diff --git a/priv/test_cases/bad_naked_number.test b/priv/test_cases/bad_naked_number.test deleted file mode 100644 index b620a6e..0000000 --- a/priv/test_cases/bad_naked_number.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "bad naked number"}. -{jsx, {error, badarg}}. -{json, "bad_naked_number.json"}. diff --git a/priv/test_cases/bom.json b/priv/test_cases/bom.json deleted file mode 100644 index ad47dbb..0000000 --- a/priv/test_cases/bom.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/priv/test_cases/bom.test b/priv/test_cases/bom.test deleted file mode 100644 index 75b9d3e..0000000 --- a/priv/test_cases/bom.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "byte order mark"}. -{jsx, [start_array, end_array, end_json]}. -{json, "bom.json"}. diff --git a/priv/test_cases/comment_style_a.json b/priv/test_cases/comment_style_a.json deleted file mode 100644 index 5bdc1f7..0000000 --- a/priv/test_cases/comment_style_a.json +++ /dev/null @@ -1,10 +0,0 @@ -// comment -{ // comment - "key" // comment - : // comment - [ // comment - true // comment - , // comment - false // comment - ] // comment -} // comment \ No newline at end of file diff --git a/priv/test_cases/comment_style_a.test b/priv/test_cases/comment_style_a.test deleted file mode 100644 index 60d9900..0000000 --- a/priv/test_cases/comment_style_a.test +++ /dev/null @@ -1,4 +0,0 @@ -{name, "comment_style_a"}. -{jsx, [start_object,{key, <<"key">>}, start_array, {literal, true}, {literal, false}, end_array, end_object,end_json]}. -{json, "comment_style_a.json"}. -{jsx_flags, [comments]}. \ No newline at end of file diff --git a/priv/test_cases/comment_style_b.json b/priv/test_cases/comment_style_b.json deleted file mode 100644 index c515fee..0000000 --- a/priv/test_cases/comment_style_b.json +++ /dev/null @@ -1 +0,0 @@ -/* comment */ { /* comment */ "key" /* comment */ : /* comment */ [ /* comment */ true /* comment */ , /* comment */ false /* comment */ ] /* comment */ } /* comment */ \ No newline at end of file diff --git a/priv/test_cases/comment_style_b.test b/priv/test_cases/comment_style_b.test deleted file mode 100644 index 60d9900..0000000 --- a/priv/test_cases/comment_style_b.test +++ /dev/null @@ -1,4 +0,0 @@ -{name, "comment_style_a"}. -{jsx, [start_object,{key, <<"key">>}, start_array, {literal, true}, {literal, false}, end_array, end_object,end_json]}. -{json, "comment_style_a.json"}. -{jsx_flags, [comments]}. \ No newline at end of file diff --git a/priv/test_cases/deep_array.json b/priv/test_cases/deep_array.json deleted file mode 100644 index 1dfb392..0000000 --- a/priv/test_cases/deep_array.json +++ /dev/null @@ -1 +0,0 @@ -[[[]]] \ No newline at end of file diff --git a/priv/test_cases/deep_array.test b/priv/test_cases/deep_array.test deleted file mode 100644 index 35ef9b9..0000000 --- a/priv/test_cases/deep_array.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "deep array"}. -{jsx, [start_array,start_array,start_array,end_array,end_array,end_array,end_json]}. -{json, "deep_array.json"}. diff --git a/priv/test_cases/empty_array.json b/priv/test_cases/empty_array.json deleted file mode 100644 index 0637a08..0000000 --- a/priv/test_cases/empty_array.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/priv/test_cases/empty_array.test b/priv/test_cases/empty_array.test deleted file mode 100644 index e538683..0000000 --- a/priv/test_cases/empty_array.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "empty_array"}. -{jsx, [start_array,end_array,end_json]}. -{json, "empty_array.json"}. diff --git a/priv/test_cases/empty_object.json b/priv/test_cases/empty_object.json deleted file mode 100644 index 9e26dfe..0000000 --- a/priv/test_cases/empty_object.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/priv/test_cases/empty_object.test b/priv/test_cases/empty_object.test deleted file mode 100644 index 0c985ab..0000000 --- a/priv/test_cases/empty_object.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "empty_object"}. -{jsx, [start_object,end_object,end_json]}. -{json, "empty_object.json"}. diff --git a/priv/test_cases/empty_object_in_array.json b/priv/test_cases/empty_object_in_array.json deleted file mode 100644 index ee1aac4..0000000 --- a/priv/test_cases/empty_object_in_array.json +++ /dev/null @@ -1 +0,0 @@ -[{}] \ No newline at end of file diff --git a/priv/test_cases/empty_object_in_array.test b/priv/test_cases/empty_object_in_array.test deleted file mode 100644 index 0a8679d..0000000 --- a/priv/test_cases/empty_object_in_array.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "empty_object_in_array"}. -{jsx, [start_array,start_object,end_object,end_array,end_json]}. -{json, "empty_object_in_array.json"}. diff --git a/priv/test_cases/empty_string.json b/priv/test_cases/empty_string.json deleted file mode 100644 index 3cc762b..0000000 --- a/priv/test_cases/empty_string.json +++ /dev/null @@ -1 +0,0 @@ -"" \ No newline at end of file diff --git a/priv/test_cases/empty_string.test b/priv/test_cases/empty_string.test deleted file mode 100644 index c6faf71..0000000 --- a/priv/test_cases/empty_string.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "empty_string"}. -{jsx, [{string, <<>>},end_json]}. -{json, "empty_string.json"}. diff --git a/priv/test_cases/encoded_surrogates.json b/priv/test_cases/encoded_surrogates.json deleted file mode 100644 index 7c0e3a9..0000000 --- a/priv/test_cases/encoded_surrogates.json +++ /dev/null @@ -1 +0,0 @@ -["\ud801\udc00"] \ No newline at end of file diff --git a/priv/test_cases/encoded_surrogates.test b/priv/test_cases/encoded_surrogates.test deleted file mode 100644 index aa5d317..0000000 --- a/priv/test_cases/encoded_surrogates.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "encoded_surrogates"}. -{jsx, [start_array,{string,<<66560/utf8>>},end_array,end_json]}. -{json, "encoded_surrogates.json"}. diff --git a/priv/test_cases/escaped_control.json b/priv/test_cases/escaped_control.json deleted file mode 100644 index 78af83f..0000000 --- a/priv/test_cases/escaped_control.json +++ /dev/null @@ -1 +0,0 @@ -"\u0012" \ No newline at end of file diff --git a/priv/test_cases/escaped_control.test b/priv/test_cases/escaped_control.test deleted file mode 100644 index 603d719..0000000 --- a/priv/test_cases/escaped_control.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "escaped_control"}. -{jsx, [{string, <<18>>},end_json]}. -{json, "escaped_control.json"}. diff --git a/priv/test_cases/escapes.json b/priv/test_cases/escapes.json deleted file mode 100644 index 7c16923..0000000 --- a/priv/test_cases/escapes.json +++ /dev/null @@ -1 +0,0 @@ -[ "\"\\\/\b\f\n\r\t\u0020" ] diff --git a/priv/test_cases/escapes.test b/priv/test_cases/escapes.test deleted file mode 100644 index ac1ea14..0000000 --- a/priv/test_cases/escapes.test +++ /dev/null @@ -1,5 +0,0 @@ -{name, "escapes"}. -{jsx, [start_array, - {string,<<"\"\\/\b\f\n\r\t ">>}, - end_array,end_json]}. -{json, "escapes.json"}. diff --git a/priv/test_cases/exp.json b/priv/test_cases/exp.json deleted file mode 100644 index 502c9b3..0000000 --- a/priv/test_cases/exp.json +++ /dev/null @@ -1 +0,0 @@ -[[2.0e7], 2.0e7, {"key":2.0e7, "another key":2.0E7}, 4.2e70 ] \ No newline at end of file diff --git a/priv/test_cases/exp.test b/priv/test_cases/exp.test deleted file mode 100644 index c4b9bd4..0000000 --- a/priv/test_cases/exp.test +++ /dev/null @@ -1,14 +0,0 @@ -{name, "exp"}. -{jsx, [start_array,start_array, - {float,2.0e7}, - end_array, - {float,2.0e7}, - start_object, - {key,<<"key">>}, - {float,2.0e7}, - {key,<<"another key">>}, - {float,2.0e7}, - end_object, - {float,4.2e70}, - end_array,end_json]}. -{json, "exp.json"}. diff --git a/priv/test_cases/false.json b/priv/test_cases/false.json deleted file mode 100644 index 02e4a84..0000000 --- a/priv/test_cases/false.json +++ /dev/null @@ -1 +0,0 @@ -false \ No newline at end of file diff --git a/priv/test_cases/false.test b/priv/test_cases/false.test deleted file mode 100644 index f40af7f..0000000 --- a/priv/test_cases/false.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "false"}. -{jsx, [{literal, false},end_json]}. -{json, "false.json"}. diff --git a/priv/test_cases/fraction.json b/priv/test_cases/fraction.json deleted file mode 100644 index 97037c7..0000000 --- a/priv/test_cases/fraction.json +++ /dev/null @@ -1 +0,0 @@ -[[2.0], 2.0, {"key":2.0e7, "another key":2.0E7}, {"key":2.0, "another key":2.0}, 4.321 ] \ No newline at end of file diff --git a/priv/test_cases/fraction.test b/priv/test_cases/fraction.test deleted file mode 100644 index 7cfdcf2..0000000 --- a/priv/test_cases/fraction.test +++ /dev/null @@ -1,19 +0,0 @@ -{name, "fraction"}. -{jsx, [start_array,start_array, - {float,2.0}, - end_array, - {float,2.0}, - start_object, - {key,<<"key">>}, - {float,2.0e7}, - {key,<<"another key">>}, - {float,2.0e7}, - end_object,start_object, - {key,<<"key">>}, - {float,2.0}, - {key,<<"another key">>}, - {float,2.0}, - end_object, - {float,4.321}, - end_array,end_json]}. -{json, "fraction.json"}. \ No newline at end of file diff --git a/priv/test_cases/integer.json b/priv/test_cases/integer.json deleted file mode 100644 index fb3cf6e..0000000 --- a/priv/test_cases/integer.json +++ /dev/null @@ -1 +0,0 @@ -[[20], 20, {"key":20, "another key":20}, 42 ] \ No newline at end of file diff --git a/priv/test_cases/integer.test b/priv/test_cases/integer.test deleted file mode 100644 index 0883c33..0000000 --- a/priv/test_cases/integer.test +++ /dev/null @@ -1,14 +0,0 @@ -{name, "integer"}. -{jsx, [start_array,start_array, - {integer,20}, - end_array, - {integer,20}, - start_object, - {key,<<"key">>}, - {integer,20}, - {key,<<"another key">>}, - {integer,20}, - end_object, - {integer,42}, - end_array,end_json]}. -{json, "integer.json"}. diff --git a/priv/test_cases/multibyte_utf.json b/priv/test_cases/multibyte_utf.json deleted file mode 100644 index b7f80d7..0000000 --- a/priv/test_cases/multibyte_utf.json +++ /dev/null @@ -1 +0,0 @@ -[ " 𝄞 " ] \ No newline at end of file diff --git a/priv/test_cases/multibyte_utf.test b/priv/test_cases/multibyte_utf.test deleted file mode 100644 index cb8be9c..0000000 --- a/priv/test_cases/multibyte_utf.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "multibyte_utf"}. -{jsx, [start_array,{string,<<32,119070/utf8,32>>},end_array,end_json]}. -{json, "multibyte_utf.json"}. diff --git a/priv/test_cases/naked_number_a.json b/priv/test_cases/naked_number_a.json deleted file mode 100644 index f70d7bb..0000000 --- a/priv/test_cases/naked_number_a.json +++ /dev/null @@ -1 +0,0 @@ -42 \ No newline at end of file diff --git a/priv/test_cases/naked_number_a.test b/priv/test_cases/naked_number_a.test deleted file mode 100644 index debbc47..0000000 --- a/priv/test_cases/naked_number_a.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "naked_number_a"}. -{jsx, [{integer,42},end_json]}. -{json, "naked_number_a.json"}. diff --git a/priv/test_cases/naked_number_b.json b/priv/test_cases/naked_number_b.json deleted file mode 100644 index 67f7ad0..0000000 --- a/priv/test_cases/naked_number_b.json +++ /dev/null @@ -1 +0,0 @@ --42 \ No newline at end of file diff --git a/priv/test_cases/naked_number_b.test b/priv/test_cases/naked_number_b.test deleted file mode 100644 index 55f50f8..0000000 --- a/priv/test_cases/naked_number_b.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "naked_number_b"}. -{jsx, [{integer,-42},end_json]}. -{json, "naked_number_b.json"}. diff --git a/priv/test_cases/naked_number_c.json b/priv/test_cases/naked_number_c.json deleted file mode 100644 index 017da84..0000000 --- a/priv/test_cases/naked_number_c.json +++ /dev/null @@ -1 +0,0 @@ --0.7 \ No newline at end of file diff --git a/priv/test_cases/naked_number_c.test b/priv/test_cases/naked_number_c.test deleted file mode 100644 index 5715167..0000000 --- a/priv/test_cases/naked_number_c.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "naked_number_c"}. -{jsx, [{float,-0.7},end_json]}. -{json, "naked_number_c.json"}. diff --git a/priv/test_cases/naked_number_d.json b/priv/test_cases/naked_number_d.json deleted file mode 100644 index 0e2c939..0000000 --- a/priv/test_cases/naked_number_d.json +++ /dev/null @@ -1 +0,0 @@ -0.7 \ No newline at end of file diff --git a/priv/test_cases/naked_number_d.test b/priv/test_cases/naked_number_d.test deleted file mode 100644 index 15202a1..0000000 --- a/priv/test_cases/naked_number_d.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "naked_number_d"}. -{jsx, [{float,0.7},end_json]}. -{json, "naked_number_d.json"}. diff --git a/priv/test_cases/naked_number_e.json b/priv/test_cases/naked_number_e.json deleted file mode 100644 index c227083..0000000 --- a/priv/test_cases/naked_number_e.json +++ /dev/null @@ -1 +0,0 @@ -0 \ No newline at end of file diff --git a/priv/test_cases/naked_number_e.test b/priv/test_cases/naked_number_e.test deleted file mode 100644 index 6bf7821..0000000 --- a/priv/test_cases/naked_number_e.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "naked_number_e"}. -{jsx, [{integer,0},end_json]}. -{json, "naked_number_e.json"}. diff --git a/priv/test_cases/naked_number_f.json b/priv/test_cases/naked_number_f.json deleted file mode 100644 index c624f44..0000000 --- a/priv/test_cases/naked_number_f.json +++ /dev/null @@ -1 +0,0 @@ -1e100 \ No newline at end of file diff --git a/priv/test_cases/naked_number_f.test b/priv/test_cases/naked_number_f.test deleted file mode 100644 index d3d8ae6..0000000 --- a/priv/test_cases/naked_number_f.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "naked_number_f"}. -{jsx, [{float,1.0e100},end_json]}. -{json, "naked_number_f.json"}. diff --git a/priv/test_cases/naked_number_g.json b/priv/test_cases/naked_number_g.json deleted file mode 100644 index c793025..0000000 --- a/priv/test_cases/naked_number_g.json +++ /dev/null @@ -1 +0,0 @@ -7 \ No newline at end of file diff --git a/priv/test_cases/naked_number_g.test b/priv/test_cases/naked_number_g.test deleted file mode 100644 index 45e324a..0000000 --- a/priv/test_cases/naked_number_g.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "naked_number_g"}. -{jsx, [{integer,7},end_json]}. -{json, "naked_number_g.json"}. diff --git a/priv/test_cases/naked_string.json b/priv/test_cases/naked_string.json deleted file mode 100644 index e030d6b..0000000 --- a/priv/test_cases/naked_string.json +++ /dev/null @@ -1 +0,0 @@ -"this is a naked string" \ No newline at end of file diff --git a/priv/test_cases/naked_string.test b/priv/test_cases/naked_string.test deleted file mode 100644 index 68fe34b..0000000 --- a/priv/test_cases/naked_string.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "naked_string"}. -{jsx, [{string,<<"this is a naked string">>},end_json]}. -{json, "naked_string.json"}. diff --git a/priv/test_cases/negative_zero.json b/priv/test_cases/negative_zero.json deleted file mode 100644 index 25147e2..0000000 --- a/priv/test_cases/negative_zero.json +++ /dev/null @@ -1 +0,0 @@ -[[-0], -0, {"key":-0, "another key":-0}, -0 ] \ No newline at end of file diff --git a/priv/test_cases/negative_zero.test b/priv/test_cases/negative_zero.test deleted file mode 100644 index 0bc734a..0000000 --- a/priv/test_cases/negative_zero.test +++ /dev/null @@ -1,14 +0,0 @@ -{name, "negative_zero"}. -{jsx, [start_array,start_array, - {integer,0}, - end_array, - {integer,0}, - start_object, - {key,<<"key">>}, - {integer,0}, - {key,<<"another key">>}, - {integer,0}, - end_object, - {integer,0}, - end_array,end_json]}. -{json, "negative_zero.json"}. diff --git a/priv/test_cases/null.json b/priv/test_cases/null.json deleted file mode 100644 index ec747fa..0000000 --- a/priv/test_cases/null.json +++ /dev/null @@ -1 +0,0 @@ -null \ No newline at end of file diff --git a/priv/test_cases/null.test b/priv/test_cases/null.test deleted file mode 100644 index ddb56d5..0000000 --- a/priv/test_cases/null.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "null"}. -{jsx, [{literal, null},end_json]}. -{json, "null.json"}. diff --git a/priv/test_cases/numbers.json b/priv/test_cases/numbers.json deleted file mode 100644 index 17f69b6..0000000 --- a/priv/test_cases/numbers.json +++ /dev/null @@ -1 +0,0 @@ -[ 1, 2, 3, 4, 5, 6, 7, 8, 9, 42, 127, 99999999999999999999999999999, 1e1, 1E1, 1.0e1, 1.325e278, -1, -1e-1, 3.7e-78 ] \ No newline at end of file diff --git a/priv/test_cases/numbers.test b/priv/test_cases/numbers.test deleted file mode 100644 index 1ed16ac..0000000 --- a/priv/test_cases/numbers.test +++ /dev/null @@ -1,23 +0,0 @@ -{name, "numbers"}. -{jsx, [start_array, - {integer,1}, - {integer,2}, - {integer,3}, - {integer,4}, - {integer,5}, - {integer,6}, - {integer,7}, - {integer,8}, - {integer,9}, - {integer,42}, - {integer,127}, - {integer,99999999999999999999999999999}, - {float,1.0e1}, - {float,1.0e1}, - {float,1.0e1}, - {float,1.325e278}, - {integer,-1}, - {float,-1.0e-1}, - {float,3.7e-78}, - end_array,end_json]}. -{json, "numbers.json"}. diff --git a/priv/test_cases/object.json b/priv/test_cases/object.json deleted file mode 100644 index f59c630..0000000 --- a/priv/test_cases/object.json +++ /dev/null @@ -1 +0,0 @@ -{"foo":"bar", "baz":true, "false":null,"object":{ "key" : "value" },"list":[null,null,null,[],"\n\r\\"]} \ No newline at end of file diff --git a/priv/test_cases/object.test b/priv/test_cases/object.test deleted file mode 100644 index 4877fc8..0000000 --- a/priv/test_cases/object.test +++ /dev/null @@ -1,22 +0,0 @@ -{name, "object"}. -{jsx, [start_object, - {key,<<"foo">>}, - {string,<<"bar">>}, - {key,<<"baz">>}, - {literal,true}, - {key,<<"false">>}, - {literal,null}, - {key,<<"object">>}, - start_object, - {key,<<"key">>}, - {string,<<"value">>}, - end_object, - {key,<<"list">>}, - start_array, - {literal,null}, - {literal,null}, - {literal,null}, - start_array,end_array, - {string,<<"\n\r\\">>}, - end_array,end_object,end_json]}. -{json, "object.json"}. diff --git a/priv/test_cases/string.json b/priv/test_cases/string.json deleted file mode 100644 index 743784d..0000000 --- a/priv/test_cases/string.json +++ /dev/null @@ -1 +0,0 @@ -[ "this is a random string with \n embedded\u0020escapes in it" ] \ No newline at end of file diff --git a/priv/test_cases/string.test b/priv/test_cases/string.test deleted file mode 100644 index e1334d3..0000000 --- a/priv/test_cases/string.test +++ /dev/null @@ -1,5 +0,0 @@ -{name, "string"}. -{jsx, [start_array, - {string,<<"this is a random string with \n embedded escapes in it">>}, - end_array,end_json]}. -{json, "string.json"}. diff --git a/priv/test_cases/string_escapes.json b/priv/test_cases/string_escapes.json deleted file mode 100644 index 3c9af78..0000000 --- a/priv/test_cases/string_escapes.json +++ /dev/null @@ -1 +0,0 @@ -["\"", "\\", "\b", "\f", "\n", "\r", "\t"] \ No newline at end of file diff --git a/priv/test_cases/string_escapes.test b/priv/test_cases/string_escapes.test deleted file mode 100644 index 8f6eeed..0000000 --- a/priv/test_cases/string_escapes.test +++ /dev/null @@ -1,11 +0,0 @@ -{name, "string_escapes"}. -{jsx, [start_array, - {string,<<"\"">>}, - {string,<<"\\">>}, - {string,<<"\b">>}, - {string,<<"\f">>}, - {string,<<"\n">>}, - {string,<<"\r">>}, - {string,<<"\t">>}, - end_array,end_json]}. -{json, "string_escapes.json"}. diff --git a/priv/test_cases/true.json b/priv/test_cases/true.json deleted file mode 100644 index f32a580..0000000 --- a/priv/test_cases/true.json +++ /dev/null @@ -1 +0,0 @@ -true \ No newline at end of file diff --git a/priv/test_cases/true.test b/priv/test_cases/true.test deleted file mode 100644 index 4dfeb8c..0000000 --- a/priv/test_cases/true.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "true"}. -{jsx, [{literal, true},end_json]}. -{json, "true.json"}. diff --git a/priv/test_cases/unbalanced_array.json b/priv/test_cases/unbalanced_array.json deleted file mode 100644 index 7f3fb61..0000000 --- a/priv/test_cases/unbalanced_array.json +++ /dev/null @@ -1 +0,0 @@ -[[[[]]] \ No newline at end of file diff --git a/priv/test_cases/unbalanced_array.test b/priv/test_cases/unbalanced_array.test deleted file mode 100644 index 41a2a5e..0000000 --- a/priv/test_cases/unbalanced_array.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "unbalanced array"}. -{jsx, {error, badarg}}. -{json, "unbalanced_array.json"}. diff --git a/priv/test_cases/unicode_to_codepoint.json b/priv/test_cases/unicode_to_codepoint.json deleted file mode 100644 index 7984511..0000000 --- a/priv/test_cases/unicode_to_codepoint.json +++ /dev/null @@ -1 +0,0 @@ -[ "arabic letter alef: ", "\u0627" ] \ No newline at end of file diff --git a/priv/test_cases/unicode_to_codepoint.test b/priv/test_cases/unicode_to_codepoint.test deleted file mode 100644 index 71f47fd..0000000 --- a/priv/test_cases/unicode_to_codepoint.test +++ /dev/null @@ -1,6 +0,0 @@ -{name, "unicode_to_codepoint"}. -{jsx, [start_array, - {string,<<"arabic letter alef: ">>}, - {string,<<1575/utf8>>}, - end_array,end_json]}. -{json, "unicode_to_codepoint.json"}. diff --git a/priv/test_cases/unpaired_surrogate.json b/priv/test_cases/unpaired_surrogate.json deleted file mode 100644 index 32497a8..0000000 --- a/priv/test_cases/unpaired_surrogate.json +++ /dev/null @@ -1 +0,0 @@ -["\ud801blah"] \ No newline at end of file diff --git a/priv/test_cases/unpaired_surrogate.test b/priv/test_cases/unpaired_surrogate.test deleted file mode 100644 index 972a34e..0000000 --- a/priv/test_cases/unpaired_surrogate.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "unpaired_surrogate"}. -{jsx, {error, badarg}}. -{json, "unpaired_surrogate.json"}. diff --git a/priv/test_cases/unpaired_surrogate_replaced.json b/priv/test_cases/unpaired_surrogate_replaced.json deleted file mode 100644 index 32497a8..0000000 --- a/priv/test_cases/unpaired_surrogate_replaced.json +++ /dev/null @@ -1 +0,0 @@ -["\ud801blah"] \ No newline at end of file diff --git a/priv/test_cases/unpaired_surrogate_replaced.test b/priv/test_cases/unpaired_surrogate_replaced.test deleted file mode 100644 index e31290d..0000000 --- a/priv/test_cases/unpaired_surrogate_replaced.test +++ /dev/null @@ -1,4 +0,0 @@ -{name, "unpaired surrogate replaced"}. -{jsx, [start_array,{string,<<65533/utf8,$b,$l,$a,$h>>},end_array,end_json]}. -{json, "unpaired_surrogate_replaced.json"}. -{jsx_flags, [replaced_bad_utf8]}. diff --git a/priv/test_cases/whitespace.json b/priv/test_cases/whitespace.json deleted file mode 100644 index 7d61621..0000000 --- a/priv/test_cases/whitespace.json +++ /dev/null @@ -1,4 +0,0 @@ - - [0.3] - - \ No newline at end of file diff --git a/priv/test_cases/whitespace.test b/priv/test_cases/whitespace.test deleted file mode 100644 index b050125..0000000 --- a/priv/test_cases/whitespace.test +++ /dev/null @@ -1,3 +0,0 @@ -{name, "whitespace"}. -{jsx, [start_array,{float,0.3},end_array,end_json]}. -{json, "whitespace.json"}. diff --git a/priv/test_cases/zero.json b/priv/test_cases/zero.json deleted file mode 100644 index 7009e04..0000000 --- a/priv/test_cases/zero.json +++ /dev/null @@ -1 +0,0 @@ -[[0], 0, {"key":0, "another key":0}, 0 ] \ No newline at end of file diff --git a/priv/test_cases/zero.test b/priv/test_cases/zero.test deleted file mode 100644 index 9515a85..0000000 --- a/priv/test_cases/zero.test +++ /dev/null @@ -1,14 +0,0 @@ -{name, "zero"}. -{jsx, [start_array,start_array, - {integer,0}, - end_array, - {integer,0}, - start_object, - {key,<<"key">>}, - {integer,0}, - {key,<<"another key">>}, - {integer,0}, - end_object, - {integer,0}, - end_array,end_json]}. -{json, "zero.json"}. From ea2b2701ea7f8f454323544d584c50067ab2e3ef Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Wed, 6 Feb 2013 11:17:11 -0800 Subject: [PATCH 16/61] add R15B01, R15B02 and R15B03-1 to travis ci config --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5cc6271..3a3fa22 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,9 @@ language: erlang script: rebar compile && rebar skip_deps=true eunit otp_release: + - R15B03-1 + - R15B02 + - R15B01 - R15B - R14B04 - R14B03 From 34ea2ba491b7e9bdc1b867ee6a36a1cc3a464d57 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Wed, 6 Feb 2013 11:25:00 -0800 Subject: [PATCH 17/61] remove r15b03-1 from travis ci config --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3a3fa22..ea2c055 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: erlang script: rebar compile && rebar skip_deps=true eunit otp_release: - - R15B03-1 - R15B02 - R15B01 - R15B From 51cafcfb640ab9a2a78219a46370c1279c607c72 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Wed, 6 Feb 2013 22:27:52 -0800 Subject: [PATCH 18/61] reduce test space by eliminating redundancies in extended codepoint parsing --- src/jsx_utils.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index 78bd7bc..b3125d3 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -649,8 +649,8 @@ extended_codepoints() -> lists:seq(16#10000, 16#1fffd) ++ [ 16#20000, 16#30000, 16#40000, 16#50000, 16#60000, 16#70000, 16#80000, 16#90000, 16#a0000, 16#b0000, - 16#c0000, 16#d0000, 16#e0000, 16#f0000 - ] ++ lists:seq(16#100000, 16#10fffd) + 16#c0000, 16#d0000, 16#e0000, 16#f0000, 16#100000 + ] ). noncharacters() -> [ to_fake_utf8(N) || N <- lists:seq(16#fffe, 16#ffff) ]. From ba42f7578066213cbad8dc8cd7112137964a9926 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Wed, 6 Feb 2013 22:28:26 -0800 Subject: [PATCH 19/61] first tentative steps towards more comprehensive and flexible test suite --- src/jsx.erl | 20 ++------------------ src/jsx_tests.hrl | 17 +++++++++++++++++ src/jsx_verify.erl | 12 ++++++++++++ 3 files changed, 31 insertions(+), 18 deletions(-) create mode 100644 src/jsx_tests.hrl diff --git a/src/jsx.erl b/src/jsx.erl index 2534540..bccf7c6 100644 --- a/src/jsx.erl +++ b/src/jsx.erl @@ -34,9 +34,8 @@ -export_type([json_term/0, json_text/0]). -%% test handler -ifdef(TEST). --export([init/1, handle_event/2]). +-include("jsx_tests.hrl"). -endif. @@ -151,19 +150,4 @@ encoder(Handler, State, Opts) -> jsx_encoder:encoder(Handler, State, Opts). -spec parser(Handler::module(), State::any(), Opts::list()) -> parser(). -parser(Handler, State, Opts) -> jsx_parser:parser(Handler, State, Opts). - - - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). - - -%% stub test handler -init([]) -> []. - -handle_event(end_json, State) -> lists:reverse([end_json] ++ State); -handle_event(Event, State) -> [Event] ++ State. - - --endif. +parser(Handler, State, Opts) -> jsx_parser:parser(Handler, State, Opts). \ No newline at end of file diff --git a/src/jsx_tests.hrl b/src/jsx_tests.hrl new file mode 100644 index 0000000..24fc814 --- /dev/null +++ b/src/jsx_tests.hrl @@ -0,0 +1,17 @@ +%% data and helper functions for tests + +-export([init/1, handle_event/2]). +-export([empty_array/0, empty_object/0]). + +-include_lib("eunit/include/eunit.hrl"). + + +%% test handler +init([]) -> []. + +handle_event(end_json, State) -> lists:reverse([end_json] ++ State); +handle_event(Event, State) -> [Event] ++ State. + + +empty_array() -> [{"empty array", <<"[]">>, [], [start_array, end_array, end_json]}]. +empty_object() -> [{"empty object", <<"{}">>, [{}], [start_object, end_object, end_json]}]. \ No newline at end of file diff --git a/src/jsx_verify.erl b/src/jsx_verify.erl index 93388b1..b9c2959 100644 --- a/src/jsx_verify.erl +++ b/src/jsx_verify.erl @@ -114,4 +114,16 @@ opts_test_() -> ]. +valid_json_test_() -> + Data = jsx:empty_array() + ++ jsx:empty_object(), + [ {Title, ?_assertEqual(true, is_json(JSON, []))} || {Title, JSON, _, _} <- Data ]. + + +valid_term_test_() -> + Data = jsx:empty_array() + ++ jsx:empty_object(), + [ {Title, ?_assertEqual(true, is_term(Term, []))} || {Title, _, Term, _} <- Data ]. + + -endif. \ No newline at end of file From 6ca54ae3d72bd2785b9687484af5b51fef9e2198 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Thu, 7 Feb 2013 06:44:36 -0800 Subject: [PATCH 20/61] pad out test tests --- src/jsx_tests.hrl | 68 +++++++++++++++++++++++++++++++++++++++++++++- src/jsx_verify.erl | 12 ++++++-- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/src/jsx_tests.hrl b/src/jsx_tests.hrl index 24fc814..3a90765 100644 --- a/src/jsx_tests.hrl +++ b/src/jsx_tests.hrl @@ -2,6 +2,9 @@ -export([init/1, handle_event/2]). -export([empty_array/0, empty_object/0]). +-export([literals/0, naked_literals/0]). +-export([integers/0, naked_integers/0]). + -include_lib("eunit/include/eunit.hrl"). @@ -14,4 +17,67 @@ handle_event(Event, State) -> [Event] ++ State. empty_array() -> [{"empty array", <<"[]">>, [], [start_array, end_array, end_json]}]. -empty_object() -> [{"empty object", <<"{}">>, [{}], [start_object, end_object, end_json]}]. \ No newline at end of file +empty_object() -> [{"empty object", <<"{}">>, [{}], [start_object, end_object, end_json]}]. + + +naked_integers() -> + Raw = [ + 1, 2, 3, + 127, 128, 129, + 255, 256, 257, + 65534, 65535, 65536, + 18446744073709551616, + 18446744073709551617 + ], + [ + { + integer_to_list(X), + list_to_binary(integer_to_list(X)), + X, + [{integer, X}, end_json] + } + || X <- Raw ++ [ -1 * Y || Y <- Raw ] ++ [0] + ] ++ [{"-0", <<"-0">>, 0, [{integer, 0}, end_json]}]. + +integers() -> + [ wrap_with_array(Test) || Test <- naked_integers() ] + ++ [ wrap_with_object(Test) || Test <- naked_integers() ]. + + +naked_literals() -> + [ + { + atom_to_list(Literal), + atom_to_binary(Literal, unicode), + Literal, + [{literal, Literal}, end_json] + } + || Literal <- [true, false, null] + ]. + +literals() -> + [ wrap_with_array(Test) || Test <- naked_literals() ] + ++ [ wrap_with_object(Test) || Test <- naked_literals() ]. + + +wrap_with_array({Title, JSON, Term, Events}) -> + { + "[" ++ Title ++ "]", + <<"[", JSON/binary, "]">>, + [Term], + [start_array, strip_end(Events), end_array, end_json] + }. + + +wrap_with_object({Title, JSON, Term, Events}) -> + { + "{\"key\":" ++ Title ++ "}", + <<"{\"key\":", JSON/binary, "}">>, + [{<<"key">>, Term}], + [start_object, {key, <<"key">>}, strip_end(Events), end_object, end_json] + }. + + +strip_end(Events) -> + [end_json|Rest] = lists:reverse(Events), + lists:reverse(Rest). \ No newline at end of file diff --git a/src/jsx_verify.erl b/src/jsx_verify.erl index b9c2959..a38c73f 100644 --- a/src/jsx_verify.erl +++ b/src/jsx_verify.erl @@ -116,13 +116,21 @@ opts_test_() -> valid_json_test_() -> Data = jsx:empty_array() - ++ jsx:empty_object(), + ++ jsx:empty_object() + ++ jsx:literals() + ++ jsx:naked_literals() + ++ jsx:integers() + ++ jsx:naked_integers(), [ {Title, ?_assertEqual(true, is_json(JSON, []))} || {Title, JSON, _, _} <- Data ]. valid_term_test_() -> Data = jsx:empty_array() - ++ jsx:empty_object(), + ++ jsx:empty_object() + ++ jsx:literals() + ++ jsx:naked_literals() + ++ jsx:integers() + ++ jsx:naked_integers(), [ {Title, ?_assertEqual(true, is_term(Term, []))} || {Title, _, Term, _} <- Data ]. From d0535dfe1c27afff8976178ceff6f97281187bf5 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Thu, 7 Feb 2013 13:30:20 -0800 Subject: [PATCH 21/61] tweak test representation to remove 'end_json' from tests and insert in handlers to aid composability --- src/jsx_tests.hrl | 21 ++++++++------------- src/jsx_verify.erl | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/jsx_tests.hrl b/src/jsx_tests.hrl index 3a90765..7114b58 100644 --- a/src/jsx_tests.hrl +++ b/src/jsx_tests.hrl @@ -16,8 +16,8 @@ handle_event(end_json, State) -> lists:reverse([end_json] ++ State); handle_event(Event, State) -> [Event] ++ State. -empty_array() -> [{"empty array", <<"[]">>, [], [start_array, end_array, end_json]}]. -empty_object() -> [{"empty object", <<"{}">>, [{}], [start_object, end_object, end_json]}]. +empty_array() -> [{"empty array", <<"[]">>, [], [start_array, end_array]}]. +empty_object() -> [{"empty object", <<"{}">>, [{}], [start_object, end_object]}]. naked_integers() -> @@ -34,10 +34,10 @@ naked_integers() -> integer_to_list(X), list_to_binary(integer_to_list(X)), X, - [{integer, X}, end_json] + [{integer, X}] } || X <- Raw ++ [ -1 * Y || Y <- Raw ] ++ [0] - ] ++ [{"-0", <<"-0">>, 0, [{integer, 0}, end_json]}]. + ] ++ [{"-0", <<"-0">>, 0, [{integer, 0}]}]. integers() -> [ wrap_with_array(Test) || Test <- naked_integers() ] @@ -50,7 +50,7 @@ naked_literals() -> atom_to_list(Literal), atom_to_binary(Literal, unicode), Literal, - [{literal, Literal}, end_json] + [{literal, Literal}] } || Literal <- [true, false, null] ]. @@ -65,7 +65,7 @@ wrap_with_array({Title, JSON, Term, Events}) -> "[" ++ Title ++ "]", <<"[", JSON/binary, "]">>, [Term], - [start_array, strip_end(Events), end_array, end_json] + [start_array, Events, end_array] }. @@ -74,10 +74,5 @@ wrap_with_object({Title, JSON, Term, Events}) -> "{\"key\":" ++ Title ++ "}", <<"{\"key\":", JSON/binary, "}">>, [{<<"key">>, Term}], - [start_object, {key, <<"key">>}, strip_end(Events), end_object, end_json] - }. - - -strip_end(Events) -> - [end_json|Rest] = lists:reverse(Events), - lists:reverse(Rest). \ No newline at end of file + [start_object, {key, <<"key">>}, Events, end_object] + }. \ No newline at end of file diff --git a/src/jsx_verify.erl b/src/jsx_verify.erl index a38c73f..eec1519 100644 --- a/src/jsx_verify.erl +++ b/src/jsx_verify.erl @@ -134,4 +134,21 @@ valid_term_test_() -> [ {Title, ?_assertEqual(true, is_term(Term, []))} || {Title, _, Term, _} <- Data ]. +handle_event_test_() -> + Data = jsx:empty_array() + ++ jsx:empty_object() + ++ jsx:literals() + ++ jsx:naked_literals() + ++ jsx:integers() + ++ jsx:naked_integers(), + [ + { + Title, ?_assertEqual( + true, + lists:foldl(fun handle_event/2, {#opts{}, []}, Events ++ [end_json]) + ) + } || {Title, _, _, Events} <- Data + ]. + + -endif. \ No newline at end of file From 7855fef0f3804164018671628f1531eca25c2239 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Thu, 7 Feb 2013 20:03:12 -0800 Subject: [PATCH 22/61] consistent test data naming --- src/jsx_tests.hrl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jsx_tests.hrl b/src/jsx_tests.hrl index 7114b58..db39a9a 100644 --- a/src/jsx_tests.hrl +++ b/src/jsx_tests.hrl @@ -16,8 +16,8 @@ handle_event(end_json, State) -> lists:reverse([end_json] ++ State); handle_event(Event, State) -> [Event] ++ State. -empty_array() -> [{"empty array", <<"[]">>, [], [start_array, end_array]}]. -empty_object() -> [{"empty object", <<"{}">>, [{}], [start_object, end_object]}]. +empty_array() -> [{"[]", <<"[]">>, [], [start_array, end_array]}]. +empty_object() -> [{"{}", <<"{}">>, [{}], [start_object, end_object]}]. naked_integers() -> From 18f36ddb0a7daeefe404e4af2f4553be8f96fe3c Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Fri, 8 Feb 2013 10:43:52 -0800 Subject: [PATCH 23/61] more test data generators --- src/jsx_tests.hrl | 22 ++++++++++++++++++---- src/jsx_verify.erl | 22 ++-------------------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/src/jsx_tests.hrl b/src/jsx_tests.hrl index db39a9a..2b92db8 100644 --- a/src/jsx_tests.hrl +++ b/src/jsx_tests.hrl @@ -1,7 +1,8 @@ %% data and helper functions for tests -export([init/1, handle_event/2]). --export([empty_array/0, empty_object/0]). +-export([empty_array/0, deep_array/0, really_deep_array/0]). +-export([empty_object/0]). -export([literals/0, naked_literals/0]). -export([integers/0, naked_integers/0]). @@ -17,6 +18,15 @@ handle_event(Event, State) -> [Event] ++ State. empty_array() -> [{"[]", <<"[]">>, [], [start_array, end_array]}]. + +deep_array() -> + [Object] = empty_array(), + [repeat(fun wrap_with_array/1, Object, 10)]. + +really_deep_array() -> + [Object] = empty_array(), + [repeat(fun wrap_with_array/1, Object, 1000)]. + empty_object() -> [{"{}", <<"{}">>, [{}], [start_object, end_object]}]. @@ -65,7 +75,7 @@ wrap_with_array({Title, JSON, Term, Events}) -> "[" ++ Title ++ "]", <<"[", JSON/binary, "]">>, [Term], - [start_array, Events, end_array] + [start_array] ++ Events ++ [end_array] }. @@ -74,5 +84,9 @@ wrap_with_object({Title, JSON, Term, Events}) -> "{\"key\":" ++ Title ++ "}", <<"{\"key\":", JSON/binary, "}">>, [{<<"key">>, Term}], - [start_object, {key, <<"key">>}, Events, end_object] - }. \ No newline at end of file + [start_object, {key, <<"key">>}] ++ Events ++ [end_object] + }. + + +repeat(_, Object, 0) -> Object; +repeat(Fun, Object, Times) -> repeat(Fun, Fun(Object), Times - 1). \ No newline at end of file diff --git a/src/jsx_verify.erl b/src/jsx_verify.erl index eec1519..590e148 100644 --- a/src/jsx_verify.erl +++ b/src/jsx_verify.erl @@ -114,28 +114,10 @@ opts_test_() -> ]. -valid_json_test_() -> - Data = jsx:empty_array() - ++ jsx:empty_object() - ++ jsx:literals() - ++ jsx:naked_literals() - ++ jsx:integers() - ++ jsx:naked_integers(), - [ {Title, ?_assertEqual(true, is_json(JSON, []))} || {Title, JSON, _, _} <- Data ]. - - -valid_term_test_() -> - Data = jsx:empty_array() - ++ jsx:empty_object() - ++ jsx:literals() - ++ jsx:naked_literals() - ++ jsx:integers() - ++ jsx:naked_integers(), - [ {Title, ?_assertEqual(true, is_term(Term, []))} || {Title, _, Term, _} <- Data ]. - - handle_event_test_() -> Data = jsx:empty_array() + ++ jsx:deep_array() + ++ jsx:really_deep_array() ++ jsx:empty_object() ++ jsx:literals() ++ jsx:naked_literals() From d6a7d85cbb1945eaa9c97ae62f530c4e4baa981c Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Fri, 8 Feb 2013 11:08:58 -0800 Subject: [PATCH 24/61] listify method to compose tests into flat array --- src/jsx_tests.hrl | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/jsx_tests.hrl b/src/jsx_tests.hrl index 2b92db8..776aff4 100644 --- a/src/jsx_tests.hrl +++ b/src/jsx_tests.hrl @@ -20,12 +20,12 @@ handle_event(Event, State) -> [Event] ++ State. empty_array() -> [{"[]", <<"[]">>, [], [start_array, end_array]}]. deep_array() -> - [Object] = empty_array(), - [repeat(fun wrap_with_array/1, Object, 10)]. + [Test] = empty_array(), + [repeat(fun wrap_with_array/1, Test, 10)]. really_deep_array() -> - [Object] = empty_array(), - [repeat(fun wrap_with_array/1, Object, 1000)]. + [Test] = empty_array(), + [repeat(fun wrap_with_array/1, Test, 1000)]. empty_object() -> [{"{}", <<"{}">>, [{}], [start_object, end_object]}]. @@ -51,7 +51,8 @@ naked_integers() -> integers() -> [ wrap_with_array(Test) || Test <- naked_integers() ] - ++ [ wrap_with_object(Test) || Test <- naked_integers() ]. + ++ [ wrap_with_object(Test) || Test <- naked_integers() ] + ++ [listify("array of integers", naked_integers())]. naked_literals() -> @@ -88,5 +89,15 @@ wrap_with_object({Title, JSON, Term, Events}) -> }. -repeat(_, Object, 0) -> Object; -repeat(Fun, Object, Times) -> repeat(Fun, Fun(Object), Times - 1). \ No newline at end of file +repeat(_, Test, 0) -> Test; +repeat(Fun, Test, Times) -> repeat(Fun, Fun(Test), Times - 1). + + +listify(Title, Tests) -> listify(Title, Tests, Acc). + +listify(Title, [], {_, JSON, Term, Events}) -> + {Title, <<"["/utf8, JSON/binary, "]"/utf8>>, Term, Events}; +listify(Title, [Test|Rest], Acc) -> + {_, A, M, X} = Test, + {_, B, N, Y} = Acc, + {Title, <>, M ++ N, X ++ Y}. From bef362ac0ebbbf35ded6ec7671d8b75fdbac2f5a Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Fri, 8 Feb 2013 21:44:15 -0800 Subject: [PATCH 25/61] include literals in compound list test --- src/jsx_tests.hrl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/jsx_tests.hrl b/src/jsx_tests.hrl index 776aff4..715ce47 100644 --- a/src/jsx_tests.hrl +++ b/src/jsx_tests.hrl @@ -68,7 +68,8 @@ naked_literals() -> literals() -> [ wrap_with_array(Test) || Test <- naked_literals() ] - ++ [ wrap_with_object(Test) || Test <- naked_literals() ]. + ++ [ wrap_with_object(Test) || Test <- naked_literals() ] + ++ [listify("array of literals", naked_literals())]. wrap_with_array({Title, JSON, Term, Events}) -> @@ -93,11 +94,11 @@ repeat(_, Test, 0) -> Test; repeat(Fun, Test, Times) -> repeat(Fun, Fun(Test), Times - 1). -listify(Title, Tests) -> listify(Title, Tests, Acc). +listify(Title, [{_, JSON, Term, Events}|Rest]) -> listify(Title, Rest, {Title, JSON, [Term], Events}). listify(Title, [], {_, JSON, Term, Events}) -> {Title, <<"["/utf8, JSON/binary, "]"/utf8>>, Term, Events}; listify(Title, [Test|Rest], Acc) -> - {_, A, M, X} = Test, - {_, B, N, Y} = Acc, - {Title, <>, M ++ N, X ++ Y}. + {_, A, M, X} = Acc, + {_, B, N, Y} = Test, + listify(Title, Rest, {Title, <>, M ++ [N], X ++ Y}). From 2ebffe0616ad0b0ebc922c63658bc670f962fb18 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sat, 9 Feb 2013 15:49:56 -0800 Subject: [PATCH 26/61] add floats to tests --- src/jsx_tests.hrl | 32 ++++++++++++++++++++++++++++++++ src/jsx_verify.erl | 4 +++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/jsx_tests.hrl b/src/jsx_tests.hrl index 715ce47..3529eb3 100644 --- a/src/jsx_tests.hrl +++ b/src/jsx_tests.hrl @@ -5,6 +5,7 @@ -export([empty_object/0]). -export([literals/0, naked_literals/0]). -export([integers/0, naked_integers/0]). +-export([floats/0, naked_floats/0]). -include_lib("eunit/include/eunit.hrl"). @@ -27,6 +28,7 @@ really_deep_array() -> [Test] = empty_array(), [repeat(fun wrap_with_array/1, Test, 1000)]. + empty_object() -> [{"{}", <<"{}">>, [{}], [start_object, end_object]}]. @@ -55,6 +57,36 @@ integers() -> ++ [listify("array of integers", naked_integers())]. +naked_floats() -> + Raw = [ + 0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, + 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, + 1234567890.0987654321, + 2.2250738585072014e-308, %% min normalized float + 1.7976931348623157e308, %% max normalized float + 5.0e-324, %% min denormalized float + 2.225073858507201e-308 %% max denormalized float + ], + [ + { + sane_float_to_list(X), + list_to_binary(sane_float_to_list(X)), + X, + [{float, X}] + } + || X <- Raw ++ [ -1 * Y || Y <- Raw ] + ] ++ [{"-0.0", <<"-0.0">>, 0.0, [{float, 0.0}]}]. + +floats() -> + [ wrap_with_array(Test) || Test <- naked_floats() ] + ++ [ wrap_with_object(Test) || Test <- naked_floats() ] + ++ [listify("array of floats", naked_floats())]. + +sane_float_to_list(X) -> + [Output] = io_lib:format("~p", [X]), + Output. + + naked_literals() -> [ { diff --git a/src/jsx_verify.erl b/src/jsx_verify.erl index 590e148..a8ba58b 100644 --- a/src/jsx_verify.erl +++ b/src/jsx_verify.erl @@ -122,7 +122,9 @@ handle_event_test_() -> ++ jsx:literals() ++ jsx:naked_literals() ++ jsx:integers() - ++ jsx:naked_integers(), + ++ jsx:naked_integers() + ++ jsx:floats() + ++ jsx:naked_floats(), [ { Title, ?_assertEqual( From a584c74fbc27d1d277f28423c39fd402f1d7d9b0 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sat, 9 Feb 2013 15:50:52 -0800 Subject: [PATCH 27/61] add formatting tests to jsx_to_json --- src/jsx_to_json.erl | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/jsx_to_json.erl b/src/jsx_to_json.erl index bea2088..44a2584 100644 --- a/src/jsx_to_json.erl +++ b/src/jsx_to_json.erl @@ -252,4 +252,43 @@ indent_or_space_test_() -> ]. +format_test_() -> + [ + {"0.0", ?_assert(encode(float, 0.0, #opts{}) =:= "0.0")}, + {"1.0", ?_assert(encode(float, 1.0, #opts{}) =:= "1.0")}, + {"-1.0", ?_assert(encode(float, -1.0, #opts{}) =:= "-1.0")}, + {"3.1234567890987654321", + ?_assert( + encode(float, 3.1234567890987654321, #opts{}) =:= "3.1234567890987655") + }, + {"1.0e23", ?_assert(encode(float, 1.0e23, #opts{}) =:= "1.0e23")}, + {"0.3", ?_assert(encode(float, 3.0/10.0, #opts{}) =:= "0.3")}, + {"0.0001", ?_assert(encode(float, 0.0001, #opts{}) =:= "0.0001")}, + {"0.00001", ?_assert(encode(float, 0.00001, #opts{}) =:= "1.0e-5")}, + {"0.00000001", ?_assert(encode(float, 0.00000001, #opts{}) =:= "1.0e-8")}, + {"1.0e-323", ?_assert(encode(float, 1.0e-323, #opts{}) =:= "1.0e-323")}, + {"1.0e308", ?_assert(encode(float, 1.0e308, #opts{}) =:= "1.0e308")}, + {"min normalized float", + ?_assert( + encode(float, math:pow(2, -1022), #opts{}) =:= "2.2250738585072014e-308" + ) + }, + {"max normalized float", + ?_assert( + encode(float, (2 - math:pow(2, -52)) * math:pow(2, 1023), #opts{}) + =:= "1.7976931348623157e308" + ) + }, + {"min denormalized float", + ?_assert(encode(float, math:pow(2, -1074), #opts{}) =:= "5.0e-324") + }, + {"max denormalized float", + ?_assert( + encode(float, (1 - math:pow(2, -52)) * math:pow(2, -1022), #opts{}) + =:= "2.225073858507201e-308" + ) + } + ]. + + -endif. \ No newline at end of file From a883db30ceed9f0c5aa5660a71f4fef28d268d92 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sat, 9 Feb 2013 15:58:53 -0800 Subject: [PATCH 28/61] add exp to floats --- src/jsx_tests.hrl | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/jsx_tests.hrl b/src/jsx_tests.hrl index 3529eb3..c7e26fb 100644 --- a/src/jsx_tests.hrl +++ b/src/jsx_tests.hrl @@ -53,8 +53,8 @@ naked_integers() -> integers() -> [ wrap_with_array(Test) || Test <- naked_integers() ] - ++ [ wrap_with_object(Test) || Test <- naked_integers() ] - ++ [listify("array of integers", naked_integers())]. + ++ [ wrap_with_object(Test) || Test <- naked_integers() ] + ++ [listify("array of integers", naked_integers())]. naked_floats() -> @@ -62,6 +62,10 @@ naked_floats() -> 0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 1234567890.0987654321, + 0.0e0, + 1234567890.0987654321e16, + 0.1e0, 0.1e1, 0.1e2, 0.1e4, 0.1e8, 0.1e16, 0.1e308, + 1.0e0, 1.0e1, 1.0e2, 1.0e4, 1.0e8, 1.0e16, 1.0e308, 2.2250738585072014e-308, %% min normalized float 1.7976931348623157e308, %% max normalized float 5.0e-324, %% min denormalized float @@ -75,12 +79,17 @@ naked_floats() -> [{float, X}] } || X <- Raw ++ [ -1 * Y || Y <- Raw ] - ] ++ [{"-0.0", <<"-0.0">>, 0.0, [{float, 0.0}]}]. + ] + ++ [{"-0.0", <<"-0.0">>, 0.0, [{float, 0.0}]}] + ++ [{"1e0", <<"1e0">>, 1.0, [{float, 1.0}]}] + ++ [{"0e0", <<"0e0">>, 0.0, [{float, 0.0}]}] + ++ [{"-1e0", <<"-1e0">>, -1.0, [{float, -1.0}]}]. + floats() -> [ wrap_with_array(Test) || Test <- naked_floats() ] - ++ [ wrap_with_object(Test) || Test <- naked_floats() ] - ++ [listify("array of floats", naked_floats())]. + ++ [ wrap_with_object(Test) || Test <- naked_floats() ] + ++ [listify("array of floats", naked_floats())]. sane_float_to_list(X) -> [Output] = io_lib:format("~p", [X]), @@ -100,8 +109,8 @@ naked_literals() -> literals() -> [ wrap_with_array(Test) || Test <- naked_literals() ] - ++ [ wrap_with_object(Test) || Test <- naked_literals() ] - ++ [listify("array of literals", naked_literals())]. + ++ [ wrap_with_object(Test) || Test <- naked_literals() ] + ++ [listify("array of literals", naked_literals())]. wrap_with_array({Title, JSON, Term, Events}) -> From eaae88f072f81b3ff244181e3d0a3845314fc644 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sat, 9 Feb 2013 16:32:17 -0800 Subject: [PATCH 29/61] add additional exp tests to float --- src/jsx_tests.hrl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/jsx_tests.hrl b/src/jsx_tests.hrl index c7e26fb..89860ea 100644 --- a/src/jsx_tests.hrl +++ b/src/jsx_tests.hrl @@ -83,6 +83,8 @@ naked_floats() -> ++ [{"-0.0", <<"-0.0">>, 0.0, [{float, 0.0}]}] ++ [{"1e0", <<"1e0">>, 1.0, [{float, 1.0}]}] ++ [{"0e0", <<"0e0">>, 0.0, [{float, 0.0}]}] + ++ [{"1e4", <<"1e4">>, 1.0e4, [{float, 1.0e4}]}] + ++ [{"0e4", <<"0e4">>, 0.0, [{float, 0.0}]}] ++ [{"-1e0", <<"-1e0">>, -1.0, [{float, -1.0}]}]. From 086803cc9550c6c03ae4a6b51391542431465a59 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sat, 9 Feb 2013 16:57:47 -0800 Subject: [PATCH 30/61] fix listify and add objectify --- src/jsx_tests.hrl | 48 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/src/jsx_tests.hrl b/src/jsx_tests.hrl index 89860ea..a560824 100644 --- a/src/jsx_tests.hrl +++ b/src/jsx_tests.hrl @@ -54,7 +54,8 @@ naked_integers() -> integers() -> [ wrap_with_array(Test) || Test <- naked_integers() ] ++ [ wrap_with_object(Test) || Test <- naked_integers() ] - ++ [listify("array of integers", naked_integers())]. + ++ [listify(naked_integers())] + ++ [objectify(naked_integers())]. naked_floats() -> @@ -91,7 +92,8 @@ naked_floats() -> floats() -> [ wrap_with_array(Test) || Test <- naked_floats() ] ++ [ wrap_with_object(Test) || Test <- naked_floats() ] - ++ [listify("array of floats", naked_floats())]. + ++ [listify(naked_floats())] + ++ [objectify(naked_floats())]. sane_float_to_list(X) -> [Output] = io_lib:format("~p", [X]), @@ -112,7 +114,8 @@ naked_literals() -> literals() -> [ wrap_with_array(Test) || Test <- naked_literals() ] ++ [ wrap_with_object(Test) || Test <- naked_literals() ] - ++ [listify("array of literals", naked_literals())]. + ++ [listify(naked_literals())] + ++ [objectify(naked_literals())]. wrap_with_array({Title, JSON, Term, Events}) -> @@ -137,11 +140,40 @@ repeat(_, Test, 0) -> Test; repeat(Fun, Test, Times) -> repeat(Fun, Fun(Test), Times - 1). -listify(Title, [{_, JSON, Term, Events}|Rest]) -> listify(Title, Rest, {Title, JSON, [Term], Events}). +listify([{_, JSON, Term, Events}|Rest]) -> listify(Rest, {null, JSON, [Term], Events}). -listify(Title, [], {_, JSON, Term, Events}) -> - {Title, <<"["/utf8, JSON/binary, "]"/utf8>>, Term, Events}; -listify(Title, [Test|Rest], Acc) -> +listify([], {_, JSON, Term, Events}) -> + { + binary_to_list(<<"["/utf8, JSON/binary, "]"/utf8>>), + <<"["/utf8, JSON/binary, "]"/utf8>>, + Term, + [start_array, Events, end_array] + }; +listify([Test|Rest], Acc) -> {_, A, M, X} = Acc, {_, B, N, Y} = Test, - listify(Title, Rest, {Title, <>, M ++ [N], X ++ Y}). + listify(Rest, {null, <>, M ++ [N], X ++ Y}). + + +objectify([{_, JSON, Term, Events}|Rest]) -> + objectify( + Rest, + {null, <<"\"", JSON/binary, "\": ", JSON/binary>>, [{JSON, Term}], [{key, JSON}] ++ Events} + ). + +objectify([], {_, JSON, Term, Events}) -> + { + binary_to_list(<<"{"/utf8, JSON/binary, "}"/utf8>>), + <<"{"/utf8, JSON/binary, "}"/utf8>>, + Term, + [start_object] ++ Events ++ [end_object] + }; +objectify([Test|Rest], Acc) -> + {_, A, M, X} = Acc, + {_, B, N, [Y]} = Test, + objectify(Rest, { + null, + <>, + M ++ [{B, N}], + X ++ [{key, B}, Y] + }). From e6380bc3a38f02fb27d393553e2edb5d7084ad8f Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sun, 10 Feb 2013 19:39:42 -0800 Subject: [PATCH 31/61] break special decodeable forms out to own test data --- src/jsx_tests.hrl | 35 +++++++++++++++++++++++------------ src/jsx_to_json.erl | 21 +++++++++++++++++++++ src/jsx_to_term.erl | 23 +++++++++++++++++++++++ src/jsx_verify.erl | 3 ++- 4 files changed, 69 insertions(+), 13 deletions(-) diff --git a/src/jsx_tests.hrl b/src/jsx_tests.hrl index a560824..2efd265 100644 --- a/src/jsx_tests.hrl +++ b/src/jsx_tests.hrl @@ -6,6 +6,7 @@ -export([literals/0, naked_literals/0]). -export([integers/0, naked_integers/0]). -export([floats/0, naked_floats/0]). +-export([decodeables/0]). -include_lib("eunit/include/eunit.hrl"). @@ -49,7 +50,7 @@ naked_integers() -> [{integer, X}] } || X <- Raw ++ [ -1 * Y || Y <- Raw ] ++ [0] - ] ++ [{"-0", <<"-0">>, 0, [{integer, 0}]}]. + ]. integers() -> [ wrap_with_array(Test) || Test <- naked_integers() ] @@ -80,13 +81,7 @@ naked_floats() -> [{float, X}] } || X <- Raw ++ [ -1 * Y || Y <- Raw ] - ] - ++ [{"-0.0", <<"-0.0">>, 0.0, [{float, 0.0}]}] - ++ [{"1e0", <<"1e0">>, 1.0, [{float, 1.0}]}] - ++ [{"0e0", <<"0e0">>, 0.0, [{float, 0.0}]}] - ++ [{"1e4", <<"1e4">>, 1.0e4, [{float, 1.0e4}]}] - ++ [{"0e4", <<"0e4">>, 0.0, [{float, 0.0}]}] - ++ [{"-1e0", <<"-1e0">>, -1.0, [{float, -1.0}]}]. + ]. floats() -> @@ -95,6 +90,22 @@ floats() -> ++ [listify(naked_floats())] ++ [objectify(naked_floats())]. +decodeables() -> + Tests = [ + {"-0.0", <<"-0.0">>, 0.0, [{float, 0.0}]}, + {"1e0", <<"1e0">>, 1.0, [{float, 1.0}]}, + {"0e0", <<"0e0">>, 0.0, [{float, 0.0}]}, + {"1e4", <<"1e4">>, 1.0e4, [{float, 1.0e4}]}, + {"0e4", <<"0e4">>, 0.0, [{float, 0.0}]}, + {"-1e0", <<"-1e0">>, -1.0, [{float, -1.0}]}, + {"-0", <<"-0">>, 0, [{integer, 0}]} + ], + [ wrap_with_array(Test) || Test <- Tests ] + ++ [ wrap_with_object(Test) || Test <- Tests ] + ++ [listify(Tests)] + ++ [objectify(Tests)]. + + sane_float_to_list(X) -> [Output] = io_lib:format("~p", [X]), Output. @@ -147,18 +158,18 @@ listify([], {_, JSON, Term, Events}) -> binary_to_list(<<"["/utf8, JSON/binary, "]"/utf8>>), <<"["/utf8, JSON/binary, "]"/utf8>>, Term, - [start_array, Events, end_array] + [start_array] ++ Events ++ [end_array] }; listify([Test|Rest], Acc) -> {_, A, M, X} = Acc, {_, B, N, Y} = Test, - listify(Rest, {null, <>, M ++ [N], X ++ Y}). + listify(Rest, {null, <>, M ++ [N], X ++ Y}). objectify([{_, JSON, Term, Events}|Rest]) -> objectify( Rest, - {null, <<"\"", JSON/binary, "\": ", JSON/binary>>, [{JSON, Term}], [{key, JSON}] ++ Events} + {null, <<"\"", JSON/binary, "\":", JSON/binary>>, [{JSON, Term}], [{key, JSON}] ++ Events} ). objectify([], {_, JSON, Term, Events}) -> @@ -173,7 +184,7 @@ objectify([Test|Rest], Acc) -> {_, B, N, [Y]} = Test, objectify(Rest, { null, - <>, + <>, M ++ [{B, N}], X ++ [{key, B}, Y] }). diff --git a/src/jsx_to_json.erl b/src/jsx_to_json.erl index 44a2584..4e3bac9 100644 --- a/src/jsx_to_json.erl +++ b/src/jsx_to_json.erl @@ -291,4 +291,25 @@ format_test_() -> ]. +handle_event_test_() -> + Data = jsx:empty_array() + ++ jsx:deep_array() + ++ jsx:really_deep_array() + ++ jsx:empty_object() + ++ jsx:literals() + ++ jsx:naked_literals() + ++ jsx:integers() + ++ jsx:naked_integers() + ++ jsx:floats() + ++ jsx:naked_floats(), + [ + { + Title, ?_assertEqual( + JSON, + lists:foldl(fun handle_event/2, {start, [], #opts{}}, Events ++ [end_json]) + ) + } || {Title, JSON, _, Events} <- Data + ]. + + -endif. \ No newline at end of file diff --git a/src/jsx_to_term.erl b/src/jsx_to_term.erl index d8fb0bd..007d8dc 100644 --- a/src/jsx_to_term.erl +++ b/src/jsx_to_term.erl @@ -265,4 +265,27 @@ post_decoders_test_() -> )} ]. + +handle_event_test_() -> + Data = jsx:empty_array() + ++ jsx:deep_array() + ++ jsx:really_deep_array() + ++ jsx:empty_object() + ++ jsx:literals() + ++ jsx:naked_literals() + ++ jsx:integers() + ++ jsx:naked_integers() + ++ jsx:floats() + ++ jsx:naked_floats() + ++ jsx:decodeables(), + [ + { + Title, ?_assertEqual( + Term, + lists:foldl(fun handle_event/2, {[[]], #opts{}}, Events ++ [end_json]) + ) + } || {Title, _, Term, Events} <- Data + ]. + + -endif. diff --git a/src/jsx_verify.erl b/src/jsx_verify.erl index a8ba58b..059385a 100644 --- a/src/jsx_verify.erl +++ b/src/jsx_verify.erl @@ -124,7 +124,8 @@ handle_event_test_() -> ++ jsx:integers() ++ jsx:naked_integers() ++ jsx:floats() - ++ jsx:naked_floats(), + ++ jsx:naked_floats() + ++ jsx:decodeables(), [ { Title, ?_assertEqual( From 4a3f2a3fa4054695b976217ef35b83df4cfddb0a Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Mon, 11 Feb 2013 16:36:46 -0800 Subject: [PATCH 32/61] add test for objectify generator --- src/jsx_tests.hrl | 57 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/src/jsx_tests.hrl b/src/jsx_tests.hrl index 2efd265..bcaf3ac 100644 --- a/src/jsx_tests.hrl +++ b/src/jsx_tests.hrl @@ -181,10 +181,63 @@ objectify([], {_, JSON, Term, Events}) -> }; objectify([Test|Rest], Acc) -> {_, A, M, X} = Acc, - {_, B, N, [Y]} = Test, + {_, B, N, Y} = Test, objectify(Rest, { null, <>, M ++ [{B, N}], - X ++ [{key, B}, Y] + X ++ [{key, B}] ++ Y }). + + +listify_test_() -> + {"listify test", ?_assertEqual( + { + "[true,1,\"hello world\",{}]", + <<"[true,1,\"hello world\",{}]">>, + [true, 1, <<"hello world">>, [{}]], + [ + start_array, + {literal, true}, + {integer, 1}, + {string, <<"hello world">>}, + start_object, + end_object, + end_array + ] + }, + listify([ + {"true", <<"true">>, true, [{literal, true}]}, + {"1", <<"1">>, 1, [{integer, 1}]}, + {"hello world", <<"\"hello world\"">>, <<"hello world">>, [{string, <<"hello world">>}]}, + {"{}", <<"{}">>, [{}], [start_object, end_object]} + ]) + )}. + +objectify_test_() -> + {"objectify test", ?_assertEqual( + { + "{\"true\":true,\"1\":1,\"\"hello world\"\":\"hello world\",\"[]\":[]}", + <<"{\"true\":true,\"1\":1,\"\"hello world\"\":\"hello world\",\"[]\":[]}">>, + [{<<"true">>, true}, {<<"1">>, 1}, {<<"\"hello world\"">>, <<"hello world">>}, {<<"[]">>, []}], + [ + start_object, + {key, <<"true">>}, + {literal, true}, + {key, <<"1">>}, + {integer, 1}, + {key, <<"\"hello world\"">>}, + {string, <<"hello world">>}, + {key, <<"[]">>}, + start_array, + end_array, + end_object + ] + }, + objectify([ + {"true", <<"true">>, true, [{literal, true}]}, + {"1", <<"1">>, 1, [{integer, 1}]}, + {"hello world", <<"\"hello world\"">>, <<"hello world">>, [{string, <<"hello world">>}]}, + {"[]", <<"[]">>, [], [start_array, end_array]} + ]) + )}. From 7f3cb6e496ad76cc45820d2b8efee95b58c16e86 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Mon, 11 Feb 2013 17:30:29 -0800 Subject: [PATCH 33/61] tweak objectify and listify test generators --- src/jsx_tests.hrl | 61 ++++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/src/jsx_tests.hrl b/src/jsx_tests.hrl index bcaf3ac..9ea1ae4 100644 --- a/src/jsx_tests.hrl +++ b/src/jsx_tests.hrl @@ -55,8 +55,8 @@ naked_integers() -> integers() -> [ wrap_with_array(Test) || Test <- naked_integers() ] ++ [ wrap_with_object(Test) || Test <- naked_integers() ] - ++ [listify(naked_integers())] - ++ [objectify(naked_integers())]. + ++ [listify("naked integers", naked_integers())] + ++ [objectify("naked integers", naked_integers())]. naked_floats() -> @@ -82,13 +82,13 @@ naked_floats() -> } || X <- Raw ++ [ -1 * Y || Y <- Raw ] ]. - floats() -> [ wrap_with_array(Test) || Test <- naked_floats() ] ++ [ wrap_with_object(Test) || Test <- naked_floats() ] - ++ [listify(naked_floats())] - ++ [objectify(naked_floats())]. + ++ [listify("naked floats", naked_floats())] + ++ [objectify("naked floats", naked_floats())]. + decodeables() -> Tests = [ @@ -102,9 +102,9 @@ decodeables() -> ], [ wrap_with_array(Test) || Test <- Tests ] ++ [ wrap_with_object(Test) || Test <- Tests ] - ++ [listify(Tests)] - ++ [objectify(Tests)]. - + ++ [listify("naked decodeables", Tests)] + ++ [objectify("naked decodeables", Tests)]. + sane_float_to_list(X) -> [Output] = io_lib:format("~p", [X]), @@ -125,8 +125,8 @@ naked_literals() -> literals() -> [ wrap_with_array(Test) || Test <- naked_literals() ] ++ [ wrap_with_object(Test) || Test <- naked_literals() ] - ++ [listify(naked_literals())] - ++ [objectify(naked_literals())]. + ++ [listify("naked literals", naked_literals())] + ++ [objectify("naked literals", naked_literals())]. wrap_with_array({Title, JSON, Term, Events}) -> @@ -151,39 +151,39 @@ repeat(_, Test, 0) -> Test; repeat(Fun, Test, Times) -> repeat(Fun, Fun(Test), Times - 1). -listify([{_, JSON, Term, Events}|Rest]) -> listify(Rest, {null, JSON, [Term], Events}). +listify(Title, [{_, JSON, Term, Events}|Rest]) -> do_listify(Rest, {Title, JSON, [Term], Events}). -listify([], {_, JSON, Term, Events}) -> +do_listify([], {Title, JSON, Term, Events}) -> { - binary_to_list(<<"["/utf8, JSON/binary, "]"/utf8>>), + Title, <<"["/utf8, JSON/binary, "]"/utf8>>, Term, [start_array] ++ Events ++ [end_array] }; -listify([Test|Rest], Acc) -> - {_, A, M, X} = Acc, +do_listify([Test|Rest], Acc) -> + {Title, A, M, X} = Acc, {_, B, N, Y} = Test, - listify(Rest, {null, <>, M ++ [N], X ++ Y}). + do_listify(Rest, {Title, <>, M ++ [N], X ++ Y}). -objectify([{_, JSON, Term, Events}|Rest]) -> - objectify( +objectify(Title, [{_, JSON, Term, Events}|Rest]) -> + do_objectify( Rest, - {null, <<"\"", JSON/binary, "\":", JSON/binary>>, [{JSON, Term}], [{key, JSON}] ++ Events} + {Title, <<"\"", JSON/binary, "\":", JSON/binary>>, [{JSON, Term}], [{key, JSON}] ++ Events} ). -objectify([], {_, JSON, Term, Events}) -> +do_objectify([], {Title, JSON, Term, Events}) -> { - binary_to_list(<<"{"/utf8, JSON/binary, "}"/utf8>>), + Title, <<"{"/utf8, JSON/binary, "}"/utf8>>, Term, [start_object] ++ Events ++ [end_object] }; -objectify([Test|Rest], Acc) -> - {_, A, M, X} = Acc, +do_objectify([Test|Rest], Acc) -> + {Title, A, M, X} = Acc, {_, B, N, Y} = Test, - objectify(Rest, { - null, + do_objectify(Rest, { + Title, <>, M ++ [{B, N}], X ++ [{key, B}] ++ Y @@ -193,7 +193,7 @@ objectify([Test|Rest], Acc) -> listify_test_() -> {"listify test", ?_assertEqual( { - "[true,1,\"hello world\",{}]", + "listify test", <<"[true,1,\"hello world\",{}]">>, [true, 1, <<"hello world">>, [{}]], [ @@ -206,7 +206,7 @@ listify_test_() -> end_array ] }, - listify([ + listify("listify test", [ {"true", <<"true">>, true, [{literal, true}]}, {"1", <<"1">>, 1, [{integer, 1}]}, {"hello world", <<"\"hello world\"">>, <<"hello world">>, [{string, <<"hello world">>}]}, @@ -214,10 +214,11 @@ listify_test_() -> ]) )}. + objectify_test_() -> {"objectify test", ?_assertEqual( { - "{\"true\":true,\"1\":1,\"\"hello world\"\":\"hello world\",\"[]\":[]}", + "objectify test", <<"{\"true\":true,\"1\":1,\"\"hello world\"\":\"hello world\",\"[]\":[]}">>, [{<<"true">>, true}, {<<"1">>, 1}, {<<"\"hello world\"">>, <<"hello world">>}, {<<"[]">>, []}], [ @@ -234,10 +235,10 @@ objectify_test_() -> end_object ] }, - objectify([ + objectify("objectify test", [ {"true", <<"true">>, true, [{literal, true}]}, {"1", <<"1">>, 1, [{integer, 1}]}, {"hello world", <<"\"hello world\"">>, <<"hello world">>, [{string, <<"hello world">>}]}, {"[]", <<"[]">>, [], [start_array, end_array]} ]) - )}. + )}. \ No newline at end of file From eaf19cde9cddc00c3576ae97ba1a2a77e6894e8c Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Mon, 11 Feb 2013 18:22:45 -0800 Subject: [PATCH 34/61] extend tests to decoder, encoder and parser --- src/jsx_decoder.erl | 23 +++++++++++++++++++++++ src/jsx_encoder.erl | 21 +++++++++++++++++++++ src/jsx_parser.erl | 22 ++++++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 8355672..925a316 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -726,6 +726,8 @@ zero(<>, Handler, [Acc, array|Stack], Opts) -> value(Rest, handle_event(format_number(Acc), Handler, Opts), [array|Stack], Opts); zero(<>, Handler, [Acc|Stack], Opts) -> initial_decimal(Rest, Handler, [{Acc, []}|Stack], Opts); +zero(<>, Handler, [Acc|Stack], Opts) when S =:= $e; S =:= $E -> + e(Rest, Handler, [{Acc, [], []}|Stack], Opts); zero(<>, Handler, [Acc|Stack], Opts) when ?is_whitespace(S) -> maybe_done(Rest, handle_event(format_number(Acc), Handler, Opts), Stack, Opts); zero(<>, Handler, [Acc|Stack], Opts=#opts{comments=true}) -> @@ -1610,4 +1612,25 @@ to_fake_utf(N, utf8) -> <<34/utf8, 2#11110:5, W:3, 2#10:2, Z:6, 2#10:2, Y:6, 2#10:2, X:6, 34/utf8>>. +decode_test_() -> + Data = jsx:empty_array() + ++ jsx:deep_array() + ++ jsx:really_deep_array() + ++ jsx:empty_object() + ++ jsx:literals() + ++ jsx:naked_literals() + ++ jsx:integers() + ++ jsx:naked_integers() + ++ jsx:floats() + ++ jsx:naked_floats() + ++ jsx:decodeables(), + [ + { + Title, ?_assertEqual( + Events ++ [end_json], + start(JSON, {jsx, []}, [], #opts{}) + ) + } || {Title, JSON, _, Events} <- Data + ]. + -endif. \ No newline at end of file diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index fa722ae..4944d5b 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -136,4 +136,25 @@ clean_string(Bin, Opts) -> jsx_utils:clean_string(Bin, Opts). -include_lib("eunit/include/eunit.hrl"). +encode_test_() -> + Data = jsx:empty_array() + ++ jsx:deep_array() + ++ jsx:really_deep_array() + ++ jsx:empty_object() + ++ jsx:literals() + ++ jsx:naked_literals() + ++ jsx:integers() + ++ jsx:naked_integers() + ++ jsx:floats() + ++ jsx:naked_floats(), + [ + { + Title, ?_assertEqual( + Events ++ [end_json], + start(Term, {jsx, []}, #opts{}) + ) + } || {Title, _, Term, Events} <- Data + ]. + + -endif. \ No newline at end of file diff --git a/src/jsx_parser.erl b/src/jsx_parser.erl index adf8126..5d0b0b8 100644 --- a/src/jsx_parser.erl +++ b/src/jsx_parser.erl @@ -169,4 +169,26 @@ clean_string(Bin, Opts) -> jsx_utils:clean_string(Bin, Opts). -include_lib("eunit/include/eunit.hrl"). +decode_test_() -> + Data = jsx:empty_array() + ++ jsx:deep_array() + ++ jsx:really_deep_array() + ++ jsx:empty_object() + ++ jsx:literals() + ++ jsx:naked_literals() + ++ jsx:integers() + ++ jsx:naked_integers() + ++ jsx:floats() + ++ jsx:naked_floats() + ++ jsx:decodeables(), + [ + { + Title, ?_assertEqual( + Events ++ [end_json], + value(Events ++ [end_json], {jsx, []}, [], #opts{}) + ) + } || {Title, _, _, Events} <- Data + ]. + + -endif. \ No newline at end of file From 2f47fdd64235a51f95de48d6859727fcbad45f71 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Mon, 11 Feb 2013 19:06:44 -0800 Subject: [PATCH 35/61] simplify exportation of tests, add strings to test suite --- src/jsx.erl | 1 + src/jsx_decoder.erl | 11 +---- src/jsx_encoder.erl | 11 +---- src/jsx_parser.erl | 11 +---- src/jsx_tests.hrl | 102 +++++++++++++++++++++++++++++++------------- src/jsx_to_json.erl | 11 +---- src/jsx_to_term.erl | 11 +---- src/jsx_verify.erl | 11 +---- 8 files changed, 79 insertions(+), 90 deletions(-) diff --git a/src/jsx.erl b/src/jsx.erl index bccf7c6..d953b8a 100644 --- a/src/jsx.erl +++ b/src/jsx.erl @@ -22,6 +22,7 @@ -module(jsx). +-compile(export_all). -export([encode/1, encode/2, decode/1, decode/2]). -export([is_json/1, is_json/2, is_term/1, is_term/2]). diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 925a316..93bac07 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -1613,16 +1613,7 @@ to_fake_utf(N, utf8) -> decode_test_() -> - Data = jsx:empty_array() - ++ jsx:deep_array() - ++ jsx:really_deep_array() - ++ jsx:empty_object() - ++ jsx:literals() - ++ jsx:naked_literals() - ++ jsx:integers() - ++ jsx:naked_integers() - ++ jsx:floats() - ++ jsx:naked_floats() + Data = jsx:universals() ++ jsx:decodeables(), [ { diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 4944d5b..ac2a72d 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -137,16 +137,7 @@ clean_string(Bin, Opts) -> jsx_utils:clean_string(Bin, Opts). encode_test_() -> - Data = jsx:empty_array() - ++ jsx:deep_array() - ++ jsx:really_deep_array() - ++ jsx:empty_object() - ++ jsx:literals() - ++ jsx:naked_literals() - ++ jsx:integers() - ++ jsx:naked_integers() - ++ jsx:floats() - ++ jsx:naked_floats(), + Data = jsx:universals(), [ { Title, ?_assertEqual( diff --git a/src/jsx_parser.erl b/src/jsx_parser.erl index 5d0b0b8..c67eefd 100644 --- a/src/jsx_parser.erl +++ b/src/jsx_parser.erl @@ -170,16 +170,7 @@ clean_string(Bin, Opts) -> jsx_utils:clean_string(Bin, Opts). decode_test_() -> - Data = jsx:empty_array() - ++ jsx:deep_array() - ++ jsx:really_deep_array() - ++ jsx:empty_object() - ++ jsx:literals() - ++ jsx:naked_literals() - ++ jsx:integers() - ++ jsx:naked_integers() - ++ jsx:floats() - ++ jsx:naked_floats() + Data = jsx:universals() ++ jsx:decodeables(), [ { diff --git a/src/jsx_tests.hrl b/src/jsx_tests.hrl index 9ea1ae4..b414a7c 100644 --- a/src/jsx_tests.hrl +++ b/src/jsx_tests.hrl @@ -1,11 +1,7 @@ %% data and helper functions for tests -export([init/1, handle_event/2]). --export([empty_array/0, deep_array/0, really_deep_array/0]). --export([empty_object/0]). --export([literals/0, naked_literals/0]). --export([integers/0, naked_integers/0]). --export([floats/0, naked_floats/0]). +-export([universals/0]). -export([decodeables/0]). @@ -19,6 +15,17 @@ handle_event(end_json, State) -> lists:reverse([end_json] ++ State); handle_event(Event, State) -> [Event] ++ State. +universals() -> + empty_array() + ++ deep_array() + ++ really_deep_array() + ++ empty_object() + ++ literals() + ++ integers() + ++ floats() + ++ strings(). + + empty_array() -> [{"[]", <<"[]">>, [], [start_array, end_array]}]. deep_array() -> @@ -33,6 +40,36 @@ really_deep_array() -> empty_object() -> [{"{}", <<"{}">>, [{}], [start_object, end_object]}]. +naked_strings() -> + Raw = [ + "", + "hello world" + ], + [ + { + String, + <<"\"", (list_to_binary(String))/binary, "\"">>, + list_to_binary(String), + [{string, list_to_binary(String)}] + } + || String <- Raw + ]. + +strings() -> + naked_strings() + ++ [ wrap_with_array(Test) || Test <- naked_strings() ] + ++ [ wrap_with_object(Test) || Test <- naked_strings() ] + ++ [listify("naked strings", naked_strings())] + ++ [ + { + "naked strings", + <<"{\"\":\"\",\"hello world\":\"hello world\"}">>, + [{<<>>, <<>>}, {<<"hello world">>, <<"hello world">>}], + [start_object, {key, <<>>}, {string, <<>>}, {key, <<"hello world">>}, {string, <<"hello world">>}, end_object] + } + ]. + + naked_integers() -> Raw = [ 1, 2, 3, @@ -53,7 +90,8 @@ naked_integers() -> ]. integers() -> - [ wrap_with_array(Test) || Test <- naked_integers() ] + naked_integers() + ++ [ wrap_with_array(Test) || Test <- naked_integers() ] ++ [ wrap_with_object(Test) || Test <- naked_integers() ] ++ [listify("naked integers", naked_integers())] ++ [objectify("naked integers", naked_integers())]. @@ -84,12 +122,34 @@ naked_floats() -> ]. floats() -> - [ wrap_with_array(Test) || Test <- naked_floats() ] + naked_floats() + ++ [ wrap_with_array(Test) || Test <- naked_floats() ] ++ [ wrap_with_object(Test) || Test <- naked_floats() ] ++ [listify("naked floats", naked_floats())] ++ [objectify("naked floats", naked_floats())]. +naked_literals() -> + [ + { + atom_to_list(Literal), + atom_to_binary(Literal, unicode), + Literal, + [{literal, Literal}] + } + || Literal <- [true, false, null] + ]. + +literals() -> + naked_literals() + ++ [ wrap_with_array(Test) || Test <- naked_literals() ] + ++ [ wrap_with_object(Test) || Test <- naked_literals() ] + ++ [listify("naked literals", naked_literals())] + ++ [objectify("naked literals", naked_literals())]. + + +%% special tests used only for things that don't round trip when decoded and re-encoded + decodeables() -> Tests = [ {"-0.0", <<"-0.0">>, 0.0, [{float, 0.0}]}, @@ -106,29 +166,6 @@ decodeables() -> ++ [objectify("naked decodeables", Tests)]. -sane_float_to_list(X) -> - [Output] = io_lib:format("~p", [X]), - Output. - - -naked_literals() -> - [ - { - atom_to_list(Literal), - atom_to_binary(Literal, unicode), - Literal, - [{literal, Literal}] - } - || Literal <- [true, false, null] - ]. - -literals() -> - [ wrap_with_array(Test) || Test <- naked_literals() ] - ++ [ wrap_with_object(Test) || Test <- naked_literals() ] - ++ [listify("naked literals", naked_literals())] - ++ [objectify("naked literals", naked_literals())]. - - wrap_with_array({Title, JSON, Term, Events}) -> { "[" ++ Title ++ "]", @@ -151,6 +188,11 @@ repeat(_, Test, 0) -> Test; repeat(Fun, Test, Times) -> repeat(Fun, Fun(Test), Times - 1). +sane_float_to_list(X) -> + [Output] = io_lib:format("~p", [X]), + Output. + + listify(Title, [{_, JSON, Term, Events}|Rest]) -> do_listify(Rest, {Title, JSON, [Term], Events}). do_listify([], {Title, JSON, Term, Events}) -> diff --git a/src/jsx_to_json.erl b/src/jsx_to_json.erl index 4e3bac9..df4f4f4 100644 --- a/src/jsx_to_json.erl +++ b/src/jsx_to_json.erl @@ -292,16 +292,7 @@ format_test_() -> handle_event_test_() -> - Data = jsx:empty_array() - ++ jsx:deep_array() - ++ jsx:really_deep_array() - ++ jsx:empty_object() - ++ jsx:literals() - ++ jsx:naked_literals() - ++ jsx:integers() - ++ jsx:naked_integers() - ++ jsx:floats() - ++ jsx:naked_floats(), + Data = jsx:universals(), [ { Title, ?_assertEqual( diff --git a/src/jsx_to_term.erl b/src/jsx_to_term.erl index 007d8dc..792b6d6 100644 --- a/src/jsx_to_term.erl +++ b/src/jsx_to_term.erl @@ -267,16 +267,7 @@ post_decoders_test_() -> handle_event_test_() -> - Data = jsx:empty_array() - ++ jsx:deep_array() - ++ jsx:really_deep_array() - ++ jsx:empty_object() - ++ jsx:literals() - ++ jsx:naked_literals() - ++ jsx:integers() - ++ jsx:naked_integers() - ++ jsx:floats() - ++ jsx:naked_floats() + Data = jsx:universals() ++ jsx:decodeables(), [ { diff --git a/src/jsx_verify.erl b/src/jsx_verify.erl index 059385a..304807e 100644 --- a/src/jsx_verify.erl +++ b/src/jsx_verify.erl @@ -115,16 +115,7 @@ opts_test_() -> handle_event_test_() -> - Data = jsx:empty_array() - ++ jsx:deep_array() - ++ jsx:really_deep_array() - ++ jsx:empty_object() - ++ jsx:literals() - ++ jsx:naked_literals() - ++ jsx:integers() - ++ jsx:naked_integers() - ++ jsx:floats() - ++ jsx:naked_floats() + Data = jsx:universals() ++ jsx:decodeables(), [ { From ff6a83598b8f14d4cdd4097f6f5af00a3f790551 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 12 Feb 2013 11:54:42 -0800 Subject: [PATCH 36/61] replace all incidences of opts with config --- src/jsx.erl | 41 +- src/{jsx_opts.hrl => jsx_config.hrl} | 2 +- src/jsx_decoder.erl | 1394 +++++++++++++------------- src/jsx_encoder.erl | 90 +- src/jsx_parser.erl | 194 ++-- src/jsx_to_json.erl | 212 ++-- src/jsx_to_term.erl | 134 +-- src/jsx_utils.erl | 732 +++++++------- src/jsx_verify.erl | 74 +- 9 files changed, 1436 insertions(+), 1437 deletions(-) rename src/{jsx_opts.hrl => jsx_config.hrl} (93%) diff --git a/src/jsx.erl b/src/jsx.erl index d953b8a..ec6cea4 100644 --- a/src/jsx.erl +++ b/src/jsx.erl @@ -22,7 +22,6 @@ -module(jsx). --compile(export_all). -export([encode/1, encode/2, decode/1, decode/2]). -export([is_json/1, is_json/2, is_term/1, is_term/2]). @@ -53,24 +52,24 @@ -spec encode(Source::json_term()) -> json_text() | {incomplete, encoder()}. --spec encode(Source::json_term(), Opts::jsx_to_json:opts()) -> json_text() | {incomplete, encoder()}. +-spec encode(Source::json_term(), Config::jsx_to_json:config()) -> json_text() | {incomplete, encoder()}. encode(Source) -> encode(Source, []). -encode(Source, Opts) -> jsx_to_json:to_json(Source, Opts). +encode(Source, Config) -> jsx_to_json:to_json(Source, Config). %% old api, alias for encode/x to_json(Source) -> encode(Source, []). -to_json(Source, Opts) -> encode(Source, Opts). +to_json(Source, Config) -> encode(Source, Config). term_to_json(Source) -> encode(Source, []). -term_to_json(Source, Opts) -> encode(Source, Opts). +term_to_json(Source, Config) -> encode(Source, Config). -spec format(Source::json_text()) -> json_text() | {incomplete, decoder()}. --spec format(Source::json_text(), Opts::jsx_to_json:opts()) -> json_text() | {incomplete, decoder()}. +-spec format(Source::json_text(), Config::jsx_to_json:config()) -> json_text() | {incomplete, decoder()}. format(Source) -> format(Source, []). -format(Source, Opts) -> jsx_to_json:format(Source, Opts). +format(Source, Config) -> jsx_to_json:format(Source, Config). -spec minify(Source::json_text()) -> json_text() | {incomplete, decoder()}. @@ -84,45 +83,45 @@ prettify(Source) -> format(Source, [space, {indent, 2}]). -spec decode(Source::json_text()) -> json_term() | {incomplete, decoder()}. --spec decode(Source::json_text(), Opts::jsx_to_term:opts()) -> json_term() | {incomplete, decoder()}. +-spec decode(Source::json_text(), Config::jsx_to_term:config()) -> json_term() | {incomplete, decoder()}. decode(Source) -> decode(Source, []). -decode(Source, Opts) -> jsx_to_term:to_term(Source, Opts). +decode(Source, Config) -> jsx_to_term:to_term(Source, Config). %% old api, alias for to_term/x to_term(Source) -> decode(Source, []). -to_term(Source, Opts) -> decode(Source, Opts). +to_term(Source, Config) -> decode(Source, Config). json_to_term(Source) -> decode(Source, []). -json_to_term(Source, Opts) -> decode(Source, Opts). +json_to_term(Source, Config) -> decode(Source, Config). -spec is_json(Source::any()) -> true | false. --spec is_json(Source::any(), Opts::jsx_verify:opts()) -> true | false. +-spec is_json(Source::any(), Config::jsx_verify:config()) -> true | false. is_json(Source) -> is_json(Source, []). -is_json(Source, Opts) -> jsx_verify:is_json(Source, Opts). +is_json(Source, Config) -> jsx_verify:is_json(Source, Config). -spec is_term(Source::any()) -> true | false. --spec is_term(Source::any(), Opts::jsx_verify:opts()) -> true | false. +-spec is_term(Source::any(), Config::jsx_verify:config()) -> true | false. is_term(Source) -> is_term(Source, []). -is_term(Source, Opts) -> jsx_verify:is_term(Source, Opts). +is_term(Source, Config) -> jsx_verify:is_term(Source, Config). -type decoder() :: fun((json_text() | end_stream) -> any()). --spec decoder(Handler::module(), State::any(), Opts::list()) -> decoder(). +-spec decoder(Handler::module(), State::any(), Config::list()) -> decoder(). -decoder(Handler, State, Opts) -> jsx_decoder:decoder(Handler, State, Opts). +decoder(Handler, State, Config) -> jsx_decoder:decoder(Handler, State, Config). -type encoder() :: fun((json_term() | end_stream) -> any()). --spec encoder(Handler::module(), State::any(), Opts::list()) -> encoder(). +-spec encoder(Handler::module(), State::any(), Config::list()) -> encoder(). -encoder(Handler, State, Opts) -> jsx_encoder:encoder(Handler, State, Opts). +encoder(Handler, State, Config) -> jsx_encoder:encoder(Handler, State, Config). -type token() :: [token()] @@ -149,6 +148,6 @@ encoder(Handler, State, Opts) -> jsx_encoder:encoder(Handler, State, Opts). -type parser() :: fun((token() | end_stream) -> any()). --spec parser(Handler::module(), State::any(), Opts::list()) -> parser(). +-spec parser(Handler::module(), State::any(), Config::list()) -> parser(). -parser(Handler, State, Opts) -> jsx_parser:parser(Handler, State, Opts). \ No newline at end of file +parser(Handler, State, Config) -> jsx_parser:parser(Handler, State, Config). \ No newline at end of file diff --git a/src/jsx_opts.hrl b/src/jsx_config.hrl similarity index 93% rename from src/jsx_opts.hrl rename to src/jsx_config.hrl index 6b3de90..502c29f 100644 --- a/src/jsx_opts.hrl +++ b/src/jsx_config.hrl @@ -1,4 +1,4 @@ --record(opts, { +-record(config, { replaced_bad_utf8 = false, escaped_forward_slashes = false, single_quoted_strings = false, diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 93bac07..9d452dd 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -26,13 +26,13 @@ -export([decoder/3]). --spec decoder(Handler::module(), State::any(), Opts::jsx:opts()) -> jsx:decoder(). +-spec decoder(Handler::module(), State::any(), Config::jsx:config()) -> jsx:decoder(). -decoder(Handler, State, Opts) -> - fun(JSON) -> start(JSON, {Handler, Handler:init(State)}, [], jsx_utils:parse_opts(Opts)) end. +decoder(Handler, State, Config) -> + fun(JSON) -> start(JSON, {Handler, Handler:init(State)}, [], jsx_utils:parse_config(Config)) end. --include("jsx_opts.hrl"). +-include("jsx_config.hrl"). %% whitespace @@ -102,15 +102,15 @@ decoder(Handler, State, Opts) -> -ifndef(incomplete). --define(incomplete(State, Rest, Handler, Stack, Opts), +-define(incomplete(State, Rest, Handler, Stack, Config), {incomplete, fun(Stream) when is_binary(Stream) -> - State(<>, Handler, Stack, Opts) + State(<>, Handler, Stack, Config) ; (end_stream) -> case State(<>/binary>>, Handler, Stack, - Opts#opts{explicit_end=false}) of - {incomplete, _} -> ?error([Rest, Handler, Stack, Opts]) + Config#config{explicit_end=false}) of + {incomplete, _} -> ?error([Rest, Handler, Stack, Config]) ; Events -> Events end end @@ -128,117 +128,117 @@ decoder(Handler, State, Opts) -> -define(end_seq(Seq), unicode:characters_to_binary(lists:reverse(Seq))). -handle_event([], Handler, _Opts) -> Handler; -handle_event([Event|Rest], Handler, Opts) -> handle_event(Rest, handle_event(Event, Handler, Opts), Opts); -handle_event(Event, {Handler, State}, _Opts) -> {Handler, Handler:handle_event(Event, State)}. +handle_event([], Handler, _Config) -> Handler; +handle_event([Event|Rest], Handler, Config) -> handle_event(Rest, handle_event(Event, Handler, Config), Config); +handle_event(Event, {Handler, State}, _Config) -> {Handler, Handler:handle_event(Event, State)}. -start(<<16#ef, Rest/binary>>, Handler, Stack, Opts) -> - maybe_bom(Rest, Handler, Stack, Opts); -start(<<>>, Handler, Stack, Opts) -> - ?incomplete(start, <<>>, Handler, Stack, Opts); -start(Bin, Handler, Stack, Opts) -> - value(Bin, Handler, Stack, Opts). +start(<<16#ef, Rest/binary>>, Handler, Stack, Config) -> + maybe_bom(Rest, Handler, Stack, Config); +start(<<>>, Handler, Stack, Config) -> + ?incomplete(start, <<>>, Handler, Stack, Config); +start(Bin, Handler, Stack, Config) -> + value(Bin, Handler, Stack, Config). -maybe_bom(<<16#bb, Rest/binary>>, Handler, Stack, Opts) -> - definitely_bom(Rest, Handler, Stack, Opts); -maybe_bom(<<>>, Handler, Stack, Opts) -> - ?incomplete(maybe_bom, <<>>, Handler, Stack, Opts); -maybe_bom(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +maybe_bom(<<16#bb, Rest/binary>>, Handler, Stack, Config) -> + definitely_bom(Rest, Handler, Stack, Config); +maybe_bom(<<>>, Handler, Stack, Config) -> + ?incomplete(maybe_bom, <<>>, Handler, Stack, Config); +maybe_bom(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -definitely_bom(<<16#bf, Rest/binary>>, Handler, Stack, Opts) -> - value(Rest, Handler, Stack, Opts); -definitely_bom(<<>>, Handler, Stack, Opts) -> - ?incomplete(definitely_bom, <<>>, Handler, Stack, Opts); -definitely_bom(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +definitely_bom(<<16#bf, Rest/binary>>, Handler, Stack, Config) -> + value(Rest, Handler, Stack, Config); +definitely_bom(<<>>, Handler, Stack, Config) -> + ?incomplete(definitely_bom, <<>>, Handler, Stack, Config); +definitely_bom(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -value(<>, Handler, Stack, Opts) -> - string(Rest, Handler, [?new_seq()|Stack], Opts); -value(<>, Handler, Stack, Opts = #opts{single_quoted_strings=true}) -> - string(Rest, Handler, [?new_seq(), single_quote|Stack], Opts); -value(<<$t, Rest/binary>>, Handler, Stack, Opts) -> - tr(Rest, Handler, Stack, Opts); -value(<<$f, Rest/binary>>, Handler, Stack, Opts) -> - fa(Rest, Handler, Stack, Opts); -value(<<$n, Rest/binary>>, Handler, Stack, Opts) -> - nu(Rest, Handler, Stack, Opts); -value(<>, Handler, Stack, Opts) -> - negative(Rest, Handler, [[$-]|Stack], Opts); -value(<>, Handler, Stack, Opts) -> - zero(Rest, Handler, [[$0]|Stack], Opts); -value(<>, Handler, Stack, Opts) when ?is_nonzero(S) -> - integer(Rest, Handler, [[S]|Stack], Opts); -value(<>, Handler, Stack, Opts) -> - object(Rest, handle_event(start_object, Handler, Opts), [key|Stack], Opts); -value(<>, Handler, Stack, Opts) -> - array(Rest, handle_event(start_array, Handler, Opts), [array|Stack], Opts); -value(<>, Handler, Stack, Opts) when ?is_whitespace(S) -> - value(Rest, Handler, Stack, Opts); -value(<>, Handler, Stack, Opts=#opts{comments=true}) -> - comment(Rest, Handler, [value|Stack], Opts); -value(<<>>, Handler, Stack, Opts) -> - ?incomplete(value, <<>>, Handler, Stack, Opts); -value(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +value(<>, Handler, Stack, Config) -> + string(Rest, Handler, [?new_seq()|Stack], Config); +value(<>, Handler, Stack, Config = #config{single_quoted_strings=true}) -> + string(Rest, Handler, [?new_seq(), single_quote|Stack], Config); +value(<<$t, Rest/binary>>, Handler, Stack, Config) -> + tr(Rest, Handler, Stack, Config); +value(<<$f, Rest/binary>>, Handler, Stack, Config) -> + fa(Rest, Handler, Stack, Config); +value(<<$n, Rest/binary>>, Handler, Stack, Config) -> + nu(Rest, Handler, Stack, Config); +value(<>, Handler, Stack, Config) -> + negative(Rest, Handler, [[$-]|Stack], Config); +value(<>, Handler, Stack, Config) -> + zero(Rest, Handler, [[$0]|Stack], Config); +value(<>, Handler, Stack, Config) when ?is_nonzero(S) -> + integer(Rest, Handler, [[S]|Stack], Config); +value(<>, Handler, Stack, Config) -> + object(Rest, handle_event(start_object, Handler, Config), [key|Stack], Config); +value(<>, Handler, Stack, Config) -> + array(Rest, handle_event(start_array, Handler, Config), [array|Stack], Config); +value(<>, Handler, Stack, Config) when ?is_whitespace(S) -> + value(Rest, Handler, Stack, Config); +value(<>, Handler, Stack, Config=#config{comments=true}) -> + comment(Rest, Handler, [value|Stack], Config); +value(<<>>, Handler, Stack, Config) -> + ?incomplete(value, <<>>, Handler, Stack, Config); +value(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -object(<>, Handler, Stack, Opts) -> - string(Rest, Handler, [?new_seq()|Stack], Opts); -object(<>, Handler, Stack, Opts = #opts{single_quoted_strings=true}) -> - string(Rest, Handler, [?new_seq(), single_quote|Stack], Opts); -object(<>, Handler, [key|Stack], Opts) -> - maybe_done(Rest, handle_event(end_object, Handler, Opts), Stack, Opts); -object(<>, Handler, Stack, Opts) when ?is_whitespace(S) -> - object(Rest, Handler, Stack, Opts); -object(<>, Handler, Stack, Opts=#opts{comments=true}) -> - comment(Rest, Handler, [object|Stack], Opts); -object(<<>>, Handler, Stack, Opts) -> - ?incomplete(object, <<>>, Handler, Stack, Opts); -object(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +object(<>, Handler, Stack, Config) -> + string(Rest, Handler, [?new_seq()|Stack], Config); +object(<>, Handler, Stack, Config = #config{single_quoted_strings=true}) -> + string(Rest, Handler, [?new_seq(), single_quote|Stack], Config); +object(<>, Handler, [key|Stack], Config) -> + maybe_done(Rest, handle_event(end_object, Handler, Config), Stack, Config); +object(<>, Handler, Stack, Config) when ?is_whitespace(S) -> + object(Rest, Handler, Stack, Config); +object(<>, Handler, Stack, Config=#config{comments=true}) -> + comment(Rest, Handler, [object|Stack], Config); +object(<<>>, Handler, Stack, Config) -> + ?incomplete(object, <<>>, Handler, Stack, Config); +object(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -array(<>, Handler, [array|Stack], Opts) -> - maybe_done(Rest, handle_event(end_array, Handler, Opts), Stack, Opts); -array(<>, Handler, Stack, Opts) when ?is_whitespace(S) -> - array(Rest, Handler, Stack, Opts); -array(<>, Handler, Stack, Opts=#opts{comments=true}) -> - comment(Rest, Handler, [array|Stack], Opts); -array(<<>>, Handler, Stack, Opts) -> - ?incomplete(array, <<>>, Handler, Stack, Opts); -array(Bin, Handler, Stack, Opts) -> - value(Bin, Handler, Stack, Opts). +array(<>, Handler, [array|Stack], Config) -> + maybe_done(Rest, handle_event(end_array, Handler, Config), Stack, Config); +array(<>, Handler, Stack, Config) when ?is_whitespace(S) -> + array(Rest, Handler, Stack, Config); +array(<>, Handler, Stack, Config=#config{comments=true}) -> + comment(Rest, Handler, [array|Stack], Config); +array(<<>>, Handler, Stack, Config) -> + ?incomplete(array, <<>>, Handler, Stack, Config); +array(Bin, Handler, Stack, Config) -> + value(Bin, Handler, Stack, Config). -colon(<>, Handler, [key|Stack], Opts) -> - value(Rest, Handler, [object|Stack], Opts); -colon(<>, Handler, Stack, Opts) when ?is_whitespace(S) -> - colon(Rest, Handler, Stack, Opts); -colon(<>, Handler, Stack, Opts=#opts{comments=true}) -> - comment(Rest, Handler, [colon|Stack], Opts); -colon(<<>>, Handler, Stack, Opts) -> - ?incomplete(colon, <<>>, Handler, Stack, Opts); -colon(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +colon(<>, Handler, [key|Stack], Config) -> + value(Rest, Handler, [object|Stack], Config); +colon(<>, Handler, Stack, Config) when ?is_whitespace(S) -> + colon(Rest, Handler, Stack, Config); +colon(<>, Handler, Stack, Config=#config{comments=true}) -> + comment(Rest, Handler, [colon|Stack], Config); +colon(<<>>, Handler, Stack, Config) -> + ?incomplete(colon, <<>>, Handler, Stack, Config); +colon(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -key(<>, Handler, Stack, Opts) -> - string(Rest, Handler, [?new_seq()|Stack], Opts); -key(<>, Handler, Stack, Opts = #opts{single_quoted_strings=true}) -> - string(Rest, Handler, [?new_seq(), single_quote|Stack], Opts); -key(<>, Handler, Stack, Opts) when ?is_whitespace(S) -> - key(Rest, Handler, Stack, Opts); -key(<>, Handler, Stack, Opts=#opts{comments=true}) -> - comment(Rest, Handler, [key|Stack], Opts); -key(<<>>, Handler, Stack, Opts) -> - ?incomplete(key, <<>>, Handler, Stack, Opts); -key(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +key(<>, Handler, Stack, Config) -> + string(Rest, Handler, [?new_seq()|Stack], Config); +key(<>, Handler, Stack, Config = #config{single_quoted_strings=true}) -> + string(Rest, Handler, [?new_seq(), single_quote|Stack], Config); +key(<>, Handler, Stack, Config) when ?is_whitespace(S) -> + key(Rest, Handler, Stack, Config); +key(<>, Handler, Stack, Config=#config{comments=true}) -> + comment(Rest, Handler, [key|Stack], Config); +key(<<>>, Handler, Stack, Config) -> + ?incomplete(key, <<>>, Handler, Stack, Config); +key(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). %% string appends it's output to the term at the top of the stack. for @@ -260,271 +260,271 @@ partial_utf(_) -> false. %% explicitly whitelist ascii set for better efficiency (seriously, it's worth %% almost a 20% increase) -string(<<32, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 32)|Stack], Opts); -string(<<33, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 33)|Stack], Opts); -string(<>, Handler, S, Opts) -> +string(<<32, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 32)|Stack], Config); +string(<<33, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 33)|Stack], Config); +string(<>, Handler, S, Config) -> case S of [Acc, key|Stack] -> - colon(Rest, handle_event({key, ?end_seq(Acc)}, Handler, Opts), [key|Stack], Opts); + colon(Rest, handle_event({key, ?end_seq(Acc)}, Handler, Config), [key|Stack], Config); [_Acc, single_quote|_Stack] -> - ?error([<>, Handler, S, Opts]); + ?error([<>, Handler, S, Config]); [Acc|Stack] -> - maybe_done(Rest, handle_event({string, ?end_seq(Acc)}, Handler, Opts), Stack, Opts) + maybe_done(Rest, handle_event({string, ?end_seq(Acc)}, Handler, Config), Stack, Config) end; -string(<<35, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 35)|Stack], Opts); -string(<<36, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 36)|Stack], Opts); -string(<<37, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 37)|Stack], Opts); -string(<<38, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 38)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) -> - case Opts#opts.single_quoted_strings of +string(<<35, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 35)|Stack], Config); +string(<<36, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 36)|Stack], Config); +string(<<37, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 37)|Stack], Config); +string(<<38, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 38)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) -> + case Config#config.single_quoted_strings of true -> case Stack of [single_quote, key|S] -> - colon(Rest, handle_event({key, ?end_seq(Acc)}, Handler, Opts), [key|S], Opts) + colon(Rest, handle_event({key, ?end_seq(Acc)}, Handler, Config), [key|S], Config) ; [single_quote|S] -> - maybe_done(Rest, handle_event({string, ?end_seq(Acc)}, Handler, Opts), S, Opts) + maybe_done(Rest, handle_event({string, ?end_seq(Acc)}, Handler, Config), S, Config) ; _ -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace(?singlequote, Opts))|Stack], Opts) + string(Rest, Handler, [?acc_seq(Acc, maybe_replace(?singlequote, Config))|Stack], Config) end ; false -> - string(Rest, Handler, [?acc_seq(Acc, ?singlequote)|Stack], Opts) + string(Rest, Handler, [?acc_seq(Acc, ?singlequote)|Stack], Config) end; -string(<<40, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 40)|Stack], Opts); -string(<<41, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 41)|Stack], Opts); -string(<<42, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 42)|Stack], Opts); -string(<<43, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 43)|Stack], Opts); -string(<<44, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 44)|Stack], Opts); -string(<<45, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 45)|Stack], Opts); -string(<<46, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 46)|Stack], Opts); -string(<<$/, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace($/, Opts))|Stack], Opts); -string(<<48, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 48)|Stack], Opts); -string(<<49, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 49)|Stack], Opts); -string(<<50, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 50)|Stack], Opts); -string(<<51, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 51)|Stack], Opts); -string(<<52, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 52)|Stack], Opts); -string(<<53, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 53)|Stack], Opts); -string(<<54, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 54)|Stack], Opts); -string(<<55, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 55)|Stack], Opts); -string(<<56, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 56)|Stack], Opts); -string(<<57, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 57)|Stack], Opts); -string(<<58, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 58)|Stack], Opts); -string(<<59, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 59)|Stack], Opts); -string(<<60, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 60)|Stack], Opts); -string(<<61, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 61)|Stack], Opts); -string(<<62, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 62)|Stack], Opts); -string(<<63, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 63)|Stack], Opts); -string(<<64, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 64)|Stack], Opts); -string(<<65, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 65)|Stack], Opts); -string(<<66, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 66)|Stack], Opts); -string(<<67, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 67)|Stack], Opts); -string(<<68, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 68)|Stack], Opts); -string(<<69, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 69)|Stack], Opts); -string(<<70, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 70)|Stack], Opts); -string(<<71, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 71)|Stack], Opts); -string(<<72, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 72)|Stack], Opts); -string(<<73, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 73)|Stack], Opts); -string(<<74, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 74)|Stack], Opts); -string(<<75, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 75)|Stack], Opts); -string(<<76, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 76)|Stack], Opts); -string(<<77, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 77)|Stack], Opts); -string(<<78, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 78)|Stack], Opts); -string(<<79, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 79)|Stack], Opts); -string(<<80, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 80)|Stack], Opts); -string(<<81, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 81)|Stack], Opts); -string(<<82, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 82)|Stack], Opts); -string(<<83, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 83)|Stack], Opts); -string(<<84, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 84)|Stack], Opts); -string(<<85, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 85)|Stack], Opts); -string(<<86, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 86)|Stack], Opts); -string(<<87, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 87)|Stack], Opts); -string(<<88, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 88)|Stack], Opts); -string(<<89, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 89)|Stack], Opts); -string(<<90, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 90)|Stack], Opts); -string(<<91, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 91)|Stack], Opts); -string(<>, Handler, Stack, Opts) -> - escape(Rest, Handler, Stack, Opts); -string(<<93, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 93)|Stack], Opts); -string(<<94, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 94)|Stack], Opts); -string(<<95, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 95)|Stack], Opts); -string(<<96, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 96)|Stack], Opts); -string(<<97, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 97)|Stack], Opts); -string(<<98, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 98)|Stack], Opts); -string(<<99, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 99)|Stack], Opts); -string(<<100, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 100)|Stack], Opts); -string(<<101, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 101)|Stack], Opts); -string(<<102, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 102)|Stack], Opts); -string(<<103, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 103)|Stack], Opts); -string(<<104, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 104)|Stack], Opts); -string(<<105, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 105)|Stack], Opts); -string(<<106, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 106)|Stack], Opts); -string(<<107, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 107)|Stack], Opts); -string(<<108, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 108)|Stack], Opts); -string(<<109, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 109)|Stack], Opts); -string(<<110, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 110)|Stack], Opts); -string(<<111, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 111)|Stack], Opts); -string(<<112, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 112)|Stack], Opts); -string(<<113, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 113)|Stack], Opts); -string(<<114, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 114)|Stack], Opts); -string(<<115, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 115)|Stack], Opts); -string(<<116, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 116)|Stack], Opts); -string(<<117, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 117)|Stack], Opts); -string(<<118, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 118)|Stack], Opts); -string(<<119, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 119)|Stack], Opts); -string(<<120, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 120)|Stack], Opts); -string(<<121, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 121)|Stack], Opts); -string(<<122, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 122)|Stack], Opts); -string(<<123, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 123)|Stack], Opts); -string(<<124, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 124)|Stack], Opts); -string(<<125, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 125)|Stack], Opts); -string(<<126, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 126)|Stack], Opts); -string(<<127, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 127)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X >= 16#20, X < 16#2028 -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X == 16#2028; X == 16#2029 -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace(X, Opts))|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X > 16#2029, X < 16#d800 -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X > 16#dfff, X < 16#fdd0 -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X > 16#fdef, X < 16#fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X >= 16#10000, X < 16#1fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X >= 16#20000, X < 16#2fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X >= 16#30000, X < 16#3fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X >= 16#40000, X < 16#4fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X >= 16#50000, X < 16#5fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X >= 16#60000, X < 16#6fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X >= 16#70000, X < 16#7fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X >= 16#80000, X < 16#8fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X >= 16#90000, X < 16#9fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X >= 16#a0000, X < 16#afffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X >= 16#b0000, X < 16#bfffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X >= 16#c0000, X < 16#cfffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X >= 16#d0000, X < 16#dfffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X >= 16#e0000, X < 16#efffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X >= 16#f0000, X < 16#ffffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) when X >= 16#100000, X < 16#10fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Opts); -string(<>, Handler, [Acc|Stack], Opts) -> - case Opts#opts.replaced_bad_utf8 of - true -> noncharacter(<>, Handler, [Acc|Stack], Opts) - ; false -> ?error([<>, Handler, [Acc|Stack], Opts]) +string(<<40, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 40)|Stack], Config); +string(<<41, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 41)|Stack], Config); +string(<<42, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 42)|Stack], Config); +string(<<43, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 43)|Stack], Config); +string(<<44, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 44)|Stack], Config); +string(<<45, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 45)|Stack], Config); +string(<<46, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 46)|Stack], Config); +string(<<$/, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace($/, Config))|Stack], Config); +string(<<48, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 48)|Stack], Config); +string(<<49, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 49)|Stack], Config); +string(<<50, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 50)|Stack], Config); +string(<<51, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 51)|Stack], Config); +string(<<52, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 52)|Stack], Config); +string(<<53, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 53)|Stack], Config); +string(<<54, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 54)|Stack], Config); +string(<<55, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 55)|Stack], Config); +string(<<56, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 56)|Stack], Config); +string(<<57, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 57)|Stack], Config); +string(<<58, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 58)|Stack], Config); +string(<<59, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 59)|Stack], Config); +string(<<60, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 60)|Stack], Config); +string(<<61, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 61)|Stack], Config); +string(<<62, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 62)|Stack], Config); +string(<<63, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 63)|Stack], Config); +string(<<64, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 64)|Stack], Config); +string(<<65, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 65)|Stack], Config); +string(<<66, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 66)|Stack], Config); +string(<<67, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 67)|Stack], Config); +string(<<68, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 68)|Stack], Config); +string(<<69, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 69)|Stack], Config); +string(<<70, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 70)|Stack], Config); +string(<<71, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 71)|Stack], Config); +string(<<72, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 72)|Stack], Config); +string(<<73, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 73)|Stack], Config); +string(<<74, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 74)|Stack], Config); +string(<<75, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 75)|Stack], Config); +string(<<76, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 76)|Stack], Config); +string(<<77, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 77)|Stack], Config); +string(<<78, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 78)|Stack], Config); +string(<<79, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 79)|Stack], Config); +string(<<80, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 80)|Stack], Config); +string(<<81, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 81)|Stack], Config); +string(<<82, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 82)|Stack], Config); +string(<<83, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 83)|Stack], Config); +string(<<84, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 84)|Stack], Config); +string(<<85, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 85)|Stack], Config); +string(<<86, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 86)|Stack], Config); +string(<<87, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 87)|Stack], Config); +string(<<88, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 88)|Stack], Config); +string(<<89, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 89)|Stack], Config); +string(<<90, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 90)|Stack], Config); +string(<<91, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 91)|Stack], Config); +string(<>, Handler, Stack, Config) -> + escape(Rest, Handler, Stack, Config); +string(<<93, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 93)|Stack], Config); +string(<<94, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 94)|Stack], Config); +string(<<95, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 95)|Stack], Config); +string(<<96, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 96)|Stack], Config); +string(<<97, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 97)|Stack], Config); +string(<<98, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 98)|Stack], Config); +string(<<99, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 99)|Stack], Config); +string(<<100, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 100)|Stack], Config); +string(<<101, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 101)|Stack], Config); +string(<<102, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 102)|Stack], Config); +string(<<103, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 103)|Stack], Config); +string(<<104, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 104)|Stack], Config); +string(<<105, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 105)|Stack], Config); +string(<<106, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 106)|Stack], Config); +string(<<107, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 107)|Stack], Config); +string(<<108, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 108)|Stack], Config); +string(<<109, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 109)|Stack], Config); +string(<<110, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 110)|Stack], Config); +string(<<111, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 111)|Stack], Config); +string(<<112, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 112)|Stack], Config); +string(<<113, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 113)|Stack], Config); +string(<<114, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 114)|Stack], Config); +string(<<115, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 115)|Stack], Config); +string(<<116, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 116)|Stack], Config); +string(<<117, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 117)|Stack], Config); +string(<<118, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 118)|Stack], Config); +string(<<119, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 119)|Stack], Config); +string(<<120, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 120)|Stack], Config); +string(<<121, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 121)|Stack], Config); +string(<<122, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 122)|Stack], Config); +string(<<123, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 123)|Stack], Config); +string(<<124, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 124)|Stack], Config); +string(<<125, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 125)|Stack], Config); +string(<<126, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 126)|Stack], Config); +string(<<127, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 127)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#20, X < 16#2028 -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X == 16#2028; X == 16#2029 -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace(X, Config))|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X > 16#2029, X < 16#d800 -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X > 16#dfff, X < 16#fdd0 -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X > 16#fdef, X < 16#fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#10000, X < 16#1fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#20000, X < 16#2fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#30000, X < 16#3fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#40000, X < 16#4fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#50000, X < 16#5fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#60000, X < 16#6fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#70000, X < 16#7fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#80000, X < 16#8fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#90000, X < 16#9fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#a0000, X < 16#afffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#b0000, X < 16#bfffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#c0000, X < 16#cfffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#d0000, X < 16#dfffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#e0000, X < 16#efffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#f0000, X < 16#ffffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#100000, X < 16#10fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) -> + case Config#config.replaced_bad_utf8 of + true -> noncharacter(<>, Handler, [Acc|Stack], Config) + ; false -> ?error([<>, Handler, [Acc|Stack], Config]) end; -string(Bin, Handler, Stack, Opts) -> +string(Bin, Handler, Stack, Config) -> case partial_utf(Bin) of - true -> ?incomplete(string, Bin, Handler, Stack, Opts) + true -> ?incomplete(string, Bin, Handler, Stack, Config) ; false -> - case Opts#opts.replaced_bad_utf8 of - true -> noncharacter(Bin, Handler, Stack, Opts) - ; false -> ?error([Bin, Handler, Stack, Opts]) + case Config#config.replaced_bad_utf8 of + true -> noncharacter(Bin, Handler, Stack, Config) + ; false -> ?error([Bin, Handler, Stack, Config]) end end. @@ -532,93 +532,93 @@ string(Bin, Handler, Stack, Opts) -> %% we don't need to guard against partial utf here, because it's already taken %% care of in string %% surrogates -noncharacter(<<237, X, _, Rest/binary>>, Handler, [Acc|Stack], Opts) when X >= 160 -> - string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Opts); +noncharacter(<<237, X, _, Rest/binary>>, Handler, [Acc|Stack], Config) when X >= 160 -> + string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config); %% u+fffe and u+ffff for R14BXX -noncharacter(<<239, 191, X, Rest/binary>>, Handler, [Acc|Stack], Opts) when X == 190; X == 191 -> - string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Opts); +noncharacter(<<239, 191, X, Rest/binary>>, Handler, [Acc|Stack], Config) when X == 190; X == 191 -> + string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config); %% u+xfffe, u+xffff and other noncharacters -noncharacter(<<_/utf8, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Opts); +noncharacter(<<_/utf8, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config); %% overlong encodings and missing continuations of a 2 byte sequence -noncharacter(<>, Handler, Stack, Opts) when X >= 192, X =< 223 -> - strip_continuations(Rest, Handler, [1|Stack], Opts); +noncharacter(<>, Handler, Stack, Config) when X >= 192, X =< 223 -> + strip_continuations(Rest, Handler, [1|Stack], Config); %% overlong encodings and missing continuations of a 3 byte sequence -noncharacter(<>, Handler, Stack, Opts) when X >= 224, X =< 239 -> - strip_continuations(Rest, Handler, [2|Stack], Opts); +noncharacter(<>, Handler, Stack, Config) when X >= 224, X =< 239 -> + strip_continuations(Rest, Handler, [2|Stack], Config); %% overlong encodings and missing continuations of a 4 byte sequence -noncharacter(<>, Handler, Stack, Opts) when X >= 240, X =< 247 -> - strip_continuations(Rest, Handler, [3|Stack], Opts); +noncharacter(<>, Handler, Stack, Config) when X >= 240, X =< 247 -> + strip_continuations(Rest, Handler, [3|Stack], Config); %% unexpected bytes, including orphan continuations -noncharacter(<<_, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Opts); -noncharacter(<<>>, Handler, Stack, Opts) -> - ?incomplete(noncharacter, <<>>, Handler, Stack, Opts). +noncharacter(<<_, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config); +noncharacter(<<>>, Handler, Stack, Config) -> + ?incomplete(noncharacter, <<>>, Handler, Stack, Config). %% strips continuation bytes after bad utf bytes, guards against both too short %% and overlong sequences. N is the maximum number of bytes to strip -strip_continuations(Rest, Handler, [0, Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Opts); -strip_continuations(<>, Handler, [N|Stack], Opts) when X >= 128, X =< 191 -> - strip_continuations(Rest, Handler, [N - 1|Stack], Opts); +strip_continuations(Rest, Handler, [0, Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config); +strip_continuations(<>, Handler, [N|Stack], Config) when X >= 128, X =< 191 -> + strip_continuations(Rest, Handler, [N - 1|Stack], Config); %% incomplete -strip_continuations(<<>>, Handler, Stack, Opts) -> - ?incomplete(strip_continuations, <<>>, Handler, Stack, Opts); +strip_continuations(<<>>, Handler, Stack, Config) -> + ?incomplete(strip_continuations, <<>>, Handler, Stack, Config); %% not a continuation byte, dispatch back to string -strip_continuations(Rest, Handler, [_, Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Opts). +strip_continuations(Rest, Handler, [_, Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config). -escape(<<$b, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\b, Opts))|Stack], Opts); -escape(<<$f, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\f, Opts))|Stack], Opts); -escape(<<$n, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\n, Opts))|Stack], Opts); -escape(<<$r, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\r, Opts))|Stack], Opts); -escape(<<$t, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\t, Opts))|Stack], Opts); -escape(<>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\\, Opts))|Stack], Opts); -escape(<>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace($/, Opts))|Stack], Opts); -escape(<>, Handler, [Acc|Stack], Opts) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\", Opts))|Stack], Opts); -escape(<>, Handler, [Acc|Stack], Opts = #opts{single_quoted_strings=true}) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace(?singlequote, Opts))|Stack], Opts); -escape(<<$u, Rest/binary>>, Handler, Stack, Opts) -> - escaped_unicode(Rest, Handler, Stack, Opts); -escape(<<>>, Handler, Stack, Opts) -> - ?incomplete(escape, <<>>, Handler, Stack, Opts); -escape(Bin, Handler, [Acc|Stack], Opts=#opts{ignored_bad_escapes=true}) -> - string(Bin, Handler, [?acc_seq(Acc, ?rsolidus)|Stack], Opts); -escape(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +escape(<<$b, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\b, Config))|Stack], Config); +escape(<<$f, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\f, Config))|Stack], Config); +escape(<<$n, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\n, Config))|Stack], Config); +escape(<<$r, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\r, Config))|Stack], Config); +escape(<<$t, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\t, Config))|Stack], Config); +escape(<>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\\, Config))|Stack], Config); +escape(<>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace($/, Config))|Stack], Config); +escape(<>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\", Config))|Stack], Config); +escape(<>, Handler, [Acc|Stack], Config = #config{single_quoted_strings=true}) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace(?singlequote, Config))|Stack], Config); +escape(<<$u, Rest/binary>>, Handler, Stack, Config) -> + escaped_unicode(Rest, Handler, Stack, Config); +escape(<<>>, Handler, Stack, Config) -> + ?incomplete(escape, <<>>, Handler, Stack, Config); +escape(Bin, Handler, [Acc|Stack], Config=#config{ignored_bad_escapes=true}) -> + string(Bin, Handler, [?acc_seq(Acc, ?rsolidus)|Stack], Config); +escape(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). %% this code is ugly and unfortunate, but so is json's handling of escaped %% unicode codepoint sequences. -escaped_unicode(<>, Handler, [Acc|Stack], Opts) +escaped_unicode(<>, Handler, [Acc|Stack], Config) when ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D) -> case erlang:list_to_integer([A, B, C, D], 16) of %% high surrogate, dispatch to low surrogate X when X >= 16#d800, X =< 16#dbff -> - low_surrogate(Rest, Handler, [X, Acc|Stack], Opts) + low_surrogate(Rest, Handler, [X, Acc|Stack], Config) %% low surrogate, illegal in this position ; X when X >= 16#dc00, X =< 16#dfff -> - case Opts#opts.replaced_bad_utf8 of - true -> string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Opts) - ; false -> ?error([<>, Handler, [Acc|Stack], Opts]) + case Config#config.replaced_bad_utf8 of + true -> string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config) + ; false -> ?error([<>, Handler, [Acc|Stack], Config]) end %% anything else - ; X -> string(Rest, Handler, [?acc_seq(Acc, maybe_replace(X, Opts))|Stack], Opts) + ; X -> string(Rest, Handler, [?acc_seq(Acc, maybe_replace(X, Config))|Stack], Config) end; -escaped_unicode(Bin, Handler, Stack, Opts) -> +escaped_unicode(Bin, Handler, Stack, Config) -> case is_partial_escape(Bin) of - true -> ?incomplete(escaped_unicode, Bin, Handler, Stack, Opts) - ; false -> ?error([Bin, Handler, Stack, Opts]) + true -> ?incomplete(escaped_unicode, Bin, Handler, Stack, Config) + ; false -> ?error([Bin, Handler, Stack, Config]) end. @@ -629,34 +629,34 @@ is_partial_escape(<<>>) -> true; is_partial_escape(_) -> false. -low_surrogate(<>, Handler, [High, Acc|Stack], Opts) +low_surrogate(<>, Handler, [High, Acc|Stack], Config) when ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D) -> case erlang:list_to_integer([A, B, C, D], 16) of X when X >= 16#dc00, X =< 16#dfff -> Y = surrogate_to_codepoint(High, X), case (Y =< 16#d800 orelse Y >= 16#e000) of - true -> string(Rest, Handler, [?acc_seq(Acc, Y)|Stack], Opts) + true -> string(Rest, Handler, [?acc_seq(Acc, Y)|Stack], Config) ; false -> - case Opts#opts.replaced_bad_utf8 of + case Config#config.replaced_bad_utf8 of true -> - string(Rest, Handler, [?acc_seq(Acc, 16#fffd, 16#fffd)|Stack], Opts) + string(Rest, Handler, [?acc_seq(Acc, 16#fffd, 16#fffd)|Stack], Config) ; false -> - ?error([<>, Handler, [High, Acc|Stack], Opts]) + ?error([<>, Handler, [High, Acc|Stack], Config]) end end ; _ -> - case Opts#opts.replaced_bad_utf8 of - true -> string(Rest, Handler, [?acc_seq(Acc, 16#fffd, 16#fffd)|Stack], Opts) - ; false -> ?error([<>, Handler, [High, Acc|Stack], Opts]) + case Config#config.replaced_bad_utf8 of + true -> string(Rest, Handler, [?acc_seq(Acc, 16#fffd, 16#fffd)|Stack], Config) + ; false -> ?error([<>, Handler, [High, Acc|Stack], Config]) end end; -low_surrogate(Bin, Handler, [High, Acc|Stack], Opts) -> +low_surrogate(Bin, Handler, [High, Acc|Stack], Config) -> case is_partial_low(Bin) of - true -> ?incomplete(low_surrogate, Bin, Handler, [High, Acc|Stack], Opts) + true -> ?incomplete(low_surrogate, Bin, Handler, [High, Acc|Stack], Config) ; false -> - case Opts#opts.replaced_bad_utf8 of - true -> string(Bin, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Opts) - ; false -> ?error([Bin, Handler, [High, Acc|Stack], Opts]) + case Config#config.replaced_bad_utf8 of + true -> string(Bin, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config) + ; false -> ?error([Bin, Handler, [High, Acc|Stack], Config]) end end. @@ -675,170 +675,170 @@ surrogate_to_codepoint(High, Low) -> (High - 16#d800) * 16#400 + (Low - 16#dc00) + 16#10000. -maybe_replace(X, #opts{dirty_strings=true}) when is_integer(X) -> [X]; -maybe_replace($\b, #opts{escaped_strings=true}) -> [$\\, $b]; -maybe_replace($\t, #opts{escaped_strings=true}) -> [$\\, $t]; -maybe_replace($\n, #opts{escaped_strings=true}) -> [$\\, $n]; -maybe_replace($\f, #opts{escaped_strings=true}) -> [$\\, $f]; -maybe_replace($\r, #opts{escaped_strings=true}) -> [$\\, $r]; -maybe_replace($\", #opts{escaped_strings=true}) -> [$\\, $\"]; -maybe_replace($', Opts=#opts{escaped_strings=true}) -> - case Opts#opts.single_quoted_strings of +maybe_replace(X, #config{dirty_strings=true}) when is_integer(X) -> [X]; +maybe_replace($\b, #config{escaped_strings=true}) -> [$\\, $b]; +maybe_replace($\t, #config{escaped_strings=true}) -> [$\\, $t]; +maybe_replace($\n, #config{escaped_strings=true}) -> [$\\, $n]; +maybe_replace($\f, #config{escaped_strings=true}) -> [$\\, $f]; +maybe_replace($\r, #config{escaped_strings=true}) -> [$\\, $r]; +maybe_replace($\", #config{escaped_strings=true}) -> [$\\, $\"]; +maybe_replace($', Config=#config{escaped_strings=true}) -> + case Config#config.single_quoted_strings of true -> [$\\, $'] ; false -> [$'] end; -maybe_replace($/, Opts=#opts{escaped_strings=true}) -> - case Opts#opts.escaped_forward_slashes of +maybe_replace($/, Config=#config{escaped_strings=true}) -> + case Config#config.escaped_forward_slashes of true -> [$\\, $/] ; false -> [$/] end; -maybe_replace($\\, #opts{escaped_strings=true}) -> [$\\, $\\]; -maybe_replace(X, Opts=#opts{escaped_strings=true}) when X == 16#2028; X == 16#2029 -> - case Opts#opts.unescaped_jsonp of +maybe_replace($\\, #config{escaped_strings=true}) -> [$\\, $\\]; +maybe_replace(X, Config=#config{escaped_strings=true}) when X == 16#2028; X == 16#2029 -> + case Config#config.unescaped_jsonp of true -> [X] ; false -> jsx_utils:json_escape_sequence(X) end; -maybe_replace(X, #opts{escaped_strings=true}) when X < 32 -> +maybe_replace(X, #config{escaped_strings=true}) when X < 32 -> jsx_utils:json_escape_sequence(X); -maybe_replace(X, _Opts) -> [X]. +maybe_replace(X, _Config) -> [X]. %% like strings, numbers are collected in an intermediate accumulator before %% being emitted to the callback handler -negative(<<$0, Rest/binary>>, Handler, [Acc|Stack], Opts) -> - zero(Rest, Handler, ["0" ++ Acc|Stack], Opts); -negative(<>, Handler, [Acc|Stack], Opts) when ?is_nonzero(S) -> - integer(Rest, Handler, [[S] ++ Acc|Stack], Opts); -negative(<<>>, Handler, Stack, Opts) -> - ?incomplete(negative, <<>>, Handler, Stack, Opts); -negative(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +negative(<<$0, Rest/binary>>, Handler, [Acc|Stack], Config) -> + zero(Rest, Handler, ["0" ++ Acc|Stack], Config); +negative(<>, Handler, [Acc|Stack], Config) when ?is_nonzero(S) -> + integer(Rest, Handler, [[S] ++ Acc|Stack], Config); +negative(<<>>, Handler, Stack, Config) -> + ?incomplete(negative, <<>>, Handler, Stack, Config); +negative(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -zero(<>, Handler, [Acc, object|Stack], Opts) -> - maybe_done(Rest, handle_event([format_number(Acc), end_object], Handler, Opts), Stack, Opts); -zero(<>, Handler, [Acc, array|Stack], Opts) -> - maybe_done(Rest, handle_event(end_array, handle_event(format_number(Acc), Handler, Opts), Opts), Stack, Opts); -zero(<>, Handler, [Acc, object|Stack], Opts) -> - key(Rest, handle_event(format_number(Acc), Handler, Opts), [key|Stack], Opts); -zero(<>, Handler, [Acc, array|Stack], Opts) -> - value(Rest, handle_event(format_number(Acc), Handler, Opts), [array|Stack], Opts); -zero(<>, Handler, [Acc|Stack], Opts) -> - initial_decimal(Rest, Handler, [{Acc, []}|Stack], Opts); -zero(<>, Handler, [Acc|Stack], Opts) when S =:= $e; S =:= $E -> - e(Rest, Handler, [{Acc, [], []}|Stack], Opts); -zero(<>, Handler, [Acc|Stack], Opts) when ?is_whitespace(S) -> - maybe_done(Rest, handle_event(format_number(Acc), Handler, Opts), Stack, Opts); -zero(<>, Handler, [Acc|Stack], Opts=#opts{comments=true}) -> - comment(Rest, handle_event(format_number(Acc), Handler, Opts), [maybe_done|Stack], Opts); -zero(<<>>, Handler, [Acc|[]], Opts = #opts{explicit_end=false}) -> - maybe_done(<<>>, handle_event(format_number(Acc), Handler, Opts), [], Opts); -zero(<<>>, Handler, Stack, Opts) -> - ?incomplete(zero, <<>>, Handler, Stack, Opts); -zero(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +zero(<>, Handler, [Acc, object|Stack], Config) -> + maybe_done(Rest, handle_event([format_number(Acc), end_object], Handler, Config), Stack, Config); +zero(<>, Handler, [Acc, array|Stack], Config) -> + maybe_done(Rest, handle_event(end_array, handle_event(format_number(Acc), Handler, Config), Config), Stack, Config); +zero(<>, Handler, [Acc, object|Stack], Config) -> + key(Rest, handle_event(format_number(Acc), Handler, Config), [key|Stack], Config); +zero(<>, Handler, [Acc, array|Stack], Config) -> + value(Rest, handle_event(format_number(Acc), Handler, Config), [array|Stack], Config); +zero(<>, Handler, [Acc|Stack], Config) -> + initial_decimal(Rest, Handler, [{Acc, []}|Stack], Config); +zero(<>, Handler, [Acc|Stack], Config) when S =:= $e; S =:= $E -> + e(Rest, Handler, [{Acc, [], []}|Stack], Config); +zero(<>, Handler, [Acc|Stack], Config) when ?is_whitespace(S) -> + maybe_done(Rest, handle_event(format_number(Acc), Handler, Config), Stack, Config); +zero(<>, Handler, [Acc|Stack], Config=#config{comments=true}) -> + comment(Rest, handle_event(format_number(Acc), Handler, Config), [maybe_done|Stack], Config); +zero(<<>>, Handler, [Acc|[]], Config = #config{explicit_end=false}) -> + maybe_done(<<>>, handle_event(format_number(Acc), Handler, Config), [], Config); +zero(<<>>, Handler, Stack, Config) -> + ?incomplete(zero, <<>>, Handler, Stack, Config); +zero(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -integer(<>, Handler, [Acc|Stack], Opts) when ?is_nonzero(S) -> - integer(Rest, Handler, [[S] ++ Acc|Stack], Opts); -integer(<>, Handler, [Acc, object|Stack], Opts) -> - maybe_done(Rest, handle_event([format_number(Acc), end_object], Handler, Opts), Stack, Opts); -integer(<>, Handler, [Acc, array|Stack], Opts) -> - maybe_done(Rest, handle_event([format_number(Acc), end_array], Handler, Opts), Stack, Opts); -integer(<>, Handler, [Acc, object|Stack], Opts) -> - key(Rest, handle_event(format_number(Acc), Handler, Opts), [key|Stack], Opts); -integer(<>, Handler, [Acc, array|Stack], Opts) -> - value(Rest, handle_event(format_number(Acc), Handler, Opts), [array|Stack], Opts); -integer(<>, Handler, [Acc|Stack], Opts) -> - initial_decimal(Rest, Handler, [{Acc, []}|Stack], Opts); -integer(<>, Handler, [Acc|Stack], Opts) -> - integer(Rest, Handler, [[?zero] ++ Acc|Stack], Opts); -integer(<>, Handler, [Acc|Stack], Opts) when S =:= $e; S =:= $E -> - e(Rest, Handler, [{Acc, [], []}|Stack], Opts); -integer(<>, Handler, [Acc|Stack], Opts) when ?is_whitespace(S) -> - maybe_done(Rest, handle_event(format_number(Acc), Handler, Opts), Stack, Opts); -integer(<>, Handler, [Acc|Stack], Opts=#opts{comments=true}) -> - comment(Rest, handle_event(format_number(Acc), Handler, Opts), [maybe_done|Stack], Opts); -integer(<<>>, Handler, [Acc|[]], Opts = #opts{explicit_end=false}) -> - maybe_done(<<>>, handle_event(format_number(Acc), Handler, Opts), [], Opts); -integer(<<>>, Handler, Stack, Opts) -> - ?incomplete(integer, <<>>, Handler, Stack, Opts); -integer(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +integer(<>, Handler, [Acc|Stack], Config) when ?is_nonzero(S) -> + integer(Rest, Handler, [[S] ++ Acc|Stack], Config); +integer(<>, Handler, [Acc, object|Stack], Config) -> + maybe_done(Rest, handle_event([format_number(Acc), end_object], Handler, Config), Stack, Config); +integer(<>, Handler, [Acc, array|Stack], Config) -> + maybe_done(Rest, handle_event([format_number(Acc), end_array], Handler, Config), Stack, Config); +integer(<>, Handler, [Acc, object|Stack], Config) -> + key(Rest, handle_event(format_number(Acc), Handler, Config), [key|Stack], Config); +integer(<>, Handler, [Acc, array|Stack], Config) -> + value(Rest, handle_event(format_number(Acc), Handler, Config), [array|Stack], Config); +integer(<>, Handler, [Acc|Stack], Config) -> + initial_decimal(Rest, Handler, [{Acc, []}|Stack], Config); +integer(<>, Handler, [Acc|Stack], Config) -> + integer(Rest, Handler, [[?zero] ++ Acc|Stack], Config); +integer(<>, Handler, [Acc|Stack], Config) when S =:= $e; S =:= $E -> + e(Rest, Handler, [{Acc, [], []}|Stack], Config); +integer(<>, Handler, [Acc|Stack], Config) when ?is_whitespace(S) -> + maybe_done(Rest, handle_event(format_number(Acc), Handler, Config), Stack, Config); +integer(<>, Handler, [Acc|Stack], Config=#config{comments=true}) -> + comment(Rest, handle_event(format_number(Acc), Handler, Config), [maybe_done|Stack], Config); +integer(<<>>, Handler, [Acc|[]], Config = #config{explicit_end=false}) -> + maybe_done(<<>>, handle_event(format_number(Acc), Handler, Config), [], Config); +integer(<<>>, Handler, Stack, Config) -> + ?incomplete(integer, <<>>, Handler, Stack, Config); +integer(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -initial_decimal(<>, Handler, [{Int, Frac}|Stack], Opts) when S =:= ?zero; ?is_nonzero(S) -> - decimal(Rest, Handler, [{Int, [S] ++ Frac}|Stack], Opts); -initial_decimal(<<>>, Handler, Stack, Opts) -> - ?incomplete(initial_decimal, <<>>, Handler, Stack, Opts); -initial_decimal(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +initial_decimal(<>, Handler, [{Int, Frac}|Stack], Config) when S =:= ?zero; ?is_nonzero(S) -> + decimal(Rest, Handler, [{Int, [S] ++ Frac}|Stack], Config); +initial_decimal(<<>>, Handler, Stack, Config) -> + ?incomplete(initial_decimal, <<>>, Handler, Stack, Config); +initial_decimal(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -decimal(<>, Handler, [{Int, Frac}|Stack], Opts) +decimal(<>, Handler, [{Int, Frac}|Stack], Config) when S=:= ?zero; ?is_nonzero(S) -> - decimal(Rest, Handler, [{Int, [S] ++ Frac}|Stack], Opts); -decimal(<>, Handler, [Acc, object|Stack], Opts) -> - maybe_done(Rest, handle_event([format_number(Acc), end_object], Handler, Opts), Stack, Opts); -decimal(<>, Handler, [Acc, array|Stack], Opts) -> - maybe_done(Rest, handle_event([format_number(Acc), end_array], Handler, Opts), Stack, Opts); -decimal(<>, Handler, [Acc, object|Stack], Opts) -> - key(Rest, handle_event(format_number(Acc), Handler, Opts), [key|Stack], Opts); -decimal(<>, Handler, [Acc, array|Stack], Opts) -> - value(Rest, handle_event(format_number(Acc), Handler, Opts), [array|Stack], Opts); -decimal(<>, Handler, [{Int, Frac}|Stack], Opts) when S =:= $e; S =:= $E -> - e(Rest, Handler, [{Int, Frac, []}|Stack], Opts); -decimal(<>, Handler, [Acc|Stack], Opts) when ?is_whitespace(S) -> - maybe_done(Rest, handle_event(format_number(Acc), Handler, Opts), Stack, Opts); -decimal(<>, Handler, [Acc|Stack], Opts=#opts{comments=true}) -> - comment(Rest, handle_event(format_number(Acc), Handler, Opts), [maybe_done|Stack], Opts); -decimal(<<>>, Handler, [Acc|[]], Opts = #opts{explicit_end=false}) -> - maybe_done(<<>>, handle_event(format_number(Acc), Handler, Opts), [], Opts); -decimal(<<>>, Handler, Stack, Opts) -> - ?incomplete(decimal, <<>>, Handler, Stack, Opts); -decimal(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). + decimal(Rest, Handler, [{Int, [S] ++ Frac}|Stack], Config); +decimal(<>, Handler, [Acc, object|Stack], Config) -> + maybe_done(Rest, handle_event([format_number(Acc), end_object], Handler, Config), Stack, Config); +decimal(<>, Handler, [Acc, array|Stack], Config) -> + maybe_done(Rest, handle_event([format_number(Acc), end_array], Handler, Config), Stack, Config); +decimal(<>, Handler, [Acc, object|Stack], Config) -> + key(Rest, handle_event(format_number(Acc), Handler, Config), [key|Stack], Config); +decimal(<>, Handler, [Acc, array|Stack], Config) -> + value(Rest, handle_event(format_number(Acc), Handler, Config), [array|Stack], Config); +decimal(<>, Handler, [{Int, Frac}|Stack], Config) when S =:= $e; S =:= $E -> + e(Rest, Handler, [{Int, Frac, []}|Stack], Config); +decimal(<>, Handler, [Acc|Stack], Config) when ?is_whitespace(S) -> + maybe_done(Rest, handle_event(format_number(Acc), Handler, Config), Stack, Config); +decimal(<>, Handler, [Acc|Stack], Config=#config{comments=true}) -> + comment(Rest, handle_event(format_number(Acc), Handler, Config), [maybe_done|Stack], Config); +decimal(<<>>, Handler, [Acc|[]], Config = #config{explicit_end=false}) -> + maybe_done(<<>>, handle_event(format_number(Acc), Handler, Config), [], Config); +decimal(<<>>, Handler, Stack, Config) -> + ?incomplete(decimal, <<>>, Handler, Stack, Config); +decimal(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -e(<>, Handler, [{Int, Frac, Exp}|Stack], Opts) when S =:= ?zero; ?is_nonzero(S) -> - exp(Rest, Handler, [{Int, Frac, [S] ++ Exp}|Stack], Opts); -e(<>, Handler, [{Int, Frac, Exp}|Stack], Opts) when S =:= ?positive; S =:= ?negative -> - ex(Rest, Handler, [{Int, Frac, [S] ++ Exp}|Stack], Opts); -e(<<>>, Handler, Stack, Opts) -> - ?incomplete(e, <<>>, Handler, Stack, Opts); -e(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +e(<>, Handler, [{Int, Frac, Exp}|Stack], Config) when S =:= ?zero; ?is_nonzero(S) -> + exp(Rest, Handler, [{Int, Frac, [S] ++ Exp}|Stack], Config); +e(<>, Handler, [{Int, Frac, Exp}|Stack], Config) when S =:= ?positive; S =:= ?negative -> + ex(Rest, Handler, [{Int, Frac, [S] ++ Exp}|Stack], Config); +e(<<>>, Handler, Stack, Config) -> + ?incomplete(e, <<>>, Handler, Stack, Config); +e(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -ex(<>, Handler, [{Int, Frac, Exp}|Stack], Opts) when S =:= ?zero; ?is_nonzero(S) -> - exp(Rest, Handler, [{Int, Frac, [S] ++ Exp}|Stack], Opts); -ex(<<>>, Handler, Stack, Opts) -> - ?incomplete(ex, <<>>, Handler, Stack, Opts); -ex(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +ex(<>, Handler, [{Int, Frac, Exp}|Stack], Config) when S =:= ?zero; ?is_nonzero(S) -> + exp(Rest, Handler, [{Int, Frac, [S] ++ Exp}|Stack], Config); +ex(<<>>, Handler, Stack, Config) -> + ?incomplete(ex, <<>>, Handler, Stack, Config); +ex(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -exp(<>, Handler, [{Int, Frac, Exp}|Stack], Opts) when S =:= ?zero; ?is_nonzero(S) -> - exp(Rest, Handler, [{Int, Frac, [S] ++ Exp}|Stack], Opts); -exp(<>, Handler, [Acc, object|Stack], Opts) -> - maybe_done(Rest, handle_event([format_number(Acc), end_object], Handler, Opts), Stack, Opts); -exp(<>, Handler, [Acc, array|Stack], Opts) -> - maybe_done(Rest, handle_event([format_number(Acc), end_array], Handler, Opts), Stack, Opts); -exp(<>, Handler, [Acc, object|Stack], Opts) -> - key(Rest, handle_event(format_number(Acc), Handler, Opts), [key|Stack], Opts); -exp(<>, Handler, [Acc, array|Stack], Opts) -> - value(Rest, handle_event(format_number(Acc), Handler, Opts), [array|Stack], Opts); -exp(<>, Handler, [Acc|Stack], Opts) when ?is_whitespace(S) -> - maybe_done(Rest, handle_event(format_number(Acc), Handler, Opts), Stack, Opts); -exp(<>, Handler, [Acc|Stack], Opts=#opts{comments=true}) -> - comment(Rest, handle_event(format_number(Acc), Handler, Opts), [maybe_done|Stack], Opts); -exp(<<>>, Handler, [Acc|[]], Opts = #opts{explicit_end=false}) -> - maybe_done(<<>>, handle_event(format_number(Acc), Handler, Opts), [], Opts); -exp(<<>>, Handler, Stack, Opts) -> - ?incomplete(exp, <<>>, Handler, Stack, Opts); -exp(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +exp(<>, Handler, [{Int, Frac, Exp}|Stack], Config) when S =:= ?zero; ?is_nonzero(S) -> + exp(Rest, Handler, [{Int, Frac, [S] ++ Exp}|Stack], Config); +exp(<>, Handler, [Acc, object|Stack], Config) -> + maybe_done(Rest, handle_event([format_number(Acc), end_object], Handler, Config), Stack, Config); +exp(<>, Handler, [Acc, array|Stack], Config) -> + maybe_done(Rest, handle_event([format_number(Acc), end_array], Handler, Config), Stack, Config); +exp(<>, Handler, [Acc, object|Stack], Config) -> + key(Rest, handle_event(format_number(Acc), Handler, Config), [key|Stack], Config); +exp(<>, Handler, [Acc, array|Stack], Config) -> + value(Rest, handle_event(format_number(Acc), Handler, Config), [array|Stack], Config); +exp(<>, Handler, [Acc|Stack], Config) when ?is_whitespace(S) -> + maybe_done(Rest, handle_event(format_number(Acc), Handler, Config), Stack, Config); +exp(<>, Handler, [Acc|Stack], Config=#config{comments=true}) -> + comment(Rest, handle_event(format_number(Acc), Handler, Config), [maybe_done|Stack], Config); +exp(<<>>, Handler, [Acc|[]], Config = #config{explicit_end=false}) -> + maybe_done(<<>>, handle_event(format_number(Acc), Handler, Config), [], Config); +exp(<<>>, Handler, Stack, Config) -> + ?incomplete(exp, <<>>, Handler, Stack, Config); +exp(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). format_number(Int) when is_list(Int) -> @@ -851,172 +851,172 @@ format_number({Int, Frac, Exp}) -> {float, list_to_float(lists:reverse(Exp ++ "e" ++ Frac ++ "." ++ Int))}. -tr(<<$r, Rest/binary>>, Handler, Stack, Opts) -> - tru(Rest, Handler, Stack, Opts); -tr(<<>>, Handler, Stack, Opts) -> - ?incomplete(tr, <<>>, Handler, Stack, Opts); -tr(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +tr(<<$r, Rest/binary>>, Handler, Stack, Config) -> + tru(Rest, Handler, Stack, Config); +tr(<<>>, Handler, Stack, Config) -> + ?incomplete(tr, <<>>, Handler, Stack, Config); +tr(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -tru(<<$u, Rest/binary>>, Handler, Stack, Opts) -> - true(Rest, Handler, Stack, Opts); -tru(<<>>, Handler, Stack, Opts) -> - ?incomplete(tru, <<>>, Handler, Stack, Opts); -tru(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +tru(<<$u, Rest/binary>>, Handler, Stack, Config) -> + true(Rest, Handler, Stack, Config); +tru(<<>>, Handler, Stack, Config) -> + ?incomplete(tru, <<>>, Handler, Stack, Config); +tru(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -true(<<$e, Rest/binary>>, Handler, Stack, Opts) -> - maybe_done(Rest, handle_event({literal, true}, Handler, Opts), Stack, Opts); -true(<<>>, Handler, Stack, Opts) -> - ?incomplete(true, <<>>, Handler, Stack, Opts); -true(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +true(<<$e, Rest/binary>>, Handler, Stack, Config) -> + maybe_done(Rest, handle_event({literal, true}, Handler, Config), Stack, Config); +true(<<>>, Handler, Stack, Config) -> + ?incomplete(true, <<>>, Handler, Stack, Config); +true(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -fa(<<$a, Rest/binary>>, Handler, Stack, Opts) -> - fal(Rest, Handler, Stack, Opts); -fa(<<>>, Handler, Stack, Opts) -> - ?incomplete(fa, <<>>, Handler, Stack, Opts); -fa(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +fa(<<$a, Rest/binary>>, Handler, Stack, Config) -> + fal(Rest, Handler, Stack, Config); +fa(<<>>, Handler, Stack, Config) -> + ?incomplete(fa, <<>>, Handler, Stack, Config); +fa(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -fal(<<$l, Rest/binary>>, Handler, Stack, Opts) -> - fals(Rest, Handler, Stack, Opts); -fal(<<>>, Handler, Stack, Opts) -> - ?incomplete(fal, <<>>, Handler, Stack, Opts); -fal(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +fal(<<$l, Rest/binary>>, Handler, Stack, Config) -> + fals(Rest, Handler, Stack, Config); +fal(<<>>, Handler, Stack, Config) -> + ?incomplete(fal, <<>>, Handler, Stack, Config); +fal(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -fals(<<$s, Rest/binary>>, Handler, Stack, Opts) -> - false(Rest, Handler, Stack, Opts); -fals(<<>>, Handler, Stack, Opts) -> - ?incomplete(fals, <<>>, Handler, Stack, Opts); -fals(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +fals(<<$s, Rest/binary>>, Handler, Stack, Config) -> + false(Rest, Handler, Stack, Config); +fals(<<>>, Handler, Stack, Config) -> + ?incomplete(fals, <<>>, Handler, Stack, Config); +fals(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -false(<<$e, Rest/binary>>, Handler, Stack, Opts) -> - maybe_done(Rest, handle_event({literal, false}, Handler, Opts), Stack, Opts); -false(<<>>, Handler, Stack, Opts) -> - ?incomplete(false, <<>>, Handler, Stack, Opts); -false(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +false(<<$e, Rest/binary>>, Handler, Stack, Config) -> + maybe_done(Rest, handle_event({literal, false}, Handler, Config), Stack, Config); +false(<<>>, Handler, Stack, Config) -> + ?incomplete(false, <<>>, Handler, Stack, Config); +false(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -nu(<<$u, Rest/binary>>, Handler, Stack, Opts) -> - nul(Rest, Handler, Stack, Opts); -nu(<<>>, Handler, Stack, Opts) -> - ?incomplete(nu, <<>>, Handler, Stack, Opts); -nu(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +nu(<<$u, Rest/binary>>, Handler, Stack, Config) -> + nul(Rest, Handler, Stack, Config); +nu(<<>>, Handler, Stack, Config) -> + ?incomplete(nu, <<>>, Handler, Stack, Config); +nu(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -nul(<<$l, Rest/binary>>, Handler, Stack, Opts) -> - null(Rest, Handler, Stack, Opts); -nul(<<>>, Handler, Stack, Opts) -> - ?incomplete(nul, <<>>, Handler, Stack, Opts); -nul(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +nul(<<$l, Rest/binary>>, Handler, Stack, Config) -> + null(Rest, Handler, Stack, Config); +nul(<<>>, Handler, Stack, Config) -> + ?incomplete(nul, <<>>, Handler, Stack, Config); +nul(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -null(<<$l, Rest/binary>>, Handler, Stack, Opts) -> - maybe_done(Rest, handle_event({literal, null}, Handler, Opts), Stack, Opts); -null(<<>>, Handler, Stack, Opts) -> - ?incomplete(null, <<>>, Handler, Stack, Opts); -null(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +null(<<$l, Rest/binary>>, Handler, Stack, Config) -> + maybe_done(Rest, handle_event({literal, null}, Handler, Config), Stack, Config); +null(<<>>, Handler, Stack, Config) -> + ?incomplete(null, <<>>, Handler, Stack, Config); +null(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -comment(<>, Handler, Stack, Opts) -> - single_comment(Rest, Handler, Stack, Opts); -comment(<>, Handler, Stack, Opts) -> - multi_comment(Rest, Handler, Stack, Opts); -comment(<<>>, Handler, Stack, Opts) -> - ?incomplete(comment, <<>>, Handler, Stack, Opts); -comment(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +comment(<>, Handler, Stack, Config) -> + single_comment(Rest, Handler, Stack, Config); +comment(<>, Handler, Stack, Config) -> + multi_comment(Rest, Handler, Stack, Config); +comment(<<>>, Handler, Stack, Config) -> + ?incomplete(comment, <<>>, Handler, Stack, Config); +comment(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -single_comment(<>, Handler, Stack, Opts) -> - end_comment(Rest, Handler, Stack, Opts); -single_comment(<<_/utf8, Rest/binary>>, Handler, Stack, Opts) -> - single_comment(Rest, Handler, Stack, Opts); -single_comment(<<>>, Handler, [done], Opts=#opts{explicit_end=false}) -> - end_comment(<<>>, Handler, [done], Opts); -single_comment(<<>>, Handler, Stack, Opts) -> - ?incomplete(single_comment, <<>>, Handler, Stack, Opts); -single_comment(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +single_comment(<>, Handler, Stack, Config) -> + end_comment(Rest, Handler, Stack, Config); +single_comment(<<_/utf8, Rest/binary>>, Handler, Stack, Config) -> + single_comment(Rest, Handler, Stack, Config); +single_comment(<<>>, Handler, [done], Config=#config{explicit_end=false}) -> + end_comment(<<>>, Handler, [done], Config); +single_comment(<<>>, Handler, Stack, Config) -> + ?incomplete(single_comment, <<>>, Handler, Stack, Config); +single_comment(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -multi_comment(<>, Handler, Stack, Opts) -> - end_multi_comment(Rest, Handler, Stack, Opts); -multi_comment(<<_S/utf8, Rest/binary>>, Handler, Stack, Opts) -> - multi_comment(Rest, Handler, Stack, Opts); -multi_comment(<<>>, Handler, Stack, Opts) -> - ?incomplete(multi_comment, <<>>, Handler, Stack, Opts); -multi_comment(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +multi_comment(<>, Handler, Stack, Config) -> + end_multi_comment(Rest, Handler, Stack, Config); +multi_comment(<<_S/utf8, Rest/binary>>, Handler, Stack, Config) -> + multi_comment(Rest, Handler, Stack, Config); +multi_comment(<<>>, Handler, Stack, Config) -> + ?incomplete(multi_comment, <<>>, Handler, Stack, Config); +multi_comment(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -end_multi_comment(<>, Handler, Stack, Opts) -> - end_comment(Rest, Handler, Stack, Opts); -end_multi_comment(<<_S/utf8, Rest/binary>>, Handler, Stack, Opts) -> - multi_comment(Rest, Handler, Stack, Opts); -end_multi_comment(<<>>, Handler, Stack, Opts) -> - ?incomplete(end_multi_comment, <<>>, Handler, Stack, Opts); -end_multi_comment(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +end_multi_comment(<>, Handler, Stack, Config) -> + end_comment(Rest, Handler, Stack, Config); +end_multi_comment(<<_S/utf8, Rest/binary>>, Handler, Stack, Config) -> + multi_comment(Rest, Handler, Stack, Config); +end_multi_comment(<<>>, Handler, Stack, Config) -> + ?incomplete(end_multi_comment, <<>>, Handler, Stack, Config); +end_multi_comment(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -end_comment(Rest, Handler, [Resume|Stack], Opts) -> +end_comment(Rest, Handler, [Resume|Stack], Config) -> case Resume of - value -> value(Rest, Handler, Stack, Opts) - ; object -> object(Rest, Handler, Stack, Opts) - ; array -> array(Rest, Handler, Stack, Opts) - ; colon -> colon(Rest, Handler, Stack, Opts) - ; key -> key(Rest, Handler, Stack, Opts) - ; maybe_done -> maybe_done(Rest, Handler, Stack, Opts) - ; done -> done(Rest, Handler, Stack, Opts) + value -> value(Rest, Handler, Stack, Config) + ; object -> object(Rest, Handler, Stack, Config) + ; array -> array(Rest, Handler, Stack, Config) + ; colon -> colon(Rest, Handler, Stack, Config) + ; key -> key(Rest, Handler, Stack, Config) + ; maybe_done -> maybe_done(Rest, Handler, Stack, Config) + ; done -> done(Rest, Handler, Stack, Config) end. -maybe_done(Rest, Handler, [], Opts) -> - done(Rest, handle_event(end_json, Handler, Opts), [], Opts); -maybe_done(<>, Handler, [object|Stack], Opts) -> - maybe_done(Rest, handle_event(end_object, Handler, Opts), Stack, Opts); -maybe_done(<>, Handler, [array|Stack], Opts) -> - maybe_done(Rest, handle_event(end_array, Handler, Opts), Stack, Opts); -maybe_done(<>, Handler, [object|Stack], Opts) -> - key(Rest, Handler, [key|Stack], Opts); -maybe_done(<>, Handler, [array|_] = Stack, Opts) -> - value(Rest, Handler, Stack, Opts); -maybe_done(<>, Handler, Stack, Opts) when ?is_whitespace(S) -> - maybe_done(Rest, Handler, Stack, Opts); -maybe_done(<>, Handler, Stack, Opts=#opts{comments=true}) -> - comment(Rest, Handler, [maybe_done|Stack], Opts); -maybe_done(<<>>, Handler, Stack, Opts) when length(Stack) > 0 -> - ?incomplete(maybe_done, <<>>, Handler, Stack, Opts); -maybe_done(Bin, Handler, Stack, Opts) -> - ?error([Bin, Handler, Stack, Opts]). +maybe_done(Rest, Handler, [], Config) -> + done(Rest, handle_event(end_json, Handler, Config), [], Config); +maybe_done(<>, Handler, [object|Stack], Config) -> + maybe_done(Rest, handle_event(end_object, Handler, Config), Stack, Config); +maybe_done(<>, Handler, [array|Stack], Config) -> + maybe_done(Rest, handle_event(end_array, Handler, Config), Stack, Config); +maybe_done(<>, Handler, [object|Stack], Config) -> + key(Rest, Handler, [key|Stack], Config); +maybe_done(<>, Handler, [array|_] = Stack, Config) -> + value(Rest, Handler, Stack, Config); +maybe_done(<>, Handler, Stack, Config) when ?is_whitespace(S) -> + maybe_done(Rest, Handler, Stack, Config); +maybe_done(<>, Handler, Stack, Config=#config{comments=true}) -> + comment(Rest, Handler, [maybe_done|Stack], Config); +maybe_done(<<>>, Handler, Stack, Config) when length(Stack) > 0 -> + ?incomplete(maybe_done, <<>>, Handler, Stack, Config); +maybe_done(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). -done(<>, Handler, [], Opts) when ?is_whitespace(S) -> - done(Rest, Handler, [], Opts); -done(<>, Handler, [], Opts=#opts{comments=true}) -> - comment(Rest, Handler, [done], Opts); -done(<<>>, {Handler, State}, [], Opts = #opts{explicit_end=true}) -> +done(<>, Handler, [], Config) when ?is_whitespace(S) -> + done(Rest, Handler, [], Config); +done(<>, Handler, [], Config=#config{comments=true}) -> + comment(Rest, Handler, [done], Config); +done(<<>>, {Handler, State}, [], Config = #config{explicit_end=true}) -> {incomplete, fun(Stream) when is_binary(Stream) -> - done(<>, {Handler, State}, [], Opts) + done(<>, {Handler, State}, [], Config) ; (end_stream) -> State end }; -done(<<>>, {_Handler, State}, [], _Opts) -> State; -done(Bin, Handler, Stack, Opts) -> ?error([Bin, Handler, Stack, Opts]). +done(<<>>, {_Handler, State}, [], _Config) -> State; +done(Bin, Handler, Stack, Config) -> ?error([Bin, Handler, Stack, Config]). @@ -1026,9 +1026,9 @@ done(Bin, Handler, Stack, Opts) -> ?error([Bin, Handler, Stack, Opts]). xcode(Bin) -> xcode(Bin, []). -xcode(Bin, Opts) -> +xcode(Bin, Config) -> Size = size(Bin), - try jsx:to_term(<<34, Bin:Size/binary, 34>>, Opts) + try jsx:to_term(<<34, Bin:Size/binary, 34>>, Config) catch error:badarg -> {error, badarg} end. @@ -1225,9 +1225,9 @@ bad_utf8_test_() -> ]. -decode(JSON, Opts) -> +decode(JSON, Config) -> try - (decoder(jsx, [], Opts))(JSON) + (decoder(jsx, [], Config))(JSON) catch error:badarg -> {error, badarg} end. @@ -1551,16 +1551,16 @@ check_replaced(List) -> check_good(List) -> check_good(List, []). -check_good(List, Opts) -> +check_good(List, Config) -> [] == lists:dropwhile(fun({_, [{string, _}|_]}) -> true ; (_) -> false end, - check(List, Opts, []) + check(List, Config, []) ). -check([], _Opts, Acc) -> Acc; -check([H|T], Opts, Acc) -> - R = decode(to_fake_utf(H, utf8), Opts), - check(T, Opts, [{H, R}] ++ Acc). +check([], _Config, Acc) -> Acc; +check([H|T], Config, Acc) -> + R = decode(to_fake_utf(H, utf8), Config), + check(T, Config, [{H, R}] ++ Acc). noncharacters() -> lists:seq(16#fffe, 16#ffff). @@ -1619,7 +1619,7 @@ decode_test_() -> { Title, ?_assertEqual( Events ++ [end_json], - start(JSON, {jsx, []}, [], #opts{}) + start(JSON, {jsx, []}, [], #config{}) ) } || {Title, JSON, _, Events} <- Data ]. diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index ac2a72d..3d7f78d 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -25,20 +25,20 @@ -export([encoder/3, pre_encode/2]). --spec encoder(Handler::module(), State::any(), Opts::jsx:opts()) -> jsx:encoder(). +-spec encoder(Handler::module(), State::any(), Config::jsx:config()) -> jsx:encoder(). -encoder(Handler, State, Opts) -> +encoder(Handler, State, Config) -> fun(JSON) -> start( JSON, {Handler, Handler:init(State)}, - jsx_utils:parse_opts(Opts) + jsx_utils:parse_config(Config) ) end. --include("jsx_opts.hrl"). +-include("jsx_config.hrl"). -ifndef(error). @@ -48,87 +48,87 @@ encoder(Handler, State, Opts) -> -endif. -start(Term, {Handler, State}, Opts) -> - Handler:handle_event(end_json, value(pre_encode(Term, Opts), {Handler, State}, Opts)). +start(Term, {Handler, State}, Config) -> + Handler:handle_event(end_json, value(pre_encode(Term, Config), {Handler, State}, Config)). -value(String, {Handler, State}, Opts) when is_binary(String) -> - Handler:handle_event({string, clean_string(String, Opts)}, State); -value(Float, {Handler, State}, _Opts) when is_float(Float) -> +value(String, {Handler, State}, Config) when is_binary(String) -> + Handler:handle_event({string, clean_string(String, Config)}, State); +value(Float, {Handler, State}, _Config) when is_float(Float) -> Handler:handle_event({float, Float}, State); -value(Int, {Handler, State}, _Opts) when is_integer(Int) -> +value(Int, {Handler, State}, _Config) when is_integer(Int) -> Handler:handle_event({integer, Int}, State); -value(Literal, {Handler, State}, _Opts) +value(Literal, {Handler, State}, _Config) when Literal == true; Literal == false; Literal == null -> Handler:handle_event({literal, Literal}, State); -value([{}], {Handler, State}, _Opts) -> +value([{}], {Handler, State}, _Config) -> Handler:handle_event(end_object, Handler:handle_event(start_object, State)); -value([], {Handler, State}, _Opts) -> +value([], {Handler, State}, _Config) -> Handler:handle_event(end_array, Handler:handle_event(start_array, State)); -value([Tuple|_] = List, Handler, Opts) when is_tuple(Tuple) -> - list_or_object(List, Handler, Opts); -value(List, Handler, Opts) when is_list(List) -> - list_or_object(List, Handler, Opts); -value(Term, Handler, Opts) -> ?error([Term, Handler, Opts]). +value([Tuple|_] = List, Handler, Config) when is_tuple(Tuple) -> + list_or_object(List, Handler, Config); +value(List, Handler, Config) when is_list(List) -> + list_or_object(List, Handler, Config); +value(Term, Handler, Config) -> ?error([Term, Handler, Config]). -list_or_object([Term|Rest], {Handler, State}, Opts) -> - case pre_encode(Term, Opts) of +list_or_object([Term|Rest], {Handler, State}, Config) -> + case pre_encode(Term, Config) of {K, V} -> - object([{K, V}|Rest], {Handler, Handler:handle_event(start_object, State)}, Opts) + object([{K, V}|Rest], {Handler, Handler:handle_event(start_object, State)}, Config) ; T -> - list([T|Rest], {Handler, Handler:handle_event(start_array, State)}, Opts) + list([T|Rest], {Handler, Handler:handle_event(start_array, State)}, Config) end. -object([{Key, Value}, Next|Rest], {Handler, State}, Opts) when is_atom(Key); is_binary(Key) -> - V = pre_encode(Value, Opts), +object([{Key, Value}, Next|Rest], {Handler, State}, Config) when is_atom(Key); is_binary(Key) -> + V = pre_encode(Value, Config), object( - [pre_encode(Next, Opts)|Rest], + [pre_encode(Next, Config)|Rest], { Handler, value( V, - {Handler, Handler:handle_event({key, clean_string(fix_key(Key), Opts)}, State)}, - Opts + {Handler, Handler:handle_event({key, clean_string(fix_key(Key), Config)}, State)}, + Config ) }, - Opts + Config ); -object([{Key, Value}], {Handler, State}, Opts) when is_atom(Key); is_binary(Key) -> +object([{Key, Value}], {Handler, State}, Config) when is_atom(Key); is_binary(Key) -> object( [], { Handler, value( - pre_encode(Value, Opts), - {Handler, Handler:handle_event({key, clean_string(fix_key(Key), Opts)}, State)}, - Opts + pre_encode(Value, Config), + {Handler, Handler:handle_event({key, clean_string(fix_key(Key), Config)}, State)}, + Config ) }, - Opts + Config ); -object([], {Handler, State}, _Opts) -> Handler:handle_event(end_object, State); -object(Term, Handler, Opts) -> ?error([Term, Handler, Opts]). +object([], {Handler, State}, _Config) -> Handler:handle_event(end_object, State); +object(Term, Handler, Config) -> ?error([Term, Handler, Config]). -list([Value, Next|Rest], {Handler, State}, Opts) -> - list([pre_encode(Next, Opts)|Rest], {Handler, value(Value, {Handler, State}, Opts)}, Opts); -list([Value], {Handler, State}, Opts) -> - list([], {Handler, value(Value, {Handler, State}, Opts)}, Opts); -list([], {Handler, State}, _Opts) -> Handler:handle_event(end_array, State); -list(Term, Handler, Opts) -> ?error([Term, Handler, Opts]). +list([Value, Next|Rest], {Handler, State}, Config) -> + list([pre_encode(Next, Config)|Rest], {Handler, value(Value, {Handler, State}, Config)}, Config); +list([Value], {Handler, State}, Config) -> + list([], {Handler, value(Value, {Handler, State}, Config)}, Config); +list([], {Handler, State}, _Config) -> Handler:handle_event(end_array, State); +list(Term, Handler, Config) -> ?error([Term, Handler, Config]). -pre_encode(Value, #opts{pre_encode=false}) -> io:format("~p~n", [Value]), Value; -pre_encode(Value, Opts) -> (Opts#opts.pre_encode)(Value). +pre_encode(Value, #config{pre_encode=false}) -> io:format("~p~n", [Value]), Value; +pre_encode(Value, Config) -> (Config#config.pre_encode)(Value). fix_key(Key) when is_atom(Key) -> fix_key(atom_to_binary(Key, utf8)); fix_key(Key) when is_binary(Key) -> Key. -clean_string(Bin, Opts) -> jsx_utils:clean_string(Bin, Opts). +clean_string(Bin, Config) -> jsx_utils:clean_string(Bin, Config). @@ -142,7 +142,7 @@ encode_test_() -> { Title, ?_assertEqual( Events ++ [end_json], - start(Term, {jsx, []}, #opts{}) + start(Term, {jsx, []}, #config{}) ) } || {Title, _, Term, Events} <- Data ]. diff --git a/src/jsx_parser.erl b/src/jsx_parser.erl index c67eefd..f5030be 100644 --- a/src/jsx_parser.erl +++ b/src/jsx_parser.erl @@ -26,13 +26,13 @@ -export([parser/3]). --spec parser(Handler::module(), State::any(), Opts::jsx:opts()) -> jsx:parser(). +-spec parser(Handler::module(), State::any(), Config::jsx:config()) -> jsx:parser(). -parser(Handler, State, Opts) -> - fun(Tokens) -> value(Tokens, {Handler, Handler:init(State)}, [], jsx_utils:parse_opts(Opts)) end. +parser(Handler, State, Config) -> + fun(Tokens) -> value(Tokens, {Handler, Handler:init(State)}, [], jsx_utils:parse_config(Config)) end. --include("jsx_opts.hrl"). +-include("jsx_config.hrl"). %% error, incomplete and event macros @@ -44,124 +44,124 @@ parser(Handler, State, Opts) -> -ifndef(incomplete). --define(incomplete(State, Handler, Stack, Opts), +-define(incomplete(State, Handler, Stack, Config), {incomplete, fun(end_stream) -> case State([end_json], Handler, Stack, - Opts) of - {incomplete, _} -> ?error([Handler, Stack, Opts]) + Config) of + {incomplete, _} -> ?error([Handler, Stack, Config]) ; Events -> Events end ; (Tokens) -> - State(Tokens, Handler, Stack, Opts) + State(Tokens, Handler, Stack, Config) end } ). -endif. -handle_event([], Handler, _Opts) -> Handler; -handle_event([Event|Rest], Handler, Opts) -> handle_event(Rest, handle_event(Event, Handler, Opts), Opts); -handle_event(Event, {Handler, State}, _Opts) -> {Handler, Handler:handle_event(Event, State)}. +handle_event([], Handler, _Config) -> Handler; +handle_event([Event|Rest], Handler, Config) -> handle_event(Rest, handle_event(Event, Handler, Config), Config); +handle_event(Event, {Handler, State}, _Config) -> {Handler, Handler:handle_event(Event, State)}. -value([start_object|Tokens], Handler, Stack, Opts) -> - object(Tokens, handle_event(start_object, Handler, Opts), [object|Stack], Opts); -value([start_array|Tokens], Handler, Stack, Opts) -> - array(Tokens, handle_event(start_array, Handler, Opts), [array|Stack], Opts); -value([{literal, true}|Tokens], Handler, [], Opts) -> - done(Tokens, handle_event({literal, true}, Handler, Opts), [], Opts); -value([{literal, false}|Tokens], Handler, [], Opts) -> - done(Tokens, handle_event({literal, false}, Handler, Opts), [], Opts); -value([{literal, null}|Tokens], Handler, [], Opts) -> - done(Tokens, handle_event({literal, null}, Handler, Opts), [], Opts); -value([{literal, true}|Tokens], Handler, Stack, Opts) -> - maybe_done(Tokens, handle_event({literal, true}, Handler, Opts), Stack, Opts); -value([{literal, false}|Tokens], Handler, Stack, Opts) -> - maybe_done(Tokens, handle_event({literal, false}, Handler, Opts), Stack, Opts); -value([{literal, null}|Tokens], Handler, Stack, Opts) -> - maybe_done(Tokens, handle_event({literal, null}, Handler, Opts), Stack, Opts); -value([Literal|Tokens], Handler, Stack, Opts) when Literal == true; Literal == false; Literal == null -> - value([{literal, Literal}] ++ Tokens, Handler, Stack, Opts); -value([{integer, Number}|Tokens], Handler, [], Opts) when is_integer(Number) -> - done(Tokens, handle_event({integer, Number}, Handler, Opts), [], Opts); -value([{float, Number}|Tokens], Handler, [], Opts) when is_float(Number) -> - done(Tokens, handle_event({float, Number}, Handler, Opts), [], Opts); -value([{integer, Number}|Tokens], Handler, Stack, Opts) when is_integer(Number) -> - maybe_done(Tokens, handle_event({integer, Number}, Handler, Opts), Stack, Opts); -value([{float, Number}|Tokens], Handler, Stack, Opts) when is_float(Number) -> - maybe_done(Tokens, handle_event({float, Number}, Handler, Opts), Stack, Opts); -value([{number, Number}|Tokens], Handler, Stack, Opts) when is_integer(Number) -> - value([{integer, Number}] ++ Tokens, Handler, Stack, Opts); -value([{number, Number}|Tokens], Handler, Stack, Opts) when is_float(Number) -> - value([{float, Number}] ++ Tokens, Handler, Stack, Opts); -value([Number|Tokens], Handler, Stack, Opts) when is_integer(Number) -> - value([{integer, Number}] ++ Tokens, Handler, Stack, Opts); -value([Number|Tokens], Handler, Stack, Opts) when is_float(Number) -> - value([{float, Number}] ++ Tokens, Handler, Stack, Opts); -value([{string, String}|Tokens], Handler, [], Opts) when is_binary(String) -> - done(Tokens, handle_event({string, clean_string(String, Opts)}, Handler, Opts), [], Opts); -value([{string, String}|Tokens], Handler, Stack, Opts) when is_binary(String) -> - maybe_done(Tokens, handle_event({string, clean_string(String, Opts)}, Handler, Opts), Stack, Opts); -value([String|Tokens], Handler, Stack, Opts) when is_binary(String) -> - value([{string, String}] ++ Tokens, Handler, Stack, Opts); -value([], Handler, Stack, Opts) -> - ?incomplete(value, Handler, Stack, Opts); -value(BadTokens, Handler, Stack, Opts) when is_list(BadTokens) -> - ?error([BadTokens, Handler, Stack, Opts]); -value(Token, Handler, Stack, Opts) -> - value([Token], Handler, Stack, Opts). +value([start_object|Tokens], Handler, Stack, Config) -> + object(Tokens, handle_event(start_object, Handler, Config), [object|Stack], Config); +value([start_array|Tokens], Handler, Stack, Config) -> + array(Tokens, handle_event(start_array, Handler, Config), [array|Stack], Config); +value([{literal, true}|Tokens], Handler, [], Config) -> + done(Tokens, handle_event({literal, true}, Handler, Config), [], Config); +value([{literal, false}|Tokens], Handler, [], Config) -> + done(Tokens, handle_event({literal, false}, Handler, Config), [], Config); +value([{literal, null}|Tokens], Handler, [], Config) -> + done(Tokens, handle_event({literal, null}, Handler, Config), [], Config); +value([{literal, true}|Tokens], Handler, Stack, Config) -> + maybe_done(Tokens, handle_event({literal, true}, Handler, Config), Stack, Config); +value([{literal, false}|Tokens], Handler, Stack, Config) -> + maybe_done(Tokens, handle_event({literal, false}, Handler, Config), Stack, Config); +value([{literal, null}|Tokens], Handler, Stack, Config) -> + maybe_done(Tokens, handle_event({literal, null}, Handler, Config), Stack, Config); +value([Literal|Tokens], Handler, Stack, Config) when Literal == true; Literal == false; Literal == null -> + value([{literal, Literal}] ++ Tokens, Handler, Stack, Config); +value([{integer, Number}|Tokens], Handler, [], Config) when is_integer(Number) -> + done(Tokens, handle_event({integer, Number}, Handler, Config), [], Config); +value([{float, Number}|Tokens], Handler, [], Config) when is_float(Number) -> + done(Tokens, handle_event({float, Number}, Handler, Config), [], Config); +value([{integer, Number}|Tokens], Handler, Stack, Config) when is_integer(Number) -> + maybe_done(Tokens, handle_event({integer, Number}, Handler, Config), Stack, Config); +value([{float, Number}|Tokens], Handler, Stack, Config) when is_float(Number) -> + maybe_done(Tokens, handle_event({float, Number}, Handler, Config), Stack, Config); +value([{number, Number}|Tokens], Handler, Stack, Config) when is_integer(Number) -> + value([{integer, Number}] ++ Tokens, Handler, Stack, Config); +value([{number, Number}|Tokens], Handler, Stack, Config) when is_float(Number) -> + value([{float, Number}] ++ Tokens, Handler, Stack, Config); +value([Number|Tokens], Handler, Stack, Config) when is_integer(Number) -> + value([{integer, Number}] ++ Tokens, Handler, Stack, Config); +value([Number|Tokens], Handler, Stack, Config) when is_float(Number) -> + value([{float, Number}] ++ Tokens, Handler, Stack, Config); +value([{string, String}|Tokens], Handler, [], Config) when is_binary(String) -> + done(Tokens, handle_event({string, clean_string(String, Config)}, Handler, Config), [], Config); +value([{string, String}|Tokens], Handler, Stack, Config) when is_binary(String) -> + maybe_done(Tokens, handle_event({string, clean_string(String, Config)}, Handler, Config), Stack, Config); +value([String|Tokens], Handler, Stack, Config) when is_binary(String) -> + value([{string, String}] ++ Tokens, Handler, Stack, Config); +value([], Handler, Stack, Config) -> + ?incomplete(value, Handler, Stack, Config); +value(BadTokens, Handler, Stack, Config) when is_list(BadTokens) -> + ?error([BadTokens, Handler, Stack, Config]); +value(Token, Handler, Stack, Config) -> + value([Token], Handler, Stack, Config). -object([end_object|Tokens], Handler, [object|Stack], Opts) -> - maybe_done(Tokens, handle_event(end_object, Handler, Opts), Stack, Opts); -object([{key, Key}|Tokens], Handler, Stack, Opts) when is_atom(Key); is_binary(Key) -> - value(Tokens, handle_event({key, clean_string(fix_key(Key), Opts)}, Handler, Opts), Stack, Opts); -object([Key|Tokens], Handler, Stack, Opts) when is_atom(Key); is_binary(Key) -> - value(Tokens, handle_event({key, clean_string(fix_key(Key), Opts)}, Handler, Opts), Stack, Opts); -object([], Handler, Stack, Opts) -> - ?incomplete(object, Handler, Stack, Opts); -object(BadTokens, Handler, Stack, Opts) when is_list(BadTokens) -> - ?error([BadTokens, Handler, Stack, Opts]); -object(Token, Handler, Stack, Opts) -> - object([Token], Handler, Stack, Opts). +object([end_object|Tokens], Handler, [object|Stack], Config) -> + maybe_done(Tokens, handle_event(end_object, Handler, Config), Stack, Config); +object([{key, Key}|Tokens], Handler, Stack, Config) when is_atom(Key); is_binary(Key) -> + value(Tokens, handle_event({key, clean_string(fix_key(Key), Config)}, Handler, Config), Stack, Config); +object([Key|Tokens], Handler, Stack, Config) when is_atom(Key); is_binary(Key) -> + value(Tokens, handle_event({key, clean_string(fix_key(Key), Config)}, Handler, Config), Stack, Config); +object([], Handler, Stack, Config) -> + ?incomplete(object, Handler, Stack, Config); +object(BadTokens, Handler, Stack, Config) when is_list(BadTokens) -> + ?error([BadTokens, Handler, Stack, Config]); +object(Token, Handler, Stack, Config) -> + object([Token], Handler, Stack, Config). -array([end_array|Tokens], Handler, [array|Stack], Opts) -> - maybe_done(Tokens, handle_event(end_array, Handler, Opts), Stack, Opts); -array([], Handler, Stack, Opts) -> - ?incomplete(array, Handler, Stack, Opts); -array(Tokens, Handler, Stack, Opts) when is_list(Tokens) -> - value(Tokens, Handler, Stack, Opts); -array(Token, Handler, Stack, Opts) -> - array([Token], Handler, Stack, Opts). +array([end_array|Tokens], Handler, [array|Stack], Config) -> + maybe_done(Tokens, handle_event(end_array, Handler, Config), Stack, Config); +array([], Handler, Stack, Config) -> + ?incomplete(array, Handler, Stack, Config); +array(Tokens, Handler, Stack, Config) when is_list(Tokens) -> + value(Tokens, Handler, Stack, Config); +array(Token, Handler, Stack, Config) -> + array([Token], Handler, Stack, Config). -maybe_done([end_json], Handler, [], Opts) -> - done([], Handler, [], Opts); -maybe_done(Tokens, Handler, [object|_] = Stack, Opts) when is_list(Tokens) -> - object(Tokens, Handler, Stack, Opts); -maybe_done(Tokens, Handler, [array|_] = Stack, Opts) when is_list(Tokens) -> - array(Tokens, Handler, Stack, Opts); -maybe_done([], Handler, Stack, Opts) -> - ?incomplete(maybe_done, Handler, Stack, Opts); -maybe_done(BadTokens, Handler, Stack, Opts) when is_list(BadTokens) -> - ?error([BadTokens, Handler, Stack, Opts]); -maybe_done(Token, Handler, Stack, Opts) -> - maybe_done([Token], Handler, Stack, Opts). +maybe_done([end_json], Handler, [], Config) -> + done([], Handler, [], Config); +maybe_done(Tokens, Handler, [object|_] = Stack, Config) when is_list(Tokens) -> + object(Tokens, Handler, Stack, Config); +maybe_done(Tokens, Handler, [array|_] = Stack, Config) when is_list(Tokens) -> + array(Tokens, Handler, Stack, Config); +maybe_done([], Handler, Stack, Config) -> + ?incomplete(maybe_done, Handler, Stack, Config); +maybe_done(BadTokens, Handler, Stack, Config) when is_list(BadTokens) -> + ?error([BadTokens, Handler, Stack, Config]); +maybe_done(Token, Handler, Stack, Config) -> + maybe_done([Token], Handler, Stack, Config). -done(Tokens, Handler, [], Opts) when Tokens == [end_json]; Tokens == [] -> - {_, State} = handle_event(end_json, Handler, Opts), +done(Tokens, Handler, [], Config) when Tokens == [end_json]; Tokens == [] -> + {_, State} = handle_event(end_json, Handler, Config), State; -done(BadTokens, Handler, Stack, Opts) when is_list(BadTokens) -> - ?error([BadTokens, Handler, Stack, Opts]); -done(Token, Handler, Stack, Opts) -> - done([Token], Handler, Stack, Opts). +done(BadTokens, Handler, Stack, Config) when is_list(BadTokens) -> + ?error([BadTokens, Handler, Stack, Config]); +done(Token, Handler, Stack, Config) -> + done([Token], Handler, Stack, Config). fix_key(Key) when is_atom(Key) -> fix_key(atom_to_binary(Key, utf8)); fix_key(Key) when is_binary(Key) -> Key. -clean_string(Bin, Opts) -> jsx_utils:clean_string(Bin, Opts). +clean_string(Bin, Config) -> jsx_utils:clean_string(Bin, Config). @@ -176,7 +176,7 @@ decode_test_() -> { Title, ?_assertEqual( Events ++ [end_json], - value(Events ++ [end_json], {jsx, []}, [], #opts{}) + value(Events ++ [end_json], {jsx, []}, [], #config{}) ) } || {Title, _, _, Events} <- Data ]. diff --git a/src/jsx_to_json.erl b/src/jsx_to_json.erl index df4f4f4..2b3a8b1 100644 --- a/src/jsx_to_json.erl +++ b/src/jsx_to_json.erl @@ -27,49 +27,49 @@ -export([init/1, handle_event/2]). --record(opts, { +-record(config, { space = 0, indent = 0, depth = 0 }). --type opts() :: list(). +-type config() :: list(). --spec to_json(Source::any(), Opts::opts()) -> binary(). +-spec to_json(Source::any(), Config::config()) -> binary(). -to_json(Source, Opts) when is_list(Opts) -> - (jsx:encoder(?MODULE, Opts, jsx_utils:extract_opts(Opts ++ [escaped_strings])))(Source). +to_json(Source, Config) when is_list(Config) -> + (jsx:encoder(?MODULE, Config, jsx_utils:extract_config(Config ++ [escaped_strings])))(Source). --spec format(Source::binary(), Opts::opts()) -> binary(). +-spec format(Source::binary(), Config::config()) -> binary(). -format(Source, Opts) when is_binary(Source) andalso is_list(Opts) -> - (jsx:decoder(?MODULE, Opts, jsx_utils:extract_opts(Opts ++ [escaped_strings])))(Source). +format(Source, Config) when is_binary(Source) andalso is_list(Config) -> + (jsx:decoder(?MODULE, Config, jsx_utils:extract_config(Config ++ [escaped_strings])))(Source). -parse_opts(Opts) -> parse_opts(Opts, #opts{}). +parse_config(Config) -> parse_config(Config, #config{}). -parse_opts([{space, Val}|Rest], Opts) when is_integer(Val), Val > 0 -> - parse_opts(Rest, Opts#opts{space = Val}); -parse_opts([space|Rest], Opts) -> - parse_opts(Rest, Opts#opts{space = 1}); -parse_opts([{indent, Val}|Rest], Opts) when is_integer(Val), Val > 0 -> - parse_opts(Rest, Opts#opts{indent = Val}); -parse_opts([indent|Rest], Opts) -> - parse_opts(Rest, Opts#opts{indent = 1}); -parse_opts([{K, _}|Rest] = Options, Opts) -> +parse_config([{space, Val}|Rest], Config) when is_integer(Val), Val > 0 -> + parse_config(Rest, Config#config{space = Val}); +parse_config([space|Rest], Config) -> + parse_config(Rest, Config#config{space = 1}); +parse_config([{indent, Val}|Rest], Config) when is_integer(Val), Val > 0 -> + parse_config(Rest, Config#config{indent = Val}); +parse_config([indent|Rest], Config) -> + parse_config(Rest, Config#config{indent = 1}); +parse_config([{K, _}|Rest] = Options, Config) -> case lists:member(K, jsx_utils:valid_flags()) of - true -> parse_opts(Rest, Opts) - ; false -> erlang:error(badarg, [Options, Opts]) + true -> parse_config(Rest, Config) + ; false -> erlang:error(badarg, [Options, Config]) end; -parse_opts([K|Rest] = Options, Opts) -> +parse_config([K|Rest] = Options, Config) -> case lists:member(K, jsx_utils:valid_flags()) of - true -> parse_opts(Rest, Opts) - ; false -> erlang:error(badarg, [Options, Opts]) + true -> parse_config(Rest, Config) + ; false -> erlang:error(badarg, [Options, Config]) end; -parse_opts([], Opts) -> - Opts. +parse_config([], Config) -> + Config. @@ -85,97 +85,97 @@ parse_opts([], Opts) -> -init(Opts) -> {start, [], parse_opts(Opts)}. +init(Config) -> {start, [], parse_config(Config)}. -handle_event(Event, {start, Acc, Opts}) -> +handle_event(Event, {start, Acc, Config}) -> case Event of - {Type, Value} -> {[], [Acc, encode(Type, Value, Opts)], Opts} - ; start_object -> {[object_start], [Acc, ?start_object], Opts} - ; start_array -> {[array_start], [Acc, ?start_array], Opts} + {Type, Value} -> {[], [Acc, encode(Type, Value, Config)], Config} + ; start_object -> {[object_start], [Acc, ?start_object], Config} + ; start_array -> {[array_start], [Acc, ?start_array], Config} end; -handle_event(Event, {[object_start|Stack], Acc, OldOpts = #opts{depth = Depth}}) -> - Opts = OldOpts#opts{depth = Depth + 1}, +handle_event(Event, {[object_start|Stack], Acc, OldConfig = #config{depth = Depth}}) -> + Config = OldConfig#config{depth = Depth + 1}, case Event of {key, Key} -> - {[object_value|Stack], [Acc, indent(Opts), encode(string, Key, Opts), ?colon, space(Opts)], Opts} + {[object_value|Stack], [Acc, indent(Config), encode(string, Key, Config), ?colon, space(Config)], Config} ; end_object -> - {Stack, [Acc, ?end_object], OldOpts} + {Stack, [Acc, ?end_object], OldConfig} end; -handle_event(Event, {[object_value|Stack], Acc, Opts}) -> +handle_event(Event, {[object_value|Stack], Acc, Config}) -> case Event of {Type, Value} when Type == string; Type == literal; Type == integer; Type == float -> - {[key|Stack], [Acc, encode(Type, Value, Opts)], Opts} - ; start_object -> {[object_start, key|Stack], [Acc, ?start_object], Opts} - ; start_array -> {[array_start, key|Stack], [Acc, ?start_array], Opts} + {[key|Stack], [Acc, encode(Type, Value, Config)], Config} + ; start_object -> {[object_start, key|Stack], [Acc, ?start_object], Config} + ; start_array -> {[array_start, key|Stack], [Acc, ?start_array], Config} end; -handle_event(Event, {[key|Stack], Acc, Opts = #opts{depth = Depth}}) -> +handle_event(Event, {[key|Stack], Acc, Config = #config{depth = Depth}}) -> case Event of {key, Key} -> - {[object_value|Stack], [Acc, ?comma, indent_or_space(Opts), encode(string, Key, Opts), ?colon, space(Opts)], Opts} + {[object_value|Stack], [Acc, ?comma, indent_or_space(Config), encode(string, Key, Config), ?colon, space(Config)], Config} ; end_object -> - NewOpts = Opts#opts{depth = Depth - 1}, - {Stack, [Acc, indent(NewOpts), ?end_object], NewOpts} + NewConfig = Config#config{depth = Depth - 1}, + {Stack, [Acc, indent(NewConfig), ?end_object], NewConfig} end; -handle_event(Event, {[array_start|Stack], Acc, OldOpts = #opts{depth = Depth}}) -> - Opts = OldOpts#opts{depth = Depth + 1}, +handle_event(Event, {[array_start|Stack], Acc, OldConfig = #config{depth = Depth}}) -> + Config = OldConfig#config{depth = Depth + 1}, case Event of {Type, Value} when Type == string; Type == literal; Type == integer; Type == float -> - {[array|Stack], [Acc, indent(Opts), encode(Type, Value, Opts)], Opts} - ; start_object -> {[object_start, array|Stack], [Acc, indent(Opts), ?start_object], Opts} - ; start_array -> {[array_start, array|Stack], [Acc, indent(Opts), ?start_array], Opts} - ; end_array -> {Stack, [Acc, ?end_array], OldOpts} + {[array|Stack], [Acc, indent(Config), encode(Type, Value, Config)], Config} + ; start_object -> {[object_start, array|Stack], [Acc, indent(Config), ?start_object], Config} + ; start_array -> {[array_start, array|Stack], [Acc, indent(Config), ?start_array], Config} + ; end_array -> {Stack, [Acc, ?end_array], OldConfig} end; -handle_event(Event, {[array|Stack], Acc, Opts = #opts{depth = Depth}}) -> +handle_event(Event, {[array|Stack], Acc, Config = #config{depth = Depth}}) -> case Event of {Type, Value} when Type == string; Type == literal; Type == integer; Type == float -> - {[array|Stack], [Acc, ?comma, indent_or_space(Opts), encode(Type, Value, Opts)], Opts} + {[array|Stack], [Acc, ?comma, indent_or_space(Config), encode(Type, Value, Config)], Config} ; end_array -> - NewOpts = Opts#opts{depth = Depth - 1}, - {Stack, [Acc, indent(NewOpts), ?end_array], NewOpts} - ; start_object -> {[object_start, array|Stack], [Acc, ?comma, indent_or_space(Opts), ?start_object], Opts} - ; start_array -> {[array_start, array|Stack], [Acc, ?comma, indent_or_space(Opts), ?start_array], Opts} + NewConfig = Config#config{depth = Depth - 1}, + {Stack, [Acc, indent(NewConfig), ?end_array], NewConfig} + ; start_object -> {[object_start, array|Stack], [Acc, ?comma, indent_or_space(Config), ?start_object], Config} + ; start_array -> {[array_start, array|Stack], [Acc, ?comma, indent_or_space(Config), ?start_array], Config} end; -handle_event(end_json, {[], Acc, _Opts}) -> unicode:characters_to_binary(Acc, utf8). +handle_event(end_json, {[], Acc, _Config}) -> unicode:characters_to_binary(Acc, utf8). -encode(string, String, _Opts) -> +encode(string, String, _Config) -> [?quote, String, ?quote]; -encode(literal, Literal, _Opts) -> +encode(literal, Literal, _Config) -> erlang:atom_to_list(Literal); -encode(integer, Integer, _Opts) -> +encode(integer, Integer, _Config) -> erlang:integer_to_list(Integer); -encode(float, Float, _Opts) -> +encode(float, Float, _Config) -> [Output] = io_lib:format("~p", [Float]), Output. -space(Opts) -> - case Opts#opts.space of +space(Config) -> + case Config#config.space of 0 -> [] ; X when X > 0 -> binary:copy(?space, X) end. -indent(Opts) -> - case Opts#opts.indent of +indent(Config) -> + case Config#config.indent of 0 -> [] ; X when X > 0 -> Indent = binary:copy(?space, X), - indent(Indent, Opts#opts.depth, [?newline]) + indent(Indent, Config#config.depth, [?newline]) end. indent(_Indent, 0, Acc) -> Acc; indent(Indent, N, Acc) -> indent(Indent, N - 1, [Acc, Indent]). -indent_or_space(Opts) -> - case Opts#opts.indent > 0 of - true -> indent(Opts) - ; false -> space(Opts) +indent_or_space(Config) -> + case Config#config.indent > 0 of + true -> indent(Config) + ; false -> space(Config) end. @@ -185,56 +185,56 @@ indent_or_space(Opts) -> -include_lib("eunit/include/eunit.hrl"). -opts_test_() -> +config_test_() -> [ - {"empty opts", ?_assertEqual(#opts{}, parse_opts([]))}, + {"empty config", ?_assertEqual(#config{}, parse_config([]))}, {"unspecified indent/space", ?_assertEqual( - #opts{space=1, indent=1}, - parse_opts([space, indent]) + #config{space=1, indent=1}, + parse_config([space, indent]) )}, {"specific indent", ?_assertEqual( - #opts{indent=4}, - parse_opts([{indent, 4}]) + #config{indent=4}, + parse_config([{indent, 4}]) )}, {"specific space", ?_assertEqual( - #opts{space=2}, - parse_opts([{space, 2}]) + #config{space=2}, + parse_config([{space, 2}]) )}, {"specific space and indent", ?_assertEqual( - #opts{space=2, indent=2}, - parse_opts([{space, 2}, {indent, 2}]) + #config{space=2, indent=2}, + parse_config([{space, 2}, {indent, 2}]) )}, - {"invalid opt flag", ?_assertError(badarg, parse_opts([error]))}, - {"invalid opt tuple", ?_assertError(badarg, parse_opts([{error, true}]))} + {"invalid opt flag", ?_assertError(badarg, parse_config([error]))}, + {"invalid opt tuple", ?_assertError(badarg, parse_config([{error, true}]))} ]. space_test_() -> [ - {"no space", ?_assertEqual([], space(#opts{space=0}))}, - {"one space", ?_assertEqual(<<" ">>, space(#opts{space=1}))}, - {"four spaces", ?_assertEqual(<<" ">>, space(#opts{space=4}))} + {"no space", ?_assertEqual([], space(#config{space=0}))}, + {"one space", ?_assertEqual(<<" ">>, space(#config{space=1}))}, + {"four spaces", ?_assertEqual(<<" ">>, space(#config{space=4}))} ]. indent_test_() -> [ - {"no indent", ?_assertEqual([], indent(#opts{indent=0, depth=1}))}, + {"no indent", ?_assertEqual([], indent(#config{indent=0, depth=1}))}, {"indent 1 depth 1", ?_assertEqual( [[?newline], ?space], - indent(#opts{indent=1, depth=1}) + indent(#config{indent=1, depth=1}) )}, {"indent 1 depth 2", ?_assertEqual( [[[?newline], ?space], ?space], - indent(#opts{indent=1, depth=2}) + indent(#config{indent=1, depth=2}) )}, {"indent 4 depth 1", ?_assertEqual( [[?newline], <<" ">>], - indent(#opts{indent=4, depth=1}) + indent(#config{indent=4, depth=1}) )}, {"indent 4 depth 2", ?_assertEqual( [[[?newline], <<" ">>], <<" ">>], - indent(#opts{indent=4, depth=2}) + indent(#config{indent=4, depth=2}) )} ]. @@ -243,48 +243,48 @@ indent_or_space_test_() -> [ {"no indent so space", ?_assertEqual( <<" ">>, - indent_or_space(#opts{space=1, indent=0, depth=1}) + indent_or_space(#config{space=1, indent=0, depth=1}) )}, {"indent so no space", ?_assertEqual( [[?newline], ?space], - indent_or_space(#opts{space=1, indent=1, depth=1}) + indent_or_space(#config{space=1, indent=1, depth=1}) )} ]. format_test_() -> [ - {"0.0", ?_assert(encode(float, 0.0, #opts{}) =:= "0.0")}, - {"1.0", ?_assert(encode(float, 1.0, #opts{}) =:= "1.0")}, - {"-1.0", ?_assert(encode(float, -1.0, #opts{}) =:= "-1.0")}, + {"0.0", ?_assert(encode(float, 0.0, #config{}) =:= "0.0")}, + {"1.0", ?_assert(encode(float, 1.0, #config{}) =:= "1.0")}, + {"-1.0", ?_assert(encode(float, -1.0, #config{}) =:= "-1.0")}, {"3.1234567890987654321", ?_assert( - encode(float, 3.1234567890987654321, #opts{}) =:= "3.1234567890987655") + encode(float, 3.1234567890987654321, #config{}) =:= "3.1234567890987655") }, - {"1.0e23", ?_assert(encode(float, 1.0e23, #opts{}) =:= "1.0e23")}, - {"0.3", ?_assert(encode(float, 3.0/10.0, #opts{}) =:= "0.3")}, - {"0.0001", ?_assert(encode(float, 0.0001, #opts{}) =:= "0.0001")}, - {"0.00001", ?_assert(encode(float, 0.00001, #opts{}) =:= "1.0e-5")}, - {"0.00000001", ?_assert(encode(float, 0.00000001, #opts{}) =:= "1.0e-8")}, - {"1.0e-323", ?_assert(encode(float, 1.0e-323, #opts{}) =:= "1.0e-323")}, - {"1.0e308", ?_assert(encode(float, 1.0e308, #opts{}) =:= "1.0e308")}, + {"1.0e23", ?_assert(encode(float, 1.0e23, #config{}) =:= "1.0e23")}, + {"0.3", ?_assert(encode(float, 3.0/10.0, #config{}) =:= "0.3")}, + {"0.0001", ?_assert(encode(float, 0.0001, #config{}) =:= "0.0001")}, + {"0.00001", ?_assert(encode(float, 0.00001, #config{}) =:= "1.0e-5")}, + {"0.00000001", ?_assert(encode(float, 0.00000001, #config{}) =:= "1.0e-8")}, + {"1.0e-323", ?_assert(encode(float, 1.0e-323, #config{}) =:= "1.0e-323")}, + {"1.0e308", ?_assert(encode(float, 1.0e308, #config{}) =:= "1.0e308")}, {"min normalized float", ?_assert( - encode(float, math:pow(2, -1022), #opts{}) =:= "2.2250738585072014e-308" + encode(float, math:pow(2, -1022), #config{}) =:= "2.2250738585072014e-308" ) }, {"max normalized float", ?_assert( - encode(float, (2 - math:pow(2, -52)) * math:pow(2, 1023), #opts{}) + encode(float, (2 - math:pow(2, -52)) * math:pow(2, 1023), #config{}) =:= "1.7976931348623157e308" ) }, {"min denormalized float", - ?_assert(encode(float, math:pow(2, -1074), #opts{}) =:= "5.0e-324") + ?_assert(encode(float, math:pow(2, -1074), #config{}) =:= "5.0e-324") }, {"max denormalized float", ?_assert( - encode(float, (1 - math:pow(2, -52)) * math:pow(2, -1022), #opts{}) + encode(float, (1 - math:pow(2, -52)) * math:pow(2, -1022), #config{}) =:= "2.225073858507201e-308" ) } @@ -297,7 +297,7 @@ handle_event_test_() -> { Title, ?_assertEqual( JSON, - lists:foldl(fun handle_event/2, {start, [], #opts{}}, Events ++ [end_json]) + lists:foldl(fun handle_event/2, {start, [], #config{}}, Events ++ [end_json]) ) } || {Title, JSON, _, Events} <- Data ]. diff --git a/src/jsx_to_term.erl b/src/jsx_to_term.erl index 792b6d6..b27d9b9 100644 --- a/src/jsx_to_term.erl +++ b/src/jsx_to_term.erl @@ -27,12 +27,12 @@ -export([init/1, handle_event/2]). --record(opts, { +-record(config, { labels = binary, post_decode = false }). --type opts() :: list(). +-type config() :: list(). -type json_value() :: list({binary(), json_value()}) | list(json_value()) @@ -44,74 +44,74 @@ | binary(). --spec to_term(Source::binary(), Opts::opts()) -> json_value(). +-spec to_term(Source::binary(), Config::config()) -> json_value(). -to_term(Source, Opts) when is_list(Opts) -> - (jsx:decoder(?MODULE, Opts, jsx_utils:extract_opts(Opts)))(Source). +to_term(Source, Config) when is_list(Config) -> + (jsx:decoder(?MODULE, Config, jsx_utils:extract_config(Config)))(Source). -parse_opts(Opts) -> parse_opts(Opts, #opts{}). +parse_config(Config) -> parse_config(Config, #config{}). -parse_opts([{labels, Val}|Rest], Opts) +parse_config([{labels, Val}|Rest], Config) when Val == binary; Val == atom; Val == existing_atom -> - parse_opts(Rest, Opts#opts{labels = Val}); -parse_opts([labels|Rest], Opts) -> - parse_opts(Rest, Opts#opts{labels = binary}); -parse_opts([{post_decode, F}|Rest], Opts=#opts{post_decode=false}) when is_function(F, 1) -> - parse_opts(Rest, Opts#opts{post_decode=F}); -parse_opts([{K, _}|Rest] = Options, Opts) -> + parse_config(Rest, Config#config{labels = Val}); +parse_config([labels|Rest], Config) -> + parse_config(Rest, Config#config{labels = binary}); +parse_config([{post_decode, F}|Rest], Config=#config{post_decode=false}) when is_function(F, 1) -> + parse_config(Rest, Config#config{post_decode=F}); +parse_config([{K, _}|Rest] = Options, Config) -> case lists:member(K, jsx_utils:valid_flags()) of - true -> parse_opts(Rest, Opts) - ; false -> erlang:error(badarg, [Options, Opts]) + true -> parse_config(Rest, Config) + ; false -> erlang:error(badarg, [Options, Config]) end; -parse_opts([K|Rest] = Options, Opts) -> +parse_config([K|Rest] = Options, Config) -> case lists:member(K, jsx_utils:valid_flags()) of - true -> parse_opts(Rest, Opts) - ; false -> erlang:error(badarg, [Options, Opts]) + true -> parse_config(Rest, Config) + ; false -> erlang:error(badarg, [Options, Config]) end; -parse_opts([], Opts) -> - Opts. +parse_config([], Config) -> + Config. -init(Opts) -> {[[]], parse_opts(Opts)}. +init(Config) -> {[[]], parse_config(Config)}. -handle_event(end_json, {[[Terms]], _Opts}) -> Terms; +handle_event(end_json, {[[Terms]], _Config}) -> Terms; -handle_event(start_object, {Terms, Opts}) -> {[[]|Terms], Opts}; -handle_event(end_object, {[[], {key, Key}, Last|Terms], Opts}) -> - {[[{Key, post_decode([{}], Opts)}] ++ Last] ++ Terms, Opts}; -handle_event(end_object, {[Object, {key, Key}, Last|Terms], Opts}) -> - {[[{Key, post_decode(lists:reverse(Object), Opts)}] ++ Last] ++ Terms, Opts}; -handle_event(end_object, {[[], Last|Terms], Opts}) -> - {[[post_decode([{}], Opts)] ++ Last] ++ Terms, Opts}; -handle_event(end_object, {[Object, Last|Terms], Opts}) -> - {[[post_decode(lists:reverse(Object), Opts)] ++ Last] ++ Terms, Opts}; +handle_event(start_object, {Terms, Config}) -> {[[]|Terms], Config}; +handle_event(end_object, {[[], {key, Key}, Last|Terms], Config}) -> + {[[{Key, post_decode([{}], Config)}] ++ Last] ++ Terms, Config}; +handle_event(end_object, {[Object, {key, Key}, Last|Terms], Config}) -> + {[[{Key, post_decode(lists:reverse(Object), Config)}] ++ Last] ++ Terms, Config}; +handle_event(end_object, {[[], Last|Terms], Config}) -> + {[[post_decode([{}], Config)] ++ Last] ++ Terms, Config}; +handle_event(end_object, {[Object, Last|Terms], Config}) -> + {[[post_decode(lists:reverse(Object), Config)] ++ Last] ++ Terms, Config}; -handle_event(start_array, {Terms, Opts}) -> {[[]|Terms], Opts}; -handle_event(end_array, {[List, {key, Key}, Last|Terms], Opts}) -> - {[[{Key, post_decode(lists:reverse(List), Opts)}] ++ Last] ++ Terms, Opts}; -handle_event(end_array, {[List, Last|Terms], Opts}) -> - {[[post_decode(lists:reverse(List), Opts)] ++ Last] ++ Terms, Opts}; +handle_event(start_array, {Terms, Config}) -> {[[]|Terms], Config}; +handle_event(end_array, {[List, {key, Key}, Last|Terms], Config}) -> + {[[{Key, post_decode(lists:reverse(List), Config)}] ++ Last] ++ Terms, Config}; +handle_event(end_array, {[List, Last|Terms], Config}) -> + {[[post_decode(lists:reverse(List), Config)] ++ Last] ++ Terms, Config}; -handle_event({key, Key}, {Terms, Opts}) -> {[{key, format_key(Key, Opts)}] ++ Terms, Opts}; +handle_event({key, Key}, {Terms, Config}) -> {[{key, format_key(Key, Config)}] ++ Terms, Config}; -handle_event({_, Event}, {[{key, Key}, Last|Terms], Opts}) -> - {[[{Key, post_decode(Event, Opts)}] ++ Last] ++ Terms, Opts}; -handle_event({_, Event}, {[Last|Terms], Opts}) -> - {[[post_decode(Event, Opts)] ++ Last] ++ Terms, Opts}. +handle_event({_, Event}, {[{key, Key}, Last|Terms], Config}) -> + {[[{Key, post_decode(Event, Config)}] ++ Last] ++ Terms, Config}; +handle_event({_, Event}, {[Last|Terms], Config}) -> + {[[post_decode(Event, Config)] ++ Last] ++ Terms, Config}. -format_key(Key, Opts) -> - case Opts#opts.labels of +format_key(Key, Config) -> + case Config#config.labels of binary -> Key ; atom -> binary_to_atom(Key, utf8) ; existing_atom -> binary_to_existing_atom(Key, utf8) end. -post_decode(Value, #opts{post_decode=false}) -> Value; -post_decode(Value, Opts) -> (Opts#opts.post_decode)(Value). +post_decode(Value, #config{post_decode=false}) -> Value; +post_decode(Value, Config) -> (Config#config.post_decode)(Value). %% eunit tests @@ -120,40 +120,40 @@ post_decode(Value, Opts) -> (Opts#opts.post_decode)(Value). -include_lib("eunit/include/eunit.hrl"). -opts_test_() -> +config_test_() -> %% for post_decode tests F = fun(X) -> X end, G = fun(X, Y) -> {X, Y} end, [ - {"empty opts", ?_assertEqual(#opts{}, parse_opts([]))}, - {"implicit binary labels", ?_assertEqual(#opts{}, parse_opts([labels]))}, - {"binary labels", ?_assertEqual(#opts{}, parse_opts([{labels, binary}]))}, - {"atom labels", ?_assertEqual(#opts{labels=atom}, parse_opts([{labels, atom}]))}, + {"empty config", ?_assertEqual(#config{}, parse_config([]))}, + {"implicit binary labels", ?_assertEqual(#config{}, parse_config([labels]))}, + {"binary labels", ?_assertEqual(#config{}, parse_config([{labels, binary}]))}, + {"atom labels", ?_assertEqual(#config{labels=atom}, parse_config([{labels, atom}]))}, {"existing atom labels", ?_assertEqual( - #opts{labels=existing_atom}, - parse_opts([{labels, existing_atom}]) + #config{labels=existing_atom}, + parse_config([{labels, existing_atom}]) )}, {"post decode", ?_assertEqual( - #opts{post_decode=F}, - parse_opts([{post_decode, F}]) + #config{post_decode=F}, + parse_config([{post_decode, F}]) )}, - {"post decode wrong arity", ?_assertError(badarg, parse_opts([{post_decode, G}]))}, - {"invalid opt flag", ?_assertError(badarg, parse_opts([error]))}, - {"invalid opt tuple", ?_assertError(badarg, parse_opts([{error, true}]))} + {"post decode wrong arity", ?_assertError(badarg, parse_config([{post_decode, G}]))}, + {"invalid opt flag", ?_assertError(badarg, parse_config([error]))}, + {"invalid opt tuple", ?_assertError(badarg, parse_config([{error, true}]))} ]. format_key_test_() -> [ - {"binary key", ?_assertEqual(<<"key">>, format_key(<<"key">>, #opts{labels=binary}))}, - {"atom key", ?_assertEqual(key, format_key(<<"key">>, #opts{labels=atom}))}, + {"binary key", ?_assertEqual(<<"key">>, format_key(<<"key">>, #config{labels=binary}))}, + {"atom key", ?_assertEqual(key, format_key(<<"key">>, #config{labels=atom}))}, {"existing atom key", ?_assertEqual( key, - format_key(<<"key">>, #opts{labels=existing_atom}) + format_key(<<"key">>, #config{labels=existing_atom}) )}, {"nonexisting atom key", ?_assertError( badarg, - format_key(<<"nonexistentatom">>, #opts{labels=existing_atom}) + format_key(<<"nonexistentatom">>, #config{labels=existing_atom}) )} ]. @@ -177,7 +177,7 @@ post_decoders_test_() -> [ {"no post_decode", ?_assertEqual( Events, - [ post_decode(Event, #opts{}) || Event <- Events ] + [ post_decode(Event, #config{}) || Event <- Events ] )}, {"replace arrays with empty arrays", ?_assertEqual( [ @@ -195,7 +195,7 @@ post_decoders_test_() -> 1, 1.0 ], - [ post_decode(Event, #opts{ + [ post_decode(Event, #config{ post_decode=fun([T|_] = V) when is_tuple(T) -> V; (V) when is_list(V) -> []; (V) -> V end }) || Event <- Events ] @@ -216,7 +216,7 @@ post_decoders_test_() -> 1, 1.0 ], - [ post_decode(Event, #opts{ + [ post_decode(Event, #config{ post_decode=fun([T|_]) when is_tuple(T) -> [{}]; (V) -> V end }) || Event <- Events ] @@ -237,7 +237,7 @@ post_decoders_test_() -> false, false ], - [ post_decode(Event, #opts{ + [ post_decode(Event, #config{ post_decode=fun(V) when is_list(V) -> V; (_) -> false end }) || Event <- Events ] @@ -258,7 +258,7 @@ post_decoders_test_() -> 1, 1.0 ], - [ post_decode(Event, #opts{ + [ post_decode(Event, #config{ post_decode=fun(V) when is_atom(V) -> unicode:characters_to_binary(atom_to_list(V)); (V) -> V end }) || Event <- Events ] @@ -273,7 +273,7 @@ handle_event_test_() -> { Title, ?_assertEqual( Term, - lists:foldl(fun handle_event/2, {[[]], #opts{}}, Events ++ [end_json]) + lists:foldl(fun handle_event/2, {[[]], #config{}}, Events ++ [end_json]) ) } || {Title, _, Term, Events} <- Data ]. diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index b3125d3..b50172a 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -23,70 +23,70 @@ -module(jsx_utils). --export([parse_opts/1]). --export([extract_opts/1, valid_flags/0]). +-export([parse_config/1]). +-export([extract_config/1, valid_flags/0]). -export([json_escape_sequence/1]). -export([clean_string/2]). --include("jsx_opts.hrl"). +-include("jsx_config.hrl"). -%% parsing of jsx opts -parse_opts(Opts) -> - parse_opts(Opts, #opts{}). +%% parsing of jsx config +parse_config(Config) -> + parse_config(Config, #config{}). -parse_opts([], Opts) -> - Opts; -parse_opts([replaced_bad_utf8|Rest], Opts) -> - parse_opts(Rest, Opts#opts{replaced_bad_utf8=true}); -parse_opts([escaped_forward_slashes|Rest], Opts) -> - parse_opts(Rest, Opts#opts{escaped_forward_slashes=true}); -parse_opts([explicit_end|Rest], Opts) -> - parse_opts(Rest, Opts#opts{explicit_end=true}); -parse_opts([single_quoted_strings|Rest], Opts) -> - parse_opts(Rest, Opts#opts{single_quoted_strings=true}); -parse_opts([unescaped_jsonp|Rest], Opts) -> - parse_opts(Rest, Opts#opts{unescaped_jsonp=true}); -parse_opts([comments|Rest], Opts) -> - parse_opts(Rest, Opts#opts{comments=true}); -parse_opts([escaped_strings|Rest], Opts) -> - parse_opts(Rest, Opts#opts{escaped_strings=true}); -parse_opts([dirty_strings|Rest], Opts) -> - parse_opts(Rest, Opts#opts{dirty_strings=true}); -parse_opts([ignored_bad_escapes|Rest], Opts) -> - parse_opts(Rest, Opts#opts{ignored_bad_escapes=true}); -parse_opts([relax|Rest], Opts) -> - parse_opts(Rest, Opts#opts{ +parse_config([], Config) -> + Config; +parse_config([replaced_bad_utf8|Rest], Config) -> + parse_config(Rest, Config#config{replaced_bad_utf8=true}); +parse_config([escaped_forward_slashes|Rest], Config) -> + parse_config(Rest, Config#config{escaped_forward_slashes=true}); +parse_config([explicit_end|Rest], Config) -> + parse_config(Rest, Config#config{explicit_end=true}); +parse_config([single_quoted_strings|Rest], Config) -> + parse_config(Rest, Config#config{single_quoted_strings=true}); +parse_config([unescaped_jsonp|Rest], Config) -> + parse_config(Rest, Config#config{unescaped_jsonp=true}); +parse_config([comments|Rest], Config) -> + parse_config(Rest, Config#config{comments=true}); +parse_config([escaped_strings|Rest], Config) -> + parse_config(Rest, Config#config{escaped_strings=true}); +parse_config([dirty_strings|Rest], Config) -> + parse_config(Rest, Config#config{dirty_strings=true}); +parse_config([ignored_bad_escapes|Rest], Config) -> + parse_config(Rest, Config#config{ignored_bad_escapes=true}); +parse_config([relax|Rest], Config) -> + parse_config(Rest, Config#config{ replaced_bad_utf8 = true, single_quoted_strings = true, comments = true, ignored_bad_escapes = true }); -parse_opts([{pre_encode, Encoder}|Rest] = Options, Opts) when is_function(Encoder, 1) -> - case Opts#opts.pre_encode of - false -> parse_opts(Rest, Opts#opts{pre_encode=Encoder}) - ; _ -> erlang:error(badarg, [Options, Opts]) +parse_config([{pre_encode, Encoder}|Rest] = Options, Config) when is_function(Encoder, 1) -> + case Config#config.pre_encode of + false -> parse_config(Rest, Config#config{pre_encode=Encoder}) + ; _ -> erlang:error(badarg, [Options, Config]) end; %% deprecated flags -parse_opts([{pre_encoder, Encoder}|Rest] = Options, Opts) when is_function(Encoder, 1) -> - case Opts#opts.pre_encode of - false -> parse_opts(Rest, Opts#opts{pre_encode=Encoder}) - ; _ -> erlang:error(badarg, [Options, Opts]) +parse_config([{pre_encoder, Encoder}|Rest] = Options, Config) when is_function(Encoder, 1) -> + case Config#config.pre_encode of + false -> parse_config(Rest, Config#config{pre_encode=Encoder}) + ; _ -> erlang:error(badarg, [Options, Config]) end; -parse_opts([loose_unicode|Rest], Opts) -> - parse_opts(Rest, Opts#opts{replaced_bad_utf8=true}); -parse_opts([escape_forward_slash|Rest], Opts) -> - parse_opts(Rest, Opts#opts{escaped_forward_slashes=true}); -parse_opts([single_quotes|Rest], Opts) -> - parse_opts(Rest, Opts#opts{single_quoted_strings=true}); -parse_opts([no_jsonp_escapes|Rest], Opts) -> - parse_opts(Rest, Opts#opts{unescaped_jsonp=true}); -parse_opts([json_escape|Rest], Opts) -> - parse_opts(Rest, Opts#opts{escaped_strings=true}); -parse_opts([ignore_bad_escapes|Rest], Opts) -> - parse_opts(Rest, Opts#opts{ignored_bad_escapes=true}); -parse_opts(Options, Opts) -> - erlang:error(badarg, [Options, Opts]). +parse_config([loose_unicode|Rest], Config) -> + parse_config(Rest, Config#config{replaced_bad_utf8=true}); +parse_config([escape_forward_slash|Rest], Config) -> + parse_config(Rest, Config#config{escaped_forward_slashes=true}); +parse_config([single_quotes|Rest], Config) -> + parse_config(Rest, Config#config{single_quoted_strings=true}); +parse_config([no_jsonp_escapes|Rest], Config) -> + parse_config(Rest, Config#config{unescaped_jsonp=true}); +parse_config([json_escape|Rest], Config) -> + parse_config(Rest, Config#config{escaped_strings=true}); +parse_config([ignore_bad_escapes|Rest], Config) -> + parse_config(Rest, Config#config{ignored_bad_escapes=true}); +parse_config(Options, Config) -> + erlang:error(badarg, [Options, Config]). valid_flags() -> @@ -113,19 +113,19 @@ valid_flags() -> ]. -extract_opts(Opts) -> - extract_parser_opts(Opts, []). +extract_config(Config) -> + extract_parser_config(Config, []). -extract_parser_opts([], Acc) -> Acc; -extract_parser_opts([{K,V}|Rest], Acc) -> +extract_parser_config([], Acc) -> Acc; +extract_parser_config([{K,V}|Rest], Acc) -> case lists:member(K, valid_flags()) of - true -> extract_parser_opts(Rest, [{K,V}] ++ Acc) - ; false -> extract_parser_opts(Rest, Acc) + true -> extract_parser_config(Rest, [{K,V}] ++ Acc) + ; false -> extract_parser_config(Rest, Acc) end; -extract_parser_opts([K|Rest], Acc) -> +extract_parser_config([K|Rest], Acc) -> case lists:member(K, valid_flags()) of - true -> extract_parser_opts(Rest, [K] ++ Acc) - ; false -> extract_parser_opts(Rest, Acc) + true -> extract_parser_config(Rest, [K] ++ Acc) + ; false -> extract_parser_config(Rest, Acc) end. @@ -144,10 +144,10 @@ to_hex(15) -> $f; to_hex(X) -> X + 48. %% ascii "1" is [49], "2" is [50], etc... -clean_string(Bin, #opts{dirty_strings=true}) -> Bin; -clean_string(Bin, Opts) -> - case Opts#opts.replaced_bad_utf8 orelse Opts#opts.escaped_strings of - true -> clean(Bin, [], Opts) +clean_string(Bin, #config{dirty_strings=true}) -> Bin; +clean_string(Bin, Config) -> + case Config#config.replaced_bad_utf8 orelse Config#config.escaped_strings of + true -> clean(Bin, [], Config) ; false -> ensure_clean(Bin), Bin end. @@ -305,197 +305,197 @@ ensure_clean(Bin) -> erlang:error(badarg, [Bin]). %% escape and/or replace bad codepoints if requested -clean(<<>>, Acc, _Opts) -> unicode:characters_to_binary(lists:reverse(Acc)); -clean(<<0, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(0, Opts) ++ Acc, Opts); -clean(<<1, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(1, Opts) ++ Acc, Opts); -clean(<<2, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(2, Opts) ++ Acc, Opts); -clean(<<3, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(3, Opts) ++ Acc, Opts); -clean(<<4, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(4, Opts) ++ Acc, Opts); -clean(<<5, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(5, Opts) ++ Acc, Opts); -clean(<<6, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(6, Opts) ++ Acc, Opts); -clean(<<7, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(7, Opts) ++ Acc, Opts); -clean(<<8, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(8, Opts) ++ Acc, Opts); -clean(<<9, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(9, Opts) ++ Acc, Opts); -clean(<<10, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(10, Opts) ++ Acc, Opts); -clean(<<11, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(11, Opts) ++ Acc, Opts); -clean(<<12, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(12, Opts) ++ Acc, Opts); -clean(<<13, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(13, Opts) ++ Acc, Opts); -clean(<<14, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(14, Opts) ++ Acc, Opts); -clean(<<15, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(15, Opts) ++ Acc, Opts); -clean(<<16, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(16, Opts) ++ Acc, Opts); -clean(<<17, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(17, Opts) ++ Acc, Opts); -clean(<<18, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(18, Opts) ++ Acc, Opts); -clean(<<19, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(19, Opts) ++ Acc, Opts); -clean(<<20, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(20, Opts) ++ Acc, Opts); -clean(<<21, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(21, Opts) ++ Acc, Opts); -clean(<<22, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(22, Opts) ++ Acc, Opts); -clean(<<23, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(23, Opts) ++ Acc, Opts); -clean(<<24, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(24, Opts) ++ Acc, Opts); -clean(<<25, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(25, Opts) ++ Acc, Opts); -clean(<<26, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(26, Opts) ++ Acc, Opts); -clean(<<27, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(27, Opts) ++ Acc, Opts); -clean(<<28, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(28, Opts) ++ Acc, Opts); -clean(<<29, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(29, Opts) ++ Acc, Opts); -clean(<<30, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(30, Opts) ++ Acc, Opts); -clean(<<31, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(31, Opts) ++ Acc, Opts); -clean(<<32, Rest/binary>>, Acc, Opts) -> clean(Rest, [32] ++ Acc, Opts); -clean(<<33, Rest/binary>>, Acc, Opts) -> clean(Rest, [33] ++ Acc, Opts); -clean(<<34, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(34, Opts) ++ Acc, Opts); -clean(<<35, Rest/binary>>, Acc, Opts) -> clean(Rest, [35] ++ Acc, Opts); -clean(<<36, Rest/binary>>, Acc, Opts) -> clean(Rest, [36] ++ Acc, Opts); -clean(<<37, Rest/binary>>, Acc, Opts) -> clean(Rest, [37] ++ Acc, Opts); -clean(<<38, Rest/binary>>, Acc, Opts) -> clean(Rest, [38] ++ Acc, Opts); -clean(<<39, Rest/binary>>, Acc, Opts) -> clean(Rest, [39] ++ Acc, Opts); -clean(<<40, Rest/binary>>, Acc, Opts) -> clean(Rest, [40] ++ Acc, Opts); -clean(<<41, Rest/binary>>, Acc, Opts) -> clean(Rest, [41] ++ Acc, Opts); -clean(<<42, Rest/binary>>, Acc, Opts) -> clean(Rest, [42] ++ Acc, Opts); -clean(<<43, Rest/binary>>, Acc, Opts) -> clean(Rest, [43] ++ Acc, Opts); -clean(<<44, Rest/binary>>, Acc, Opts) -> clean(Rest, [44] ++ Acc, Opts); -clean(<<45, Rest/binary>>, Acc, Opts) -> clean(Rest, [45] ++ Acc, Opts); -clean(<<46, Rest/binary>>, Acc, Opts) -> clean(Rest, [46] ++ Acc, Opts); -clean(<<47, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(47, Opts) ++ Acc, Opts); -clean(<<48, Rest/binary>>, Acc, Opts) -> clean(Rest, [48] ++ Acc, Opts); -clean(<<49, Rest/binary>>, Acc, Opts) -> clean(Rest, [49] ++ Acc, Opts); -clean(<<50, Rest/binary>>, Acc, Opts) -> clean(Rest, [50] ++ Acc, Opts); -clean(<<51, Rest/binary>>, Acc, Opts) -> clean(Rest, [51] ++ Acc, Opts); -clean(<<52, Rest/binary>>, Acc, Opts) -> clean(Rest, [52] ++ Acc, Opts); -clean(<<53, Rest/binary>>, Acc, Opts) -> clean(Rest, [53] ++ Acc, Opts); -clean(<<54, Rest/binary>>, Acc, Opts) -> clean(Rest, [54] ++ Acc, Opts); -clean(<<55, Rest/binary>>, Acc, Opts) -> clean(Rest, [55] ++ Acc, Opts); -clean(<<56, Rest/binary>>, Acc, Opts) -> clean(Rest, [56] ++ Acc, Opts); -clean(<<57, Rest/binary>>, Acc, Opts) -> clean(Rest, [57] ++ Acc, Opts); -clean(<<58, Rest/binary>>, Acc, Opts) -> clean(Rest, [58] ++ Acc, Opts); -clean(<<59, Rest/binary>>, Acc, Opts) -> clean(Rest, [59] ++ Acc, Opts); -clean(<<60, Rest/binary>>, Acc, Opts) -> clean(Rest, [60] ++ Acc, Opts); -clean(<<61, Rest/binary>>, Acc, Opts) -> clean(Rest, [61] ++ Acc, Opts); -clean(<<62, Rest/binary>>, Acc, Opts) -> clean(Rest, [62] ++ Acc, Opts); -clean(<<63, Rest/binary>>, Acc, Opts) -> clean(Rest, [63] ++ Acc, Opts); -clean(<<64, Rest/binary>>, Acc, Opts) -> clean(Rest, [64] ++ Acc, Opts); -clean(<<65, Rest/binary>>, Acc, Opts) -> clean(Rest, [65] ++ Acc, Opts); -clean(<<66, Rest/binary>>, Acc, Opts) -> clean(Rest, [66] ++ Acc, Opts); -clean(<<67, Rest/binary>>, Acc, Opts) -> clean(Rest, [67] ++ Acc, Opts); -clean(<<68, Rest/binary>>, Acc, Opts) -> clean(Rest, [68] ++ Acc, Opts); -clean(<<69, Rest/binary>>, Acc, Opts) -> clean(Rest, [69] ++ Acc, Opts); -clean(<<70, Rest/binary>>, Acc, Opts) -> clean(Rest, [70] ++ Acc, Opts); -clean(<<71, Rest/binary>>, Acc, Opts) -> clean(Rest, [71] ++ Acc, Opts); -clean(<<72, Rest/binary>>, Acc, Opts) -> clean(Rest, [72] ++ Acc, Opts); -clean(<<73, Rest/binary>>, Acc, Opts) -> clean(Rest, [73] ++ Acc, Opts); -clean(<<74, Rest/binary>>, Acc, Opts) -> clean(Rest, [74] ++ Acc, Opts); -clean(<<75, Rest/binary>>, Acc, Opts) -> clean(Rest, [75] ++ Acc, Opts); -clean(<<76, Rest/binary>>, Acc, Opts) -> clean(Rest, [76] ++ Acc, Opts); -clean(<<77, Rest/binary>>, Acc, Opts) -> clean(Rest, [77] ++ Acc, Opts); -clean(<<78, Rest/binary>>, Acc, Opts) -> clean(Rest, [78] ++ Acc, Opts); -clean(<<79, Rest/binary>>, Acc, Opts) -> clean(Rest, [79] ++ Acc, Opts); -clean(<<80, Rest/binary>>, Acc, Opts) -> clean(Rest, [80] ++ Acc, Opts); -clean(<<81, Rest/binary>>, Acc, Opts) -> clean(Rest, [81] ++ Acc, Opts); -clean(<<82, Rest/binary>>, Acc, Opts) -> clean(Rest, [82] ++ Acc, Opts); -clean(<<83, Rest/binary>>, Acc, Opts) -> clean(Rest, [83] ++ Acc, Opts); -clean(<<84, Rest/binary>>, Acc, Opts) -> clean(Rest, [84] ++ Acc, Opts); -clean(<<85, Rest/binary>>, Acc, Opts) -> clean(Rest, [85] ++ Acc, Opts); -clean(<<86, Rest/binary>>, Acc, Opts) -> clean(Rest, [86] ++ Acc, Opts); -clean(<<87, Rest/binary>>, Acc, Opts) -> clean(Rest, [87] ++ Acc, Opts); -clean(<<88, Rest/binary>>, Acc, Opts) -> clean(Rest, [88] ++ Acc, Opts); -clean(<<89, Rest/binary>>, Acc, Opts) -> clean(Rest, [89] ++ Acc, Opts); -clean(<<90, Rest/binary>>, Acc, Opts) -> clean(Rest, [90] ++ Acc, Opts); -clean(<<91, Rest/binary>>, Acc, Opts) -> clean(Rest, [91] ++ Acc, Opts); -clean(<<92, Rest/binary>>, Acc, Opts) -> clean(Rest, maybe_replace(92, Opts) ++ Acc, Opts); -clean(<<93, Rest/binary>>, Acc, Opts) -> clean(Rest, [93] ++ Acc, Opts); -clean(<<94, Rest/binary>>, Acc, Opts) -> clean(Rest, [94] ++ Acc, Opts); -clean(<<95, Rest/binary>>, Acc, Opts) -> clean(Rest, [95] ++ Acc, Opts); -clean(<<96, Rest/binary>>, Acc, Opts) -> clean(Rest, [96] ++ Acc, Opts); -clean(<<97, Rest/binary>>, Acc, Opts) -> clean(Rest, [97] ++ Acc, Opts); -clean(<<98, Rest/binary>>, Acc, Opts) -> clean(Rest, [98] ++ Acc, Opts); -clean(<<99, Rest/binary>>, Acc, Opts) -> clean(Rest, [99] ++ Acc, Opts); -clean(<<100, Rest/binary>>, Acc, Opts) -> clean(Rest, [100] ++ Acc, Opts); -clean(<<101, Rest/binary>>, Acc, Opts) -> clean(Rest, [101] ++ Acc, Opts); -clean(<<102, Rest/binary>>, Acc, Opts) -> clean(Rest, [102] ++ Acc, Opts); -clean(<<103, Rest/binary>>, Acc, Opts) -> clean(Rest, [103] ++ Acc, Opts); -clean(<<104, Rest/binary>>, Acc, Opts) -> clean(Rest, [104] ++ Acc, Opts); -clean(<<105, Rest/binary>>, Acc, Opts) -> clean(Rest, [105] ++ Acc, Opts); -clean(<<106, Rest/binary>>, Acc, Opts) -> clean(Rest, [106] ++ Acc, Opts); -clean(<<107, Rest/binary>>, Acc, Opts) -> clean(Rest, [107] ++ Acc, Opts); -clean(<<108, Rest/binary>>, Acc, Opts) -> clean(Rest, [108] ++ Acc, Opts); -clean(<<109, Rest/binary>>, Acc, Opts) -> clean(Rest, [109] ++ Acc, Opts); -clean(<<110, Rest/binary>>, Acc, Opts) -> clean(Rest, [110] ++ Acc, Opts); -clean(<<111, Rest/binary>>, Acc, Opts) -> clean(Rest, [111] ++ Acc, Opts); -clean(<<112, Rest/binary>>, Acc, Opts) -> clean(Rest, [112] ++ Acc, Opts); -clean(<<113, Rest/binary>>, Acc, Opts) -> clean(Rest, [113] ++ Acc, Opts); -clean(<<114, Rest/binary>>, Acc, Opts) -> clean(Rest, [114] ++ Acc, Opts); -clean(<<115, Rest/binary>>, Acc, Opts) -> clean(Rest, [115] ++ Acc, Opts); -clean(<<116, Rest/binary>>, Acc, Opts) -> clean(Rest, [116] ++ Acc, Opts); -clean(<<117, Rest/binary>>, Acc, Opts) -> clean(Rest, [117] ++ Acc, Opts); -clean(<<118, Rest/binary>>, Acc, Opts) -> clean(Rest, [118] ++ Acc, Opts); -clean(<<119, Rest/binary>>, Acc, Opts) -> clean(Rest, [119] ++ Acc, Opts); -clean(<<120, Rest/binary>>, Acc, Opts) -> clean(Rest, [120] ++ Acc, Opts); -clean(<<121, Rest/binary>>, Acc, Opts) -> clean(Rest, [121] ++ Acc, Opts); -clean(<<122, Rest/binary>>, Acc, Opts) -> clean(Rest, [122] ++ Acc, Opts); -clean(<<123, Rest/binary>>, Acc, Opts) -> clean(Rest, [123] ++ Acc, Opts); -clean(<<124, Rest/binary>>, Acc, Opts) -> clean(Rest, [124] ++ Acc, Opts); -clean(<<125, Rest/binary>>, Acc, Opts) -> clean(Rest, [125] ++ Acc, Opts); -clean(<<126, Rest/binary>>, Acc, Opts) -> clean(Rest, [126] ++ Acc, Opts); -clean(<<127, Rest/binary>>, Acc, Opts) -> clean(Rest, [127] ++ Acc, Opts); -clean(<>, Acc, Opts) when X < 16#800 -> - clean(Rest, [X] ++ Acc, Opts); -clean(<>, Acc, Opts) when X == 16#2028; X == 16#2029 -> - clean(Rest, maybe_replace(X, Opts) ++ Acc, Opts); -clean(<>, Acc, Opts) when X < 16#d800 -> - clean(Rest, [X] ++ Acc, Opts); -clean(<>, Acc, Opts) when X > 16#dfff, X < 16#fdd0 -> - clean(Rest, [X] ++ Acc, Opts); -clean(<>, Acc, Opts) when X > 16#fdef, X < 16#fffe -> - clean(Rest, [X] ++ Acc, Opts); -clean(<>, Acc, Opts) when X >= 16#10000, X < 16#1fffe -> - clean(Rest, [X] ++ Acc, Opts); -clean(<>, Acc, Opts) when X >= 16#20000, X < 16#2fffe -> - clean(Rest, [X] ++ Acc, Opts); -clean(<>, Acc, Opts) when X >= 16#30000, X < 16#3fffe -> - clean(Rest, [X] ++ Acc, Opts); -clean(<>, Acc, Opts) when X >= 16#40000, X < 16#4fffe -> - clean(Rest, [X] ++ Acc, Opts); -clean(<>, Acc, Opts) when X >= 16#50000, X < 16#5fffe -> - clean(Rest, [X] ++ Acc, Opts); -clean(<>, Acc, Opts) when X >= 16#60000, X < 16#6fffe -> - clean(Rest, [X] ++ Acc, Opts); -clean(<>, Acc, Opts) when X >= 16#70000, X < 16#7fffe -> - clean(Rest, [X] ++ Acc, Opts); -clean(<>, Acc, Opts) when X >= 16#80000, X < 16#8fffe -> - clean(Rest, [X] ++ Acc, Opts); -clean(<>, Acc, Opts) when X >= 16#90000, X < 16#9fffe -> - clean(Rest, [X] ++ Acc, Opts); -clean(<>, Acc, Opts) when X >= 16#a0000, X < 16#afffe -> - clean(Rest, [X] ++ Acc, Opts); -clean(<>, Acc, Opts) when X >= 16#b0000, X < 16#bfffe -> - clean(Rest, [X] ++ Acc, Opts); -clean(<>, Acc, Opts) when X >= 16#c0000, X < 16#cfffe -> - clean(Rest, [X] ++ Acc, Opts); -clean(<>, Acc, Opts) when X >= 16#d0000, X < 16#dfffe -> - clean(Rest, [X] ++ Acc, Opts); -clean(<>, Acc, Opts) when X >= 16#e0000, X < 16#efffe -> - clean(Rest, [X] ++ Acc, Opts); -clean(<>, Acc, Opts) when X >= 16#f0000, X < 16#ffffe -> - clean(Rest, [X] ++ Acc, Opts); -clean(<>, Acc, Opts) when X >= 16#100000, X < 16#10fffe -> - clean(Rest, [X] ++ Acc, Opts); +clean(<<>>, Acc, _Config) -> unicode:characters_to_binary(lists:reverse(Acc)); +clean(<<0, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(0, Config) ++ Acc, Config); +clean(<<1, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(1, Config) ++ Acc, Config); +clean(<<2, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(2, Config) ++ Acc, Config); +clean(<<3, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(3, Config) ++ Acc, Config); +clean(<<4, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(4, Config) ++ Acc, Config); +clean(<<5, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(5, Config) ++ Acc, Config); +clean(<<6, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(6, Config) ++ Acc, Config); +clean(<<7, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(7, Config) ++ Acc, Config); +clean(<<8, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(8, Config) ++ Acc, Config); +clean(<<9, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(9, Config) ++ Acc, Config); +clean(<<10, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(10, Config) ++ Acc, Config); +clean(<<11, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(11, Config) ++ Acc, Config); +clean(<<12, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(12, Config) ++ Acc, Config); +clean(<<13, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(13, Config) ++ Acc, Config); +clean(<<14, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(14, Config) ++ Acc, Config); +clean(<<15, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(15, Config) ++ Acc, Config); +clean(<<16, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(16, Config) ++ Acc, Config); +clean(<<17, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(17, Config) ++ Acc, Config); +clean(<<18, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(18, Config) ++ Acc, Config); +clean(<<19, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(19, Config) ++ Acc, Config); +clean(<<20, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(20, Config) ++ Acc, Config); +clean(<<21, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(21, Config) ++ Acc, Config); +clean(<<22, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(22, Config) ++ Acc, Config); +clean(<<23, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(23, Config) ++ Acc, Config); +clean(<<24, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(24, Config) ++ Acc, Config); +clean(<<25, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(25, Config) ++ Acc, Config); +clean(<<26, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(26, Config) ++ Acc, Config); +clean(<<27, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(27, Config) ++ Acc, Config); +clean(<<28, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(28, Config) ++ Acc, Config); +clean(<<29, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(29, Config) ++ Acc, Config); +clean(<<30, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(30, Config) ++ Acc, Config); +clean(<<31, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(31, Config) ++ Acc, Config); +clean(<<32, Rest/binary>>, Acc, Config) -> clean(Rest, [32] ++ Acc, Config); +clean(<<33, Rest/binary>>, Acc, Config) -> clean(Rest, [33] ++ Acc, Config); +clean(<<34, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(34, Config) ++ Acc, Config); +clean(<<35, Rest/binary>>, Acc, Config) -> clean(Rest, [35] ++ Acc, Config); +clean(<<36, Rest/binary>>, Acc, Config) -> clean(Rest, [36] ++ Acc, Config); +clean(<<37, Rest/binary>>, Acc, Config) -> clean(Rest, [37] ++ Acc, Config); +clean(<<38, Rest/binary>>, Acc, Config) -> clean(Rest, [38] ++ Acc, Config); +clean(<<39, Rest/binary>>, Acc, Config) -> clean(Rest, [39] ++ Acc, Config); +clean(<<40, Rest/binary>>, Acc, Config) -> clean(Rest, [40] ++ Acc, Config); +clean(<<41, Rest/binary>>, Acc, Config) -> clean(Rest, [41] ++ Acc, Config); +clean(<<42, Rest/binary>>, Acc, Config) -> clean(Rest, [42] ++ Acc, Config); +clean(<<43, Rest/binary>>, Acc, Config) -> clean(Rest, [43] ++ Acc, Config); +clean(<<44, Rest/binary>>, Acc, Config) -> clean(Rest, [44] ++ Acc, Config); +clean(<<45, Rest/binary>>, Acc, Config) -> clean(Rest, [45] ++ Acc, Config); +clean(<<46, Rest/binary>>, Acc, Config) -> clean(Rest, [46] ++ Acc, Config); +clean(<<47, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(47, Config) ++ Acc, Config); +clean(<<48, Rest/binary>>, Acc, Config) -> clean(Rest, [48] ++ Acc, Config); +clean(<<49, Rest/binary>>, Acc, Config) -> clean(Rest, [49] ++ Acc, Config); +clean(<<50, Rest/binary>>, Acc, Config) -> clean(Rest, [50] ++ Acc, Config); +clean(<<51, Rest/binary>>, Acc, Config) -> clean(Rest, [51] ++ Acc, Config); +clean(<<52, Rest/binary>>, Acc, Config) -> clean(Rest, [52] ++ Acc, Config); +clean(<<53, Rest/binary>>, Acc, Config) -> clean(Rest, [53] ++ Acc, Config); +clean(<<54, Rest/binary>>, Acc, Config) -> clean(Rest, [54] ++ Acc, Config); +clean(<<55, Rest/binary>>, Acc, Config) -> clean(Rest, [55] ++ Acc, Config); +clean(<<56, Rest/binary>>, Acc, Config) -> clean(Rest, [56] ++ Acc, Config); +clean(<<57, Rest/binary>>, Acc, Config) -> clean(Rest, [57] ++ Acc, Config); +clean(<<58, Rest/binary>>, Acc, Config) -> clean(Rest, [58] ++ Acc, Config); +clean(<<59, Rest/binary>>, Acc, Config) -> clean(Rest, [59] ++ Acc, Config); +clean(<<60, Rest/binary>>, Acc, Config) -> clean(Rest, [60] ++ Acc, Config); +clean(<<61, Rest/binary>>, Acc, Config) -> clean(Rest, [61] ++ Acc, Config); +clean(<<62, Rest/binary>>, Acc, Config) -> clean(Rest, [62] ++ Acc, Config); +clean(<<63, Rest/binary>>, Acc, Config) -> clean(Rest, [63] ++ Acc, Config); +clean(<<64, Rest/binary>>, Acc, Config) -> clean(Rest, [64] ++ Acc, Config); +clean(<<65, Rest/binary>>, Acc, Config) -> clean(Rest, [65] ++ Acc, Config); +clean(<<66, Rest/binary>>, Acc, Config) -> clean(Rest, [66] ++ Acc, Config); +clean(<<67, Rest/binary>>, Acc, Config) -> clean(Rest, [67] ++ Acc, Config); +clean(<<68, Rest/binary>>, Acc, Config) -> clean(Rest, [68] ++ Acc, Config); +clean(<<69, Rest/binary>>, Acc, Config) -> clean(Rest, [69] ++ Acc, Config); +clean(<<70, Rest/binary>>, Acc, Config) -> clean(Rest, [70] ++ Acc, Config); +clean(<<71, Rest/binary>>, Acc, Config) -> clean(Rest, [71] ++ Acc, Config); +clean(<<72, Rest/binary>>, Acc, Config) -> clean(Rest, [72] ++ Acc, Config); +clean(<<73, Rest/binary>>, Acc, Config) -> clean(Rest, [73] ++ Acc, Config); +clean(<<74, Rest/binary>>, Acc, Config) -> clean(Rest, [74] ++ Acc, Config); +clean(<<75, Rest/binary>>, Acc, Config) -> clean(Rest, [75] ++ Acc, Config); +clean(<<76, Rest/binary>>, Acc, Config) -> clean(Rest, [76] ++ Acc, Config); +clean(<<77, Rest/binary>>, Acc, Config) -> clean(Rest, [77] ++ Acc, Config); +clean(<<78, Rest/binary>>, Acc, Config) -> clean(Rest, [78] ++ Acc, Config); +clean(<<79, Rest/binary>>, Acc, Config) -> clean(Rest, [79] ++ Acc, Config); +clean(<<80, Rest/binary>>, Acc, Config) -> clean(Rest, [80] ++ Acc, Config); +clean(<<81, Rest/binary>>, Acc, Config) -> clean(Rest, [81] ++ Acc, Config); +clean(<<82, Rest/binary>>, Acc, Config) -> clean(Rest, [82] ++ Acc, Config); +clean(<<83, Rest/binary>>, Acc, Config) -> clean(Rest, [83] ++ Acc, Config); +clean(<<84, Rest/binary>>, Acc, Config) -> clean(Rest, [84] ++ Acc, Config); +clean(<<85, Rest/binary>>, Acc, Config) -> clean(Rest, [85] ++ Acc, Config); +clean(<<86, Rest/binary>>, Acc, Config) -> clean(Rest, [86] ++ Acc, Config); +clean(<<87, Rest/binary>>, Acc, Config) -> clean(Rest, [87] ++ Acc, Config); +clean(<<88, Rest/binary>>, Acc, Config) -> clean(Rest, [88] ++ Acc, Config); +clean(<<89, Rest/binary>>, Acc, Config) -> clean(Rest, [89] ++ Acc, Config); +clean(<<90, Rest/binary>>, Acc, Config) -> clean(Rest, [90] ++ Acc, Config); +clean(<<91, Rest/binary>>, Acc, Config) -> clean(Rest, [91] ++ Acc, Config); +clean(<<92, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(92, Config) ++ Acc, Config); +clean(<<93, Rest/binary>>, Acc, Config) -> clean(Rest, [93] ++ Acc, Config); +clean(<<94, Rest/binary>>, Acc, Config) -> clean(Rest, [94] ++ Acc, Config); +clean(<<95, Rest/binary>>, Acc, Config) -> clean(Rest, [95] ++ Acc, Config); +clean(<<96, Rest/binary>>, Acc, Config) -> clean(Rest, [96] ++ Acc, Config); +clean(<<97, Rest/binary>>, Acc, Config) -> clean(Rest, [97] ++ Acc, Config); +clean(<<98, Rest/binary>>, Acc, Config) -> clean(Rest, [98] ++ Acc, Config); +clean(<<99, Rest/binary>>, Acc, Config) -> clean(Rest, [99] ++ Acc, Config); +clean(<<100, Rest/binary>>, Acc, Config) -> clean(Rest, [100] ++ Acc, Config); +clean(<<101, Rest/binary>>, Acc, Config) -> clean(Rest, [101] ++ Acc, Config); +clean(<<102, Rest/binary>>, Acc, Config) -> clean(Rest, [102] ++ Acc, Config); +clean(<<103, Rest/binary>>, Acc, Config) -> clean(Rest, [103] ++ Acc, Config); +clean(<<104, Rest/binary>>, Acc, Config) -> clean(Rest, [104] ++ Acc, Config); +clean(<<105, Rest/binary>>, Acc, Config) -> clean(Rest, [105] ++ Acc, Config); +clean(<<106, Rest/binary>>, Acc, Config) -> clean(Rest, [106] ++ Acc, Config); +clean(<<107, Rest/binary>>, Acc, Config) -> clean(Rest, [107] ++ Acc, Config); +clean(<<108, Rest/binary>>, Acc, Config) -> clean(Rest, [108] ++ Acc, Config); +clean(<<109, Rest/binary>>, Acc, Config) -> clean(Rest, [109] ++ Acc, Config); +clean(<<110, Rest/binary>>, Acc, Config) -> clean(Rest, [110] ++ Acc, Config); +clean(<<111, Rest/binary>>, Acc, Config) -> clean(Rest, [111] ++ Acc, Config); +clean(<<112, Rest/binary>>, Acc, Config) -> clean(Rest, [112] ++ Acc, Config); +clean(<<113, Rest/binary>>, Acc, Config) -> clean(Rest, [113] ++ Acc, Config); +clean(<<114, Rest/binary>>, Acc, Config) -> clean(Rest, [114] ++ Acc, Config); +clean(<<115, Rest/binary>>, Acc, Config) -> clean(Rest, [115] ++ Acc, Config); +clean(<<116, Rest/binary>>, Acc, Config) -> clean(Rest, [116] ++ Acc, Config); +clean(<<117, Rest/binary>>, Acc, Config) -> clean(Rest, [117] ++ Acc, Config); +clean(<<118, Rest/binary>>, Acc, Config) -> clean(Rest, [118] ++ Acc, Config); +clean(<<119, Rest/binary>>, Acc, Config) -> clean(Rest, [119] ++ Acc, Config); +clean(<<120, Rest/binary>>, Acc, Config) -> clean(Rest, [120] ++ Acc, Config); +clean(<<121, Rest/binary>>, Acc, Config) -> clean(Rest, [121] ++ Acc, Config); +clean(<<122, Rest/binary>>, Acc, Config) -> clean(Rest, [122] ++ Acc, Config); +clean(<<123, Rest/binary>>, Acc, Config) -> clean(Rest, [123] ++ Acc, Config); +clean(<<124, Rest/binary>>, Acc, Config) -> clean(Rest, [124] ++ Acc, Config); +clean(<<125, Rest/binary>>, Acc, Config) -> clean(Rest, [125] ++ Acc, Config); +clean(<<126, Rest/binary>>, Acc, Config) -> clean(Rest, [126] ++ Acc, Config); +clean(<<127, Rest/binary>>, Acc, Config) -> clean(Rest, [127] ++ Acc, Config); +clean(<>, Acc, Config) when X < 16#800 -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X == 16#2028; X == 16#2029 -> + clean(Rest, maybe_replace(X, Config) ++ Acc, Config); +clean(<>, Acc, Config) when X < 16#d800 -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X > 16#dfff, X < 16#fdd0 -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X > 16#fdef, X < 16#fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#10000, X < 16#1fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#20000, X < 16#2fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#30000, X < 16#3fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#40000, X < 16#4fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#50000, X < 16#5fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#60000, X < 16#6fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#70000, X < 16#7fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#80000, X < 16#8fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#90000, X < 16#9fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#a0000, X < 16#afffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#b0000, X < 16#bfffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#c0000, X < 16#cfffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#d0000, X < 16#dfffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#e0000, X < 16#efffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#f0000, X < 16#ffffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#100000, X < 16#10fffe -> + clean(Rest, [X] ++ Acc, Config); %% noncharacters -clean(<<_/utf8, Rest/binary>>, Acc, Opts) -> - clean(Rest, maybe_replace(noncharacter, Opts) ++ Acc, Opts); +clean(<<_/utf8, Rest/binary>>, Acc, Config) -> + clean(Rest, maybe_replace(noncharacter, Config) ++ Acc, Config); %% surrogates -clean(<<237, X, _, Rest/binary>>, Acc, Opts) when X >= 160 -> - clean(Rest, maybe_replace(surrogate, Opts) ++ Acc, Opts); +clean(<<237, X, _, Rest/binary>>, Acc, Config) when X >= 160 -> + clean(Rest, maybe_replace(surrogate, Config) ++ Acc, Config); %% u+fffe and u+ffff for R14BXX -clean(<<239, 191, X, Rest/binary>>, Acc, Opts) when X == 190; X == 191 -> - clean(Rest, maybe_replace(noncharacter, Opts) ++ Acc, Opts); +clean(<<239, 191, X, Rest/binary>>, Acc, Config) when X == 190; X == 191 -> + clean(Rest, maybe_replace(noncharacter, Config) ++ Acc, Config); %% overlong encodings and missing continuations of a 2 byte sequence -clean(<>, Acc, Opts) when X >= 192, X =< 223 -> - clean(strip_continuations(Rest, 1), maybe_replace(badutf, Opts) ++ Acc, Opts); +clean(<>, Acc, Config) when X >= 192, X =< 223 -> + clean(strip_continuations(Rest, 1), maybe_replace(badutf, Config) ++ Acc, Config); %% overlong encodings and missing continuations of a 3 byte sequence -clean(<>, Acc, Opts) when X >= 224, X =< 239 -> - clean(strip_continuations(Rest, 2), maybe_replace(badutf, Opts) ++ Acc, Opts); +clean(<>, Acc, Config) when X >= 224, X =< 239 -> + clean(strip_continuations(Rest, 2), maybe_replace(badutf, Config) ++ Acc, Config); %% overlong encodings and missing continuations of a 4 byte sequence -clean(<>, Acc, Opts) when X >= 240, X =< 247 -> - clean(strip_continuations(Rest, 3), maybe_replace(badutf, Opts) ++ Acc, Opts); -clean(<<_, Rest/binary>>, Acc, Opts) -> - clean(Rest, maybe_replace(badutf, Opts) ++ Acc, Opts). +clean(<>, Acc, Config) when X >= 240, X =< 247 -> + clean(strip_continuations(Rest, 3), maybe_replace(badutf, Config) ++ Acc, Config); +clean(<<_, Rest/binary>>, Acc, Config) -> + clean(Rest, maybe_replace(badutf, Config) ++ Acc, Config). strip_continuations(Bin, 0) -> Bin; @@ -505,28 +505,28 @@ strip_continuations(<>, N) when X >= 128, X =< 191 -> strip_continuations(Bin, _) -> Bin. -maybe_replace($\b, #opts{escaped_strings=true}) -> [$b, $\\]; -maybe_replace($\t, #opts{escaped_strings=true}) -> [$t, $\\]; -maybe_replace($\n, #opts{escaped_strings=true}) -> [$n, $\\]; -maybe_replace($\f, #opts{escaped_strings=true}) -> [$f, $\\]; -maybe_replace($\r, #opts{escaped_strings=true}) -> [$r, $\\]; -maybe_replace($\", #opts{escaped_strings=true}) -> [$\", $\\]; -maybe_replace($\\, #opts{escaped_strings=true}) -> [$\\, $\\]; -maybe_replace($/, Opts) -> - case Opts#opts.escaped_forward_slashes of +maybe_replace($\b, #config{escaped_strings=true}) -> [$b, $\\]; +maybe_replace($\t, #config{escaped_strings=true}) -> [$t, $\\]; +maybe_replace($\n, #config{escaped_strings=true}) -> [$n, $\\]; +maybe_replace($\f, #config{escaped_strings=true}) -> [$f, $\\]; +maybe_replace($\r, #config{escaped_strings=true}) -> [$r, $\\]; +maybe_replace($\", #config{escaped_strings=true}) -> [$\", $\\]; +maybe_replace($\\, #config{escaped_strings=true}) -> [$\\, $\\]; +maybe_replace($/, Config) -> + case Config#config.escaped_forward_slashes of true -> [$/, $\\] ; false -> [$/] end; -maybe_replace(X, Opts=#opts{escaped_strings=true}) when X == 16#2028; X == 16#2029 -> - case Opts#opts.unescaped_jsonp of +maybe_replace(X, Config=#config{escaped_strings=true}) when X == 16#2028; X == 16#2029 -> + case Config#config.unescaped_jsonp of true -> [X] ; false -> lists:reverse(jsx_utils:json_escape_sequence(X)) end; -maybe_replace(X, #opts{escaped_strings=true}) when X < 32 -> +maybe_replace(X, #config{escaped_strings=true}) when X < 32 -> lists:reverse(jsx_utils:json_escape_sequence(X)); -maybe_replace(noncharacter, #opts{replaced_bad_utf8=true}) -> [16#fffd]; -maybe_replace(surrogate, #opts{replaced_bad_utf8=true}) -> [16#fffd]; -maybe_replace(badutf, #opts{replaced_bad_utf8=true}) -> [16#fffd]; +maybe_replace(noncharacter, #config{replaced_bad_utf8=true}) -> [16#fffd]; +maybe_replace(surrogate, #config{replaced_bad_utf8=true}) -> [16#fffd]; +maybe_replace(badutf, #config{replaced_bad_utf8=true}) -> [16#fffd]; maybe_replace(_, _) -> erlang:error(badarg). @@ -544,11 +544,11 @@ json_escape_sequence_test_() -> ]. -opts_test_() -> +config_test_() -> [ {"all flags", ?_assertEqual( - #opts{ + #config{ replaced_bad_utf8=true, escaped_forward_slashes=true, explicit_end=true, @@ -558,7 +558,7 @@ opts_test_() -> dirty_strings=true, ignored_bad_escapes=true }, - parse_opts([ + parse_config([ replaced_bad_utf8, escaped_forward_slashes, explicit_end, @@ -572,17 +572,17 @@ opts_test_() -> }, {"relax flag", ?_assertEqual( - #opts{ + #config{ replaced_bad_utf8=true, single_quoted_strings=true, comments=true, ignored_bad_escapes=true }, - parse_opts([relax]) + parse_config([relax]) ) }, {"deprecated flags", ?_assertEqual( - #opts{ + #config{ pre_encode=fun lists:length/1, replaced_bad_utf8=true, escaped_forward_slashes=true, @@ -591,7 +591,7 @@ opts_test_() -> escaped_strings=true, ignored_bad_escapes=true }, - parse_opts([ + parse_config([ {pre_encoder, fun lists:length/1}, loose_unicode, escape_forward_slash, @@ -602,17 +602,17 @@ opts_test_() -> ]) )}, {"pre_encode flag", ?_assertEqual( - #opts{pre_encode=fun lists:length/1}, - parse_opts([{pre_encode, fun lists:length/1}]) + #config{pre_encode=fun lists:length/1}, + parse_config([{pre_encode, fun lists:length/1}]) )}, {"two pre_encoders defined", ?_assertError( badarg, - parse_opts([ + parse_config([ {pre_encode, fun(_) -> true end}, {pre_encode, fun(_) -> false end} ]) )}, - {"bad option flag", ?_assertError(badarg, parse_opts([error]))} + {"bad option flag", ?_assertError(badarg, parse_config([error]))} ]. @@ -686,7 +686,7 @@ fail_clean(Codepoints, Title) -> {generator, fun() -> case Codepoints of [N|Rest] -> - [ {Title, ?_assertError(badarg, clean(N, [], #opts{}))} + [ {Title, ?_assertError(badarg, clean(N, [], #config{}))} | fail_clean(Rest, Title) ] ; [] -> [] @@ -697,7 +697,7 @@ fail_bad(Codepoints, Title) -> {generator, fun() -> case Codepoints of [N|Rest] -> - [ {Title, ?_assertError(badarg, clean(N, [], #opts{}))} + [ {Title, ?_assertError(badarg, clean(N, [], #config{}))} | fail_bad(Rest, Title) ] ; [] -> [] @@ -708,7 +708,7 @@ replace_bad(Codepoints, Title) -> {generator, fun() -> case Codepoints of [N|Rest] -> - [ {Title, ?_assertEqual(<<16#fffd/utf8>>, clean(N, [], #opts{replaced_bad_utf8=true}))} + [ {Title, ?_assertEqual(<<16#fffd/utf8>>, clean(N, [], #config{replaced_bad_utf8=true}))} | replace_bad(Rest, Title) ] ; [] -> [] @@ -732,12 +732,12 @@ clean_test_() -> [ {"basic codepoints", ?_assertEqual( codepoints(), - clean(codepoints(), [], #opts{}) + clean(codepoints(), [], #config{}) )}, fail_clean(escapables(), "escapables"), {"extended codepoints", ?_assertEqual( extended_codepoints(), - clean(extended_codepoints(), [], #opts{}) + clean(extended_codepoints(), [], #config{}) )}, fail_clean(noncharacters(), "noncharacters"), fail_clean(extended_noncharacters(), "extended noncharacters"), @@ -750,182 +750,182 @@ escape_test_() -> [ {"escape backspace", ?_assertEqual( <<"\\b">>, - clean_string(to_fake_utf8(16#0008), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0008), #config{escaped_strings=true}) )}, {"escape tab", ?_assertEqual( <<"\\t">>, - clean_string(to_fake_utf8(16#0009), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0009), #config{escaped_strings=true}) )}, {"escape newline", ?_assertEqual( <<"\\n">>, - clean_string(to_fake_utf8(16#000a), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#000a), #config{escaped_strings=true}) )}, {"escape formfeed", ?_assertEqual( <<"\\f">>, - clean_string(to_fake_utf8(16#000c), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#000c), #config{escaped_strings=true}) )}, {"escape carriage return", ?_assertEqual( <<"\\r">>, - clean_string(to_fake_utf8(16#000d), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#000d), #config{escaped_strings=true}) )}, {"escape quote", ?_assertEqual( <<"\\\"">>, - clean_string(to_fake_utf8(16#0022), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0022), #config{escaped_strings=true}) )}, {"escape forward slash", ?_assertEqual( <<"\\/">>, - clean_string(to_fake_utf8(16#002f), #opts{escaped_strings=true, escaped_forward_slashes=true}) + clean_string(to_fake_utf8(16#002f), #config{escaped_strings=true, escaped_forward_slashes=true}) )}, {"do not escape forward slash", ?_assertEqual( <<"/">>, - clean_string(to_fake_utf8(16#002f), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#002f), #config{escaped_strings=true}) )}, {"escape backslash", ?_assertEqual( <<"\\\\">>, - clean_string(to_fake_utf8(16#005c), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#005c), #config{escaped_strings=true}) )}, {"escape jsonp (u2028)", ?_assertEqual( <<"\\u2028">>, - clean_string(to_fake_utf8(16#2028), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#2028), #config{escaped_strings=true}) )}, {"do not escape jsonp (u2028)", ?_assertEqual( <<16#2028/utf8>>, - clean_string(to_fake_utf8(16#2028), #opts{escaped_strings=true, unescaped_jsonp=true}) + clean_string(to_fake_utf8(16#2028), #config{escaped_strings=true, unescaped_jsonp=true}) )}, {"escape jsonp (u2029)", ?_assertEqual( <<"\\u2029">>, - clean_string(to_fake_utf8(16#2029), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#2029), #config{escaped_strings=true}) )}, {"do not escape jsonp (u2029)", ?_assertEqual( <<16#2029/utf8>>, - clean_string(to_fake_utf8(16#2029), #opts{escaped_strings=true, unescaped_jsonp=true}) + clean_string(to_fake_utf8(16#2029), #config{escaped_strings=true, unescaped_jsonp=true}) )}, {"dirty string", ?_assertEqual( <<"\n">>, - clean_string(to_fake_utf8(16#000a), #opts{escaped_strings=true, dirty_strings=true}) + clean_string(to_fake_utf8(16#000a), #config{escaped_strings=true, dirty_strings=true}) )}, {"escape u0000", ?_assertEqual( <<"\\u0000">>, - clean_string(to_fake_utf8(16#0000), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0000), #config{escaped_strings=true}) )}, {"escape u0001", ?_assertEqual( <<"\\u0001">>, - clean_string(to_fake_utf8(16#0001), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0001), #config{escaped_strings=true}) )}, {"escape u0002", ?_assertEqual( <<"\\u0002">>, - clean_string(to_fake_utf8(16#0002), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0002), #config{escaped_strings=true}) )}, {"escape u0003", ?_assertEqual( <<"\\u0003">>, - clean_string(to_fake_utf8(16#0003), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0003), #config{escaped_strings=true}) )}, {"escape u0004", ?_assertEqual( <<"\\u0004">>, - clean_string(to_fake_utf8(16#0004), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0004), #config{escaped_strings=true}) )}, {"escape u0005", ?_assertEqual( <<"\\u0005">>, - clean_string(to_fake_utf8(16#0005), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0005), #config{escaped_strings=true}) )}, {"escape u0006", ?_assertEqual( <<"\\u0006">>, - clean_string(to_fake_utf8(16#0006), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0006), #config{escaped_strings=true}) )}, {"escape u0007", ?_assertEqual( <<"\\u0007">>, - clean_string(to_fake_utf8(16#0007), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0007), #config{escaped_strings=true}) )}, {"escape u000b", ?_assertEqual( <<"\\u000b">>, - clean_string(to_fake_utf8(16#000b), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#000b), #config{escaped_strings=true}) )}, {"escape u000e", ?_assertEqual( <<"\\u000e">>, - clean_string(to_fake_utf8(16#000e), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#000e), #config{escaped_strings=true}) )}, {"escape u000f", ?_assertEqual( <<"\\u000f">>, - clean_string(to_fake_utf8(16#000f), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#000f), #config{escaped_strings=true}) )}, {"escape u0010", ?_assertEqual( <<"\\u0010">>, - clean_string(to_fake_utf8(16#0010), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0010), #config{escaped_strings=true}) )}, {"escape u0011", ?_assertEqual( <<"\\u0011">>, - clean_string(to_fake_utf8(16#0011), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0011), #config{escaped_strings=true}) )}, {"escape u0012", ?_assertEqual( <<"\\u0012">>, - clean_string(to_fake_utf8(16#0012), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0012), #config{escaped_strings=true}) )}, {"escape u0013", ?_assertEqual( <<"\\u0013">>, - clean_string(to_fake_utf8(16#0013), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0013), #config{escaped_strings=true}) )}, {"escape u0014", ?_assertEqual( <<"\\u0014">>, - clean_string(to_fake_utf8(16#0014), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0014), #config{escaped_strings=true}) )}, {"escape u0015", ?_assertEqual( <<"\\u0015">>, - clean_string(to_fake_utf8(16#0015), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0015), #config{escaped_strings=true}) )}, {"escape u0016", ?_assertEqual( <<"\\u0016">>, - clean_string(to_fake_utf8(16#0016), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0016), #config{escaped_strings=true}) )}, {"escape u0017", ?_assertEqual( <<"\\u0017">>, - clean_string(to_fake_utf8(16#0017), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0017), #config{escaped_strings=true}) )}, {"escape u0018", ?_assertEqual( <<"\\u0018">>, - clean_string(to_fake_utf8(16#0018), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0018), #config{escaped_strings=true}) )}, {"escape u0019", ?_assertEqual( <<"\\u0019">>, - clean_string(to_fake_utf8(16#0019), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#0019), #config{escaped_strings=true}) )}, {"escape u001a", ?_assertEqual( <<"\\u001a">>, - clean_string(to_fake_utf8(16#001a), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#001a), #config{escaped_strings=true}) )}, {"escape u001b", ?_assertEqual( <<"\\u001b">>, - clean_string(to_fake_utf8(16#001b), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#001b), #config{escaped_strings=true}) )}, {"escape u001c", ?_assertEqual( <<"\\u001c">>, - clean_string(to_fake_utf8(16#001c), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#001c), #config{escaped_strings=true}) )}, {"escape u001d", ?_assertEqual( <<"\\u001d">>, - clean_string(to_fake_utf8(16#001d), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#001d), #config{escaped_strings=true}) )}, {"escape u001e", ?_assertEqual( <<"\\u001e">>, - clean_string(to_fake_utf8(16#001e), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#001e), #config{escaped_strings=true}) )}, {"escape u001f", ?_assertEqual( <<"\\u001f">>, - clean_string(to_fake_utf8(16#001f), #opts{escaped_strings=true}) + clean_string(to_fake_utf8(16#001f), #config{escaped_strings=true}) )} ]. bad_utf8_test_() -> [ - {"noncharacter u+fffe", ?_assertError(badarg, clean_string(to_fake_utf8(16#fffe), #opts{}))}, + {"noncharacter u+fffe", ?_assertError(badarg, clean_string(to_fake_utf8(16#fffe), #config{}))}, {"noncharacter u+fffe replaced", ?_assertEqual( <<16#fffd/utf8>>, - clean_string(to_fake_utf8(16#fffe), #opts{replaced_bad_utf8=true}) + clean_string(to_fake_utf8(16#fffe), #config{replaced_bad_utf8=true}) )}, - {"noncharacter u+ffff", ?_assertError(badarg, clean_string(to_fake_utf8(16#ffff), #opts{}))}, + {"noncharacter u+ffff", ?_assertError(badarg, clean_string(to_fake_utf8(16#ffff), #config{}))}, {"noncharacter u+ffff replaced", ?_assertEqual( <<16#fffd/utf8>>, - clean_string(to_fake_utf8(16#ffff), #opts{replaced_bad_utf8=true}) + clean_string(to_fake_utf8(16#ffff), #config{replaced_bad_utf8=true}) )}, fail_bad(extended_noncharacters(), "extended noncharacters"), replace_bad(extended_noncharacters(), "extended noncharacters replaced"), @@ -935,167 +935,167 @@ bad_utf8_test_() -> replace_bad(reserved_space(), "reserved space replaced"), {"orphan continuation byte u+0080", ?_assertError( badarg, - clean_string(<<16#0080>>, #opts{}) + clean_string(<<16#0080>>, #config{}) )}, {"orphan continuation byte u+0080 replaced", ?_assertEqual( <<16#fffd/utf8>>, - clean_string(<<16#0080>>, #opts{replaced_bad_utf8=true}) + clean_string(<<16#0080>>, #config{replaced_bad_utf8=true}) )}, {"orphan continuation byte u+00bf", ?_assertError( badarg, - clean_string(<<16#00bf>>, #opts{}) + clean_string(<<16#00bf>>, #config{}) )}, {"orphan continuation byte u+00bf replaced", ?_assertEqual( <<16#fffd/utf8>>, - clean_string(<<16#00bf>>, #opts{replaced_bad_utf8=true}) + clean_string(<<16#00bf>>, #config{replaced_bad_utf8=true}) )}, {"2 continuation bytes", ?_assertError( badarg, - clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #opts{}) + clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #config{}) )}, {"2 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 2), - clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #opts{replaced_bad_utf8=true}) + clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #config{replaced_bad_utf8=true}) )}, {"3 continuation bytes", - ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #opts{})) + ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{})) }, {"3 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 3), - clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #opts{replaced_bad_utf8=true}) + clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{replaced_bad_utf8=true}) )}, {"4 continuation bytes", - ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #opts{})) + ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{})) }, {"4 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 4), - clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #opts{replaced_bad_utf8=true}) + clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{replaced_bad_utf8=true}) )}, {"5 continuation bytes", - ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #opts{})) + ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{})) }, {"5 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 5), - clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #opts{replaced_bad_utf8=true}) + clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{replaced_bad_utf8=true}) )}, {"6 continuation bytes", - ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #opts{})) + ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{})) }, {"6 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 6), - clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #opts{replaced_bad_utf8=true}) + clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{replaced_bad_utf8=true}) )}, {"all continuation bytes", ?_assertError( badarg, - clean_string(<<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, #opts{}) + clean_string(<<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, #config{}) )}, {"all continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, length(lists:seq(16#0080, 16#00bf))), clean_string( <<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, - #opts{replaced_bad_utf8=true} + #config{replaced_bad_utf8=true} ) )}, - {"lonely start byte", ?_assertError(badarg, clean_string(<<16#00c0>>, #opts{}))}, + {"lonely start byte", ?_assertError(badarg, clean_string(<<16#00c0>>, #config{}))}, {"lonely start byte replaced", ?_assertEqual( <<16#fffd/utf8>>, - clean_string(<<16#00c0>>, #opts{replaced_bad_utf8=true}) + clean_string(<<16#00c0>>, #config{replaced_bad_utf8=true}) )}, {"lonely start bytes (2 byte)", ?_assertError( badarg, - clean_string(<<16#00c0, 32, 16#00df>>, #opts{}) + clean_string(<<16#00c0, 32, 16#00df>>, #config{}) )}, {"lonely start bytes (2 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, - clean_string(<<16#00c0, 32, 16#00df>>, #opts{replaced_bad_utf8=true}) + clean_string(<<16#00c0, 32, 16#00df>>, #config{replaced_bad_utf8=true}) )}, {"lonely start bytes (3 byte)", ?_assertError( badarg, - clean_string(<<16#00e0, 32, 16#00ef>>, #opts{}) + clean_string(<<16#00e0, 32, 16#00ef>>, #config{}) )}, {"lonely start bytes (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, - clean_string(<<16#00e0, 32, 16#00ef>>, #opts{replaced_bad_utf8=true}) + clean_string(<<16#00e0, 32, 16#00ef>>, #config{replaced_bad_utf8=true}) )}, {"lonely start bytes (4 byte)", ?_assertError( badarg, - clean_string(<<16#00f0, 32, 16#00f7>>, #opts{}) + clean_string(<<16#00f0, 32, 16#00f7>>, #config{}) )}, {"lonely start bytes (4 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, - clean_string(<<16#00f0, 32, 16#00f7>>, #opts{replaced_bad_utf8=true}) + clean_string(<<16#00f0, 32, 16#00f7>>, #config{replaced_bad_utf8=true}) )}, {"missing continuation byte (3 byte)", ?_assertError( badarg, - clean_string(<<224, 160, 32>>, #opts{}) + clean_string(<<224, 160, 32>>, #config{}) )}, {"missing continuation byte (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - clean_string(<<224, 160, 32>>, #opts{replaced_bad_utf8=true}) + clean_string(<<224, 160, 32>>, #config{replaced_bad_utf8=true}) )}, {"missing continuation byte (4 byte missing one)", ?_assertError( badarg, - clean_string(<<240, 144, 128, 32>>, #opts{}) + clean_string(<<240, 144, 128, 32>>, #config{}) )}, {"missing continuation byte (4 byte missing one) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - clean_string(<<240, 144, 128, 32>>, #opts{replaced_bad_utf8=true}) + clean_string(<<240, 144, 128, 32>>, #config{replaced_bad_utf8=true}) )}, {"missing continuation byte (4 byte missing two)", ?_assertError( badarg, - clean_string(<<240, 144, 32>>, #opts{}) + clean_string(<<240, 144, 32>>, #config{}) )}, {"missing continuation byte (4 byte missing two) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - clean_string(<<240, 144, 32>>, #opts{replaced_bad_utf8=true}) + clean_string(<<240, 144, 32>>, #config{replaced_bad_utf8=true}) )}, {"overlong encoding of u+002f (2 byte)", ?_assertError( badarg, - clean_string(<<16#c0, 16#af, 32>>, #opts{}) + clean_string(<<16#c0, 16#af, 32>>, #config{}) )}, {"overlong encoding of u+002f (2 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - clean_string(<<16#c0, 16#af, 32>>, #opts{replaced_bad_utf8=true}) + clean_string(<<16#c0, 16#af, 32>>, #config{replaced_bad_utf8=true}) )}, {"overlong encoding of u+002f (3 byte)", ?_assertError( badarg, - clean_string(<<16#e0, 16#80, 16#af, 32>>, #opts{}) + clean_string(<<16#e0, 16#80, 16#af, 32>>, #config{}) )}, {"overlong encoding of u+002f (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - clean_string(<<16#e0, 16#80, 16#af, 32>>, #opts{replaced_bad_utf8=true}) + clean_string(<<16#e0, 16#80, 16#af, 32>>, #config{replaced_bad_utf8=true}) )}, {"overlong encoding of u+002f (4 byte)", ?_assertError( badarg, - clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #opts{}) + clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #config{}) )}, {"overlong encoding of u+002f (4 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #opts{replaced_bad_utf8=true}) + clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #config{replaced_bad_utf8=true}) )}, {"highest overlong 2 byte sequence", ?_assertError( badarg, - clean_string(<<16#c1, 16#bf, 32>>, #opts{}) + clean_string(<<16#c1, 16#bf, 32>>, #config{}) )}, {"highest overlong 2 byte sequence replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - clean_string(<<16#c1, 16#bf, 32>>, #opts{replaced_bad_utf8=true}) + clean_string(<<16#c1, 16#bf, 32>>, #config{replaced_bad_utf8=true}) )}, {"highest overlong 3 byte sequence", ?_assertError( badarg, - clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #opts{}) + clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #config{}) )}, {"highest overlong 3 byte sequence replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #opts{replaced_bad_utf8=true}) + clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #config{replaced_bad_utf8=true}) )}, {"highest overlong 4 byte sequence", ?_assertError( badarg, - clean_string(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, #opts{}) + clean_string(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, #config{}) )}, {"highest overlong 4 byte sequence replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, - clean_string(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, #opts{replaced_bad_utf8=true}) + clean_string(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, #config{replaced_bad_utf8=true}) )} ]. diff --git a/src/jsx_verify.erl b/src/jsx_verify.erl index 304807e..4818998 100644 --- a/src/jsx_verify.erl +++ b/src/jsx_verify.erl @@ -27,65 +27,65 @@ -export([init/1, handle_event/2]). --record(opts, { +-record(config, { repeated_keys = true }). --type opts() :: []. +-type config() :: []. --spec is_json(Source::binary(), Opts::opts()) -> true | false. +-spec is_json(Source::binary(), Config::config()) -> true | false. -is_json(Source, Opts) when is_list(Opts) -> - try (jsx:decoder(?MODULE, Opts, jsx_utils:extract_opts(Opts)))(Source) +is_json(Source, Config) when is_list(Config) -> + try (jsx:decoder(?MODULE, Config, jsx_utils:extract_config(Config)))(Source) catch error:badarg -> false end. --spec is_term(Source::any(), Opts::opts()) -> true | false. +-spec is_term(Source::any(), Config::config()) -> true | false. -is_term(Source, Opts) when is_list(Opts) -> - try (jsx:encoder(?MODULE, Opts, jsx_utils:extract_opts(Opts)))(Source) +is_term(Source, Config) when is_list(Config) -> + try (jsx:encoder(?MODULE, Config, jsx_utils:extract_config(Config)))(Source) catch error:badarg -> false end. -parse_opts(Opts) -> parse_opts(Opts, #opts{}). +parse_config(Config) -> parse_config(Config, #config{}). -parse_opts([{repeated_keys, Val}|Rest], Opts) when Val == true; Val == false -> - parse_opts(Rest, Opts#opts{repeated_keys = Val}); -parse_opts([repeated_keys|Rest], Opts) -> - parse_opts(Rest, Opts#opts{repeated_keys = true}); -parse_opts([{K, _}|Rest] = Options, Opts) -> +parse_config([{repeated_keys, Val}|Rest], Config) when Val == true; Val == false -> + parse_config(Rest, Config#config{repeated_keys = Val}); +parse_config([repeated_keys|Rest], Config) -> + parse_config(Rest, Config#config{repeated_keys = true}); +parse_config([{K, _}|Rest] = Options, Config) -> case lists:member(K, jsx_utils:valid_flags()) of - true -> parse_opts(Rest, Opts) - ; false -> erlang:error(badarg, [Options, Opts]) + true -> parse_config(Rest, Config) + ; false -> erlang:error(badarg, [Options, Config]) end; -parse_opts([K|Rest] = Options, Opts) -> +parse_config([K|Rest] = Options, Config) -> case lists:member(K, jsx_utils:valid_flags()) of - true -> parse_opts(Rest, Opts) - ; false -> erlang:error(badarg, [Options, Opts]) + true -> parse_config(Rest, Config) + ; false -> erlang:error(badarg, [Options, Config]) end; -parse_opts([], Opts) -> - Opts. +parse_config([], Config) -> + Config. -init(Opts) -> {parse_opts(Opts), []}. +init(Config) -> {parse_config(Config), []}. handle_event(end_json, _) -> true; -handle_event(_, {Opts, _} = State) when Opts#opts.repeated_keys == true -> State; +handle_event(_, {Config, _} = State) when Config#config.repeated_keys == true -> State; -handle_event(start_object, {Opts, Keys}) -> {Opts, [dict:new()] ++ Keys}; -handle_event(end_object, {Opts, [_|Keys]}) -> {Opts, Keys}; +handle_event(start_object, {Config, Keys}) -> {Config, [dict:new()] ++ Keys}; +handle_event(end_object, {Config, [_|Keys]}) -> {Config, Keys}; -handle_event({key, Key}, {Opts, [CurrentKeys|Keys]}) -> +handle_event({key, Key}, {Config, [CurrentKeys|Keys]}) -> case dict:is_key(Key, CurrentKeys) of true -> erlang:error(badarg) - ; false -> {Opts, [dict:store(Key, blah, CurrentKeys)|Keys]} + ; false -> {Config, [dict:store(Key, blah, CurrentKeys)|Keys]} end; handle_event(_, State) -> State. @@ -97,20 +97,20 @@ handle_event(_, State) -> State. -include_lib("eunit/include/eunit.hrl"). -opts_test_() -> +config_test_() -> [ - {"empty opts", ?_assertEqual(#opts{}, parse_opts([]))}, - {"bare repeated keys", ?_assertEqual(#opts{}, parse_opts([repeated_keys]))}, + {"empty config", ?_assertEqual(#config{}, parse_config([]))}, + {"bare repeated keys", ?_assertEqual(#config{}, parse_config([repeated_keys]))}, {"repeated keys true", ?_assertEqual( - #opts{}, - parse_opts([{repeated_keys, true}]) + #config{}, + parse_config([{repeated_keys, true}]) )}, {"repeated keys false", ?_assertEqual( - #opts{repeated_keys=false}, - parse_opts([{repeated_keys, false}]) + #config{repeated_keys=false}, + parse_config([{repeated_keys, false}]) )}, - {"invalid opt flag", ?_assertError(badarg, parse_opts([error]))}, - {"invalid opt tuple", ?_assertError(badarg, parse_opts([{error, true}]))} + {"invalid opt flag", ?_assertError(badarg, parse_config([error]))}, + {"invalid opt tuple", ?_assertError(badarg, parse_config([{error, true}]))} ]. @@ -121,7 +121,7 @@ handle_event_test_() -> { Title, ?_assertEqual( true, - lists:foldl(fun handle_event/2, {#opts{}, []}, Events ++ [end_json]) + lists:foldl(fun handle_event/2, {#config{}, []}, Events ++ [end_json]) ) } || {Title, _, _, Events} <- Data ]. From 5c96222cbf2dcd10c9a1d87c0a96ffc86d467f50 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 12 Feb 2013 12:06:45 -0800 Subject: [PATCH 37/61] add additional comment tests --- src/jsx_decoder.erl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 9d452dd..38a2493 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -1423,6 +1423,22 @@ comments_test_() -> {"/**/ comment terminating exp", ?_assertEqual( decode(<<"[ 1e1/* comment */ ]">>, [comments]), [start_array, {float, 1.0e1}, end_array, end_json] + )}, + {"/**/ comment following /**/ comment", ?_assertEqual( + decode(<<"[/* comment *//* comment */true]">>, [comments]), + [start_array, {literal, true}, end_array, end_json] + )}, + {"/**/ comment following // comment", ?_assertEqual( + decode(<<"[// comment", ?newline, "/* comment */true]">>, [comments]), + [start_array, {literal, true}, end_array, end_json] + )}, + {"// comment following /**/ comment", ?_assertEqual( + decode(<<"[/* comment */// comment", ?newline, "true]">>, [comments]), + [start_array, {literal, true}, end_array, end_json] + )}, + {"// comment following // comment", ?_assertEqual( + decode(<<"[// comment", ?newline, "// comment", ?newline, "true]">>, [comments]), + [start_array, {literal, true}, end_array, end_json] )} ]. From 9e01e3a7040f4667e521fa92f4d492408fa7bf41 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Wed, 13 Feb 2013 19:13:50 -0800 Subject: [PATCH 38/61] reduce test cases to saner set --- src/jsx_decoder.erl | 3 +- src/jsx_encoder.erl | 2 +- src/jsx_parser.erl | 3 +- src/jsx_tests.hrl | 184 +++++++++----------------------------------- src/jsx_to_json.erl | 2 +- src/jsx_to_term.erl | 3 +- src/jsx_verify.erl | 3 +- 7 files changed, 41 insertions(+), 159 deletions(-) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 38a2493..2faf5db 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -1629,8 +1629,7 @@ to_fake_utf(N, utf8) -> decode_test_() -> - Data = jsx:universals() - ++ jsx:decodeables(), + Data = jsx:test_cases(), [ { Title, ?_assertEqual( diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 3d7f78d..4784b96 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -137,7 +137,7 @@ clean_string(Bin, Config) -> jsx_utils:clean_string(Bin, Config). encode_test_() -> - Data = jsx:universals(), + Data = jsx:test_cases(), [ { Title, ?_assertEqual( diff --git a/src/jsx_parser.erl b/src/jsx_parser.erl index f5030be..556d8ff 100644 --- a/src/jsx_parser.erl +++ b/src/jsx_parser.erl @@ -170,8 +170,7 @@ clean_string(Bin, Config) -> jsx_utils:clean_string(Bin, Config). decode_test_() -> - Data = jsx:universals() - ++ jsx:decodeables(), + Data = jsx:test_cases(), [ { Title, ?_assertEqual( diff --git a/src/jsx_tests.hrl b/src/jsx_tests.hrl index b414a7c..2fe4f0f 100644 --- a/src/jsx_tests.hrl +++ b/src/jsx_tests.hrl @@ -1,8 +1,7 @@ %% data and helper functions for tests -export([init/1, handle_event/2]). --export([universals/0]). --export([decodeables/0]). +-export([test_cases/0]). -include_lib("eunit/include/eunit.hrl"). @@ -15,30 +14,47 @@ handle_event(end_json, State) -> lists:reverse([end_json] ++ State); handle_event(Event, State) -> [Event] ++ State. -universals() -> +test_cases() -> empty_array() - ++ deep_array() - ++ really_deep_array() + ++ nested_array() ++ empty_object() + ++ nested_object() + ++ strings() ++ literals() ++ integers() - ++ floats() - ++ strings(). + ++ floats(). empty_array() -> [{"[]", <<"[]">>, [], [start_array, end_array]}]. -deep_array() -> - [Test] = empty_array(), - [repeat(fun wrap_with_array/1, Test, 10)]. - -really_deep_array() -> - [Test] = empty_array(), - [repeat(fun wrap_with_array/1, Test, 1000)]. +nested_array() -> + [{ + "[[[]]]", + <<"[[[]]]">>, + [[[]]], + [start_array, start_array, start_array, end_array, end_array, end_array] + }]. empty_object() -> [{"{}", <<"{}">>, [{}], [start_object, end_object]}]. +nested_object() -> + [{ + "{\"key\":{\"key\":{}}}", + <<"{\"key\":{\"key\":{}}}">>, + [{<<"key">>, [{<<"key">>, [{}]}]}], + [ + start_object, + {key, <<"key">>}, + start_object, + {key, <<"key">>}, + start_object, + end_object, + end_object, + end_object + ] + }]. + naked_strings() -> Raw = [ @@ -58,16 +74,7 @@ naked_strings() -> strings() -> naked_strings() ++ [ wrap_with_array(Test) || Test <- naked_strings() ] - ++ [ wrap_with_object(Test) || Test <- naked_strings() ] - ++ [listify("naked strings", naked_strings())] - ++ [ - { - "naked strings", - <<"{\"\":\"\",\"hello world\":\"hello world\"}">>, - [{<<>>, <<>>}, {<<"hello world">>, <<"hello world">>}], - [start_object, {key, <<>>}, {string, <<>>}, {key, <<"hello world">>}, {string, <<"hello world">>}, end_object] - } - ]. + ++ [ wrap_with_object(Test) || Test <- naked_strings() ]. naked_integers() -> @@ -92,9 +99,7 @@ naked_integers() -> integers() -> naked_integers() ++ [ wrap_with_array(Test) || Test <- naked_integers() ] - ++ [ wrap_with_object(Test) || Test <- naked_integers() ] - ++ [listify("naked integers", naked_integers())] - ++ [objectify("naked integers", naked_integers())]. + ++ [ wrap_with_object(Test) || Test <- naked_integers() ]. naked_floats() -> @@ -124,9 +129,7 @@ naked_floats() -> floats() -> naked_floats() ++ [ wrap_with_array(Test) || Test <- naked_floats() ] - ++ [ wrap_with_object(Test) || Test <- naked_floats() ] - ++ [listify("naked floats", naked_floats())] - ++ [objectify("naked floats", naked_floats())]. + ++ [ wrap_with_object(Test) || Test <- naked_floats() ]. naked_literals() -> @@ -143,27 +146,7 @@ naked_literals() -> literals() -> naked_literals() ++ [ wrap_with_array(Test) || Test <- naked_literals() ] - ++ [ wrap_with_object(Test) || Test <- naked_literals() ] - ++ [listify("naked literals", naked_literals())] - ++ [objectify("naked literals", naked_literals())]. - - -%% special tests used only for things that don't round trip when decoded and re-encoded - -decodeables() -> - Tests = [ - {"-0.0", <<"-0.0">>, 0.0, [{float, 0.0}]}, - {"1e0", <<"1e0">>, 1.0, [{float, 1.0}]}, - {"0e0", <<"0e0">>, 0.0, [{float, 0.0}]}, - {"1e4", <<"1e4">>, 1.0e4, [{float, 1.0e4}]}, - {"0e4", <<"0e4">>, 0.0, [{float, 0.0}]}, - {"-1e0", <<"-1e0">>, -1.0, [{float, -1.0}]}, - {"-0", <<"-0">>, 0, [{integer, 0}]} - ], - [ wrap_with_array(Test) || Test <- Tests ] - ++ [ wrap_with_object(Test) || Test <- Tests ] - ++ [listify("naked decodeables", Tests)] - ++ [objectify("naked decodeables", Tests)]. + ++ [ wrap_with_object(Test) || Test <- naked_literals() ]. wrap_with_array({Title, JSON, Term, Events}) -> @@ -184,103 +167,6 @@ wrap_with_object({Title, JSON, Term, Events}) -> }. -repeat(_, Test, 0) -> Test; -repeat(Fun, Test, Times) -> repeat(Fun, Fun(Test), Times - 1). - - sane_float_to_list(X) -> [Output] = io_lib:format("~p", [X]), - Output. - - -listify(Title, [{_, JSON, Term, Events}|Rest]) -> do_listify(Rest, {Title, JSON, [Term], Events}). - -do_listify([], {Title, JSON, Term, Events}) -> - { - Title, - <<"["/utf8, JSON/binary, "]"/utf8>>, - Term, - [start_array] ++ Events ++ [end_array] - }; -do_listify([Test|Rest], Acc) -> - {Title, A, M, X} = Acc, - {_, B, N, Y} = Test, - do_listify(Rest, {Title, <>, M ++ [N], X ++ Y}). - - -objectify(Title, [{_, JSON, Term, Events}|Rest]) -> - do_objectify( - Rest, - {Title, <<"\"", JSON/binary, "\":", JSON/binary>>, [{JSON, Term}], [{key, JSON}] ++ Events} - ). - -do_objectify([], {Title, JSON, Term, Events}) -> - { - Title, - <<"{"/utf8, JSON/binary, "}"/utf8>>, - Term, - [start_object] ++ Events ++ [end_object] - }; -do_objectify([Test|Rest], Acc) -> - {Title, A, M, X} = Acc, - {_, B, N, Y} = Test, - do_objectify(Rest, { - Title, - <>, - M ++ [{B, N}], - X ++ [{key, B}] ++ Y - }). - - -listify_test_() -> - {"listify test", ?_assertEqual( - { - "listify test", - <<"[true,1,\"hello world\",{}]">>, - [true, 1, <<"hello world">>, [{}]], - [ - start_array, - {literal, true}, - {integer, 1}, - {string, <<"hello world">>}, - start_object, - end_object, - end_array - ] - }, - listify("listify test", [ - {"true", <<"true">>, true, [{literal, true}]}, - {"1", <<"1">>, 1, [{integer, 1}]}, - {"hello world", <<"\"hello world\"">>, <<"hello world">>, [{string, <<"hello world">>}]}, - {"{}", <<"{}">>, [{}], [start_object, end_object]} - ]) - )}. - - -objectify_test_() -> - {"objectify test", ?_assertEqual( - { - "objectify test", - <<"{\"true\":true,\"1\":1,\"\"hello world\"\":\"hello world\",\"[]\":[]}">>, - [{<<"true">>, true}, {<<"1">>, 1}, {<<"\"hello world\"">>, <<"hello world">>}, {<<"[]">>, []}], - [ - start_object, - {key, <<"true">>}, - {literal, true}, - {key, <<"1">>}, - {integer, 1}, - {key, <<"\"hello world\"">>}, - {string, <<"hello world">>}, - {key, <<"[]">>}, - start_array, - end_array, - end_object - ] - }, - objectify("objectify test", [ - {"true", <<"true">>, true, [{literal, true}]}, - {"1", <<"1">>, 1, [{integer, 1}]}, - {"hello world", <<"\"hello world\"">>, <<"hello world">>, [{string, <<"hello world">>}]}, - {"[]", <<"[]">>, [], [start_array, end_array]} - ]) - )}. \ No newline at end of file + Output. \ No newline at end of file diff --git a/src/jsx_to_json.erl b/src/jsx_to_json.erl index 2b3a8b1..b73363f 100644 --- a/src/jsx_to_json.erl +++ b/src/jsx_to_json.erl @@ -292,7 +292,7 @@ format_test_() -> handle_event_test_() -> - Data = jsx:universals(), + Data = jsx:test_cases(), [ { Title, ?_assertEqual( diff --git a/src/jsx_to_term.erl b/src/jsx_to_term.erl index b27d9b9..e8c9e64 100644 --- a/src/jsx_to_term.erl +++ b/src/jsx_to_term.erl @@ -267,8 +267,7 @@ post_decoders_test_() -> handle_event_test_() -> - Data = jsx:universals() - ++ jsx:decodeables(), + Data = jsx:test_cases(), [ { Title, ?_assertEqual( diff --git a/src/jsx_verify.erl b/src/jsx_verify.erl index 4818998..3a461d9 100644 --- a/src/jsx_verify.erl +++ b/src/jsx_verify.erl @@ -115,8 +115,7 @@ config_test_() -> handle_event_test_() -> - Data = jsx:universals() - ++ jsx:decodeables(), + Data = jsx:test_cases(), [ { Title, ?_assertEqual( From cea0cb55c7d7c29b56449b465de1e96de9b48577 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Wed, 13 Feb 2013 21:45:56 -0800 Subject: [PATCH 39/61] add compound object test --- src/jsx_tests.hrl | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/jsx_tests.hrl b/src/jsx_tests.hrl index 2fe4f0f..563729a 100644 --- a/src/jsx_tests.hrl +++ b/src/jsx_tests.hrl @@ -22,7 +22,8 @@ test_cases() -> ++ strings() ++ literals() ++ integers() - ++ floats(). + ++ floats() + ++ compound_object(). empty_array() -> [{"[]", <<"[]">>, [], [start_array, end_array]}]. @@ -149,6 +150,44 @@ literals() -> ++ [ wrap_with_object(Test) || Test <- naked_literals() ]. +compound_object() -> + [{ + "[{\"alpha\":[1,2,3],\"beta\":{\"alpha\":[1.0,2.0,3.0],\"beta\":[true,false]}},[{}]]", + <<"[{\"alpha\":[1,2,3],\"beta\":{\"alpha\":[1.0,2.0,3.0],\"beta\":[true,false]}},[{}]]">>, + [[{<<"alpha">>, [1, 2, 3]}, {<<"beta">>, [{<<"alpha">>, [1.0, 2.0, 3.0]}, {<<"beta">>, [true, false]}]}], [[{}]]], + [ + start_array, + start_object, + {key, <<"alpha">>}, + start_array, + {integer, 1}, + {integer, 2}, + {integer, 3}, + end_array, + {key, <<"beta">>}, + start_object, + {key, <<"alpha">>}, + start_array, + {float, 1.0}, + {float, 2.0}, + {float, 3.0}, + end_array, + {key, <<"beta">>}, + start_array, + {literal, true}, + {literal, false}, + end_array, + end_object, + end_object, + start_array, + start_object, + end_object, + end_array, + end_array + ] + }]. + + wrap_with_array({Title, JSON, Term, Events}) -> { "[" ++ Title ++ "]", From 4d60fa2a08523a2bdd767f1889cc2a1ac5224282 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Wed, 13 Feb 2013 21:54:09 -0800 Subject: [PATCH 40/61] add special number tests for floats and integers that have different representation in erlang and json --- src/jsx_decoder.erl | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 2faf5db..b5c7069 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -1024,6 +1024,41 @@ done(Bin, Handler, Stack, Config) -> ?error([Bin, Handler, Stack, Config]). -include_lib("eunit/include/eunit.hrl"). +%% all these numbers have different representation in erlang than in javascript and +%% do not roundtrip like most integers/floats +special_number_test_() -> + [ + {"-0", ?_assertEqual( + [{integer, 0}, end_json], + start(<<"-0">>, {jsx, []}, [], #config{}) + )}, + {"-0.0", ?_assertEqual( + [{float, 0.0}, end_json], + start(<<"-0.0">>, {jsx, []}, [], #config{}) + )}, + {"0e0", ?_assertEqual( + [{float, 0.0}, end_json], + start(<<"0e0">>, {jsx, []}, [], #config{}) + )}, + {"0e4", ?_assertEqual( + [{float, 0.0}, end_json], + start(<<"0e4">>, {jsx, []}, [], #config{}) + )}, + {"1e0", ?_assertEqual( + [{float, 1.0}, end_json], + start(<<"1e0">>, {jsx, []}, [], #config{}) + )}, + {"-1e0", ?_assertEqual( + [{float, -1.0}, end_json], + start(<<"-1e0">>, {jsx, []}, [], #config{}) + )}, + {"1e4", ?_assertEqual( + [{float, 1.0e4}, end_json], + start(<<"1e4">>, {jsx, []}, [], #config{}) + )} + ]. + + xcode(Bin) -> xcode(Bin, []). xcode(Bin, Config) -> From 0392be060d052fdc70dfa00190f43588f43de13d Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 19 Feb 2013 00:26:18 -0800 Subject: [PATCH 41/61] add tests for strings that need unescaping in decoding --- src/jsx_decoder.erl | 68 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index b5c7069..8b67904 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -1059,6 +1059,57 @@ special_number_test_() -> ]. +unescape_test_() -> + [ + {"unescape backspace", ?_assertEqual( + [{string, <<"\b">>}, end_json], + start(<<"\"\\b\"">>, {jsx, []}, [], #config{}) + )}, + {"unescape tab", ?_assertEqual( + [{string, <<"\t">>}, end_json], + start(<<"\"\\t\"">>, {jsx, []}, [], #config{}) + )}, + {"unescape newline", ?_assertEqual( + [{string, <<"\n">>}, end_json], + start(<<"\"\\n\"">>, {jsx, []}, [], #config{}) + )}, + {"unescape formfeed", ?_assertEqual( + [{string, <<"\f">>}, end_json], + start(<<"\"\\f\"">>, {jsx, []}, [], #config{}) + )}, + {"unescape carriage return", ?_assertEqual( + [{string, <<"\r">>}, end_json], + start(<<"\"\\r\"">>, {jsx, []}, [], #config{}) + )}, + {"unescape quote", ?_assertEqual( + [{string, <<"\"">>}, end_json], + start(<<"\"\\\"\"">>, {jsx, []}, [], #config{}) + )}, + {"unescape forward slash", ?_assertEqual( + [{string, <<"/">>}, end_json], + start(<<"\"\\/\"">>, {jsx, []}, [], #config{}) + )}, + {"unescape back slash", ?_assertEqual( + [{string, <<"\\">>}, end_json], + start(<<"\"\\\\\"">>, {jsx, []}, [], #config{}) + )}, + {"unescape control code", ?_assertEqual( + [{string, <<0>>}, end_json], + start(<<"\"\\u0000\"">>, {jsx, []}, [], #config{}) + )} + ]. + + +ignored_bad_escapes_test_() -> + [ + {"ignore unrecognized escape sequence", ?_assertEqual( + [{string, <<"\\x25">>}, end_json], + start(<<"\"\\x25\"">>, {jsx, []}, [], #config{ignored_bad_escapes=true}) + )} + ]. + + + xcode(Bin) -> xcode(Bin, []). xcode(Bin, Config) -> @@ -1268,15 +1319,6 @@ decode(JSON, Config) -> end. -ignored_bad_escapes_test_() -> - [ - {"ignore unrecognized escape sequence", ?_assertEqual( - decode(<<"[\"\\x25\"]">>, [ignored_bad_escapes]), - [start_array, {string, <<"\\x25">>}, end_array, end_json] - )} - ]. - - comments_test_() -> [ {"preceeding // comment", ?_assertEqual( @@ -1478,13 +1520,7 @@ comments_test_() -> ]. -escaped_forward_slashes_test_() -> - [ - {"escape forward slash test", ?_assertEqual( - decode(<<"[ \" \/ \" ]">>, [escaped_forward_slashes]), - [start_array, {string, <<" / ">>}, end_array, end_json] - )} - ]. + escapes_test_() -> From ff170d5f4e656c3dafb132546889b509d4fd543b Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 19 Feb 2013 01:00:35 -0800 Subject: [PATCH 42/61] add maybe_escape/2 function for future use --- src/jsx_utils.erl | 190 ++++++++++++++++++++++++++-------------------- 1 file changed, 106 insertions(+), 84 deletions(-) diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index b50172a..8184423 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -505,6 +505,28 @@ strip_continuations(<>, N) when X >= 128, X =< 191 -> strip_continuations(Bin, _) -> Bin. +maybe_escape(String, #config{dirty_strings=true}) -> String; +maybe_escape(String, #config{escaped_strings=true} = Config) -> escape(String, Config); +maybe_escape(String, _Config) -> String. + +escape($\b, _) -> <<"\\b">>; +escape($\t, _) -> <<"\\t">>; +escape($\n, _) -> <<"\\n">>; +escape($\f, _) -> <<"\\f">>; +escape($\r, _) -> <<"\\r">>; +escape($\", _) -> <<"\\\"">>; +escape($\\, _) -> <<"\\\\">>; +escape($/, #config{escaped_forward_slashes=true}) -> <<"\\/">>; +escape($/, _) -> <<"/">>; +escape(16#2028, #config{unescaped_jsonp=true}) -> <<16#2028/utf8>>; +escape(16#2028, _) -> <<"\\u2028">>; +escape(16#2029, #config{unescaped_jsonp=true}) -> <<16#2029/utf8>>; +escape(16#2029, _) -> <<"\\u2029">>; +escape(X, _) when X < 32 -> + <> = <>, + <<"\\u", (to_hex(A)), (to_hex(B)), (to_hex(C)), (to_hex(D))>>. + + maybe_replace($\b, #config{escaped_strings=true}) -> [$b, $\\]; maybe_replace($\t, #config{escaped_strings=true}) -> [$t, $\\]; maybe_replace($\n, #config{escaped_strings=true}) -> [$n, $\\]; @@ -748,169 +770,169 @@ clean_test_() -> escape_test_() -> [ - {"escape backspace", ?_assertEqual( + {"maybe_escape backspace", ?_assertEqual( <<"\\b">>, - clean_string(to_fake_utf8(16#0008), #config{escaped_strings=true}) + maybe_escape(16#0008, #config{escaped_strings=true}) )}, - {"escape tab", ?_assertEqual( + {"maybe_escape tab", ?_assertEqual( <<"\\t">>, - clean_string(to_fake_utf8(16#0009), #config{escaped_strings=true}) + maybe_escape(16#0009, #config{escaped_strings=true}) )}, - {"escape newline", ?_assertEqual( + {"maybe_escape newline", ?_assertEqual( <<"\\n">>, - clean_string(to_fake_utf8(16#000a), #config{escaped_strings=true}) + maybe_escape(16#000a, #config{escaped_strings=true}) )}, - {"escape formfeed", ?_assertEqual( + {"maybe_escape formfeed", ?_assertEqual( <<"\\f">>, - clean_string(to_fake_utf8(16#000c), #config{escaped_strings=true}) + maybe_escape(16#000c, #config{escaped_strings=true}) )}, - {"escape carriage return", ?_assertEqual( + {"maybe_escape carriage return", ?_assertEqual( <<"\\r">>, - clean_string(to_fake_utf8(16#000d), #config{escaped_strings=true}) + maybe_escape(16#000d, #config{escaped_strings=true}) )}, - {"escape quote", ?_assertEqual( + {"maybe_escape quote", ?_assertEqual( <<"\\\"">>, - clean_string(to_fake_utf8(16#0022), #config{escaped_strings=true}) + maybe_escape(16#0022, #config{escaped_strings=true}) )}, - {"escape forward slash", ?_assertEqual( + {"maybe_escape forward slash", ?_assertEqual( <<"\\/">>, - clean_string(to_fake_utf8(16#002f), #config{escaped_strings=true, escaped_forward_slashes=true}) + maybe_escape(16#002f, #config{escaped_strings=true, escaped_forward_slashes=true}) )}, - {"do not escape forward slash", ?_assertEqual( + {"do not maybe_escape forward slash", ?_assertEqual( <<"/">>, - clean_string(to_fake_utf8(16#002f), #config{escaped_strings=true}) + maybe_escape(16#002f, #config{escaped_strings=true}) )}, - {"escape backslash", ?_assertEqual( + {"maybe_escape backslash", ?_assertEqual( <<"\\\\">>, - clean_string(to_fake_utf8(16#005c), #config{escaped_strings=true}) + maybe_escape(16#005c, #config{escaped_strings=true}) )}, - {"escape jsonp (u2028)", ?_assertEqual( + {"maybe_escape jsonp (u2028)", ?_assertEqual( <<"\\u2028">>, - clean_string(to_fake_utf8(16#2028), #config{escaped_strings=true}) + maybe_escape(16#2028, #config{escaped_strings=true}) )}, - {"do not escape jsonp (u2028)", ?_assertEqual( + {"do not maybe_escape jsonp (u2028)", ?_assertEqual( <<16#2028/utf8>>, - clean_string(to_fake_utf8(16#2028), #config{escaped_strings=true, unescaped_jsonp=true}) + maybe_escape(16#2028, #config{escaped_strings=true, unescaped_jsonp=true}) )}, - {"escape jsonp (u2029)", ?_assertEqual( + {"maybe_escape jsonp (u2029)", ?_assertEqual( <<"\\u2029">>, - clean_string(to_fake_utf8(16#2029), #config{escaped_strings=true}) + maybe_escape(16#2029, #config{escaped_strings=true}) )}, - {"do not escape jsonp (u2029)", ?_assertEqual( + {"do not maybe_escape jsonp (u2029)", ?_assertEqual( <<16#2029/utf8>>, - clean_string(to_fake_utf8(16#2029), #config{escaped_strings=true, unescaped_jsonp=true}) + maybe_escape(16#2029, #config{escaped_strings=true, unescaped_jsonp=true}) )}, - {"dirty string", ?_assertEqual( - <<"\n">>, - clean_string(to_fake_utf8(16#000a), #config{escaped_strings=true, dirty_strings=true}) - )}, - {"escape u0000", ?_assertEqual( + {"maybe_escape u0000", ?_assertEqual( <<"\\u0000">>, - clean_string(to_fake_utf8(16#0000), #config{escaped_strings=true}) + maybe_escape(16#0000, #config{escaped_strings=true}) )}, - {"escape u0001", ?_assertEqual( + {"maybe_escape u0001", ?_assertEqual( <<"\\u0001">>, - clean_string(to_fake_utf8(16#0001), #config{escaped_strings=true}) + maybe_escape(16#0001, #config{escaped_strings=true}) )}, - {"escape u0002", ?_assertEqual( + {"maybe_escape u0002", ?_assertEqual( <<"\\u0002">>, - clean_string(to_fake_utf8(16#0002), #config{escaped_strings=true}) + maybe_escape(16#0002, #config{escaped_strings=true}) )}, - {"escape u0003", ?_assertEqual( + {"maybe_escape u0003", ?_assertEqual( <<"\\u0003">>, - clean_string(to_fake_utf8(16#0003), #config{escaped_strings=true}) + maybe_escape(16#0003, #config{escaped_strings=true}) )}, - {"escape u0004", ?_assertEqual( + {"maybe_escape u0004", ?_assertEqual( <<"\\u0004">>, - clean_string(to_fake_utf8(16#0004), #config{escaped_strings=true}) + maybe_escape(16#0004, #config{escaped_strings=true}) )}, - {"escape u0005", ?_assertEqual( + {"maybe_escape u0005", ?_assertEqual( <<"\\u0005">>, - clean_string(to_fake_utf8(16#0005), #config{escaped_strings=true}) + maybe_escape(16#0005, #config{escaped_strings=true}) )}, - {"escape u0006", ?_assertEqual( + {"maybe_escape u0006", ?_assertEqual( <<"\\u0006">>, - clean_string(to_fake_utf8(16#0006), #config{escaped_strings=true}) + maybe_escape(16#0006, #config{escaped_strings=true}) )}, - {"escape u0007", ?_assertEqual( + {"maybe_escape u0007", ?_assertEqual( <<"\\u0007">>, - clean_string(to_fake_utf8(16#0007), #config{escaped_strings=true}) + maybe_escape(16#0007, #config{escaped_strings=true}) )}, - {"escape u000b", ?_assertEqual( + {"maybe_escape u000b", ?_assertEqual( <<"\\u000b">>, - clean_string(to_fake_utf8(16#000b), #config{escaped_strings=true}) + maybe_escape(16#000b, #config{escaped_strings=true}) )}, - {"escape u000e", ?_assertEqual( + {"maybe_escape u000e", ?_assertEqual( <<"\\u000e">>, - clean_string(to_fake_utf8(16#000e), #config{escaped_strings=true}) + maybe_escape(16#000e, #config{escaped_strings=true}) )}, - {"escape u000f", ?_assertEqual( + {"maybe_escape u000f", ?_assertEqual( <<"\\u000f">>, - clean_string(to_fake_utf8(16#000f), #config{escaped_strings=true}) + maybe_escape(16#000f, #config{escaped_strings=true}) )}, - {"escape u0010", ?_assertEqual( + {"maybe_escape u0010", ?_assertEqual( <<"\\u0010">>, - clean_string(to_fake_utf8(16#0010), #config{escaped_strings=true}) + maybe_escape(16#0010, #config{escaped_strings=true}) )}, - {"escape u0011", ?_assertEqual( + {"maybe_escape u0011", ?_assertEqual( <<"\\u0011">>, - clean_string(to_fake_utf8(16#0011), #config{escaped_strings=true}) + maybe_escape(16#0011, #config{escaped_strings=true}) )}, - {"escape u0012", ?_assertEqual( + {"maybe_escape u0012", ?_assertEqual( <<"\\u0012">>, - clean_string(to_fake_utf8(16#0012), #config{escaped_strings=true}) + maybe_escape(16#0012, #config{escaped_strings=true}) )}, - {"escape u0013", ?_assertEqual( + {"maybe_escape u0013", ?_assertEqual( <<"\\u0013">>, - clean_string(to_fake_utf8(16#0013), #config{escaped_strings=true}) + maybe_escape(16#0013, #config{escaped_strings=true}) )}, - {"escape u0014", ?_assertEqual( + {"maybe_escape u0014", ?_assertEqual( <<"\\u0014">>, - clean_string(to_fake_utf8(16#0014), #config{escaped_strings=true}) + maybe_escape(16#0014, #config{escaped_strings=true}) )}, - {"escape u0015", ?_assertEqual( + {"maybe_escape u0015", ?_assertEqual( <<"\\u0015">>, - clean_string(to_fake_utf8(16#0015), #config{escaped_strings=true}) + maybe_escape(16#0015, #config{escaped_strings=true}) )}, - {"escape u0016", ?_assertEqual( + {"maybe_escape u0016", ?_assertEqual( <<"\\u0016">>, - clean_string(to_fake_utf8(16#0016), #config{escaped_strings=true}) + maybe_escape(16#0016, #config{escaped_strings=true}) )}, - {"escape u0017", ?_assertEqual( + {"maybe_escape u0017", ?_assertEqual( <<"\\u0017">>, - clean_string(to_fake_utf8(16#0017), #config{escaped_strings=true}) + maybe_escape(16#0017, #config{escaped_strings=true}) )}, - {"escape u0018", ?_assertEqual( + {"maybe_escape u0018", ?_assertEqual( <<"\\u0018">>, - clean_string(to_fake_utf8(16#0018), #config{escaped_strings=true}) + maybe_escape(16#0018, #config{escaped_strings=true}) )}, - {"escape u0019", ?_assertEqual( + {"maybe_escape u0019", ?_assertEqual( <<"\\u0019">>, - clean_string(to_fake_utf8(16#0019), #config{escaped_strings=true}) + maybe_escape(16#0019, #config{escaped_strings=true}) )}, - {"escape u001a", ?_assertEqual( + {"maybe_escape u001a", ?_assertEqual( <<"\\u001a">>, - clean_string(to_fake_utf8(16#001a), #config{escaped_strings=true}) + maybe_escape(16#001a, #config{escaped_strings=true}) )}, - {"escape u001b", ?_assertEqual( + {"maybe_escape u001b", ?_assertEqual( <<"\\u001b">>, - clean_string(to_fake_utf8(16#001b), #config{escaped_strings=true}) + maybe_escape(16#001b, #config{escaped_strings=true}) )}, - {"escape u001c", ?_assertEqual( + {"maybe_escape u001c", ?_assertEqual( <<"\\u001c">>, - clean_string(to_fake_utf8(16#001c), #config{escaped_strings=true}) + maybe_escape(16#001c, #config{escaped_strings=true}) )}, - {"escape u001d", ?_assertEqual( + {"maybe_escape u001d", ?_assertEqual( <<"\\u001d">>, - clean_string(to_fake_utf8(16#001d), #config{escaped_strings=true}) + maybe_escape(16#001d, #config{escaped_strings=true}) )}, - {"escape u001e", ?_assertEqual( + {"maybe_escape u001e", ?_assertEqual( <<"\\u001e">>, - clean_string(to_fake_utf8(16#001e), #config{escaped_strings=true}) + maybe_escape(16#001e, #config{escaped_strings=true}) )}, - {"escape u001f", ?_assertEqual( + {"maybe_escape u001f", ?_assertEqual( <<"\\u001f">>, - clean_string(to_fake_utf8(16#001f), #config{escaped_strings=true}) + maybe_escape(16#001f, #config{escaped_strings=true}) + )}, + {"dirty strings", ?_assertEqual( + <<0>>, + maybe_escape(<<0>>, #config{escaped_strings=true, dirty_strings=true}) )} ]. From fbd90e4df7165c4f8d4dd7857d8061a0b416e350 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 19 Feb 2013 15:09:28 -0800 Subject: [PATCH 43/61] tighten input/output of maybe_escape --- src/jsx_utils.erl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index 8184423..105082d 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -505,9 +505,9 @@ strip_continuations(<>, N) when X >= 128, X =< 191 -> strip_continuations(Bin, _) -> Bin. -maybe_escape(String, #config{dirty_strings=true}) -> String; -maybe_escape(String, #config{escaped_strings=true} = Config) -> escape(String, Config); -maybe_escape(String, _Config) -> String. +maybe_escape(Escaped, #config{dirty_strings=true}) -> <>; +maybe_escape(Escaped, #config{escaped_strings=true} = Config) -> escape(Escaped, Config); +maybe_escape(Escaped, _Config) -> <>. escape($\b, _) -> <<"\\b">>; escape($\t, _) -> <<"\\t">>; @@ -774,6 +774,10 @@ escape_test_() -> <<"\\b">>, maybe_escape(16#0008, #config{escaped_strings=true}) )}, + {"don't escape backspace", ?_assertEqual( + <<"\b">>, + maybe_escape(16#0008, #config{}) + )}, {"maybe_escape tab", ?_assertEqual( <<"\\t">>, maybe_escape(16#0009, #config{escaped_strings=true}) @@ -932,7 +936,7 @@ escape_test_() -> )}, {"dirty strings", ?_assertEqual( <<0>>, - maybe_escape(<<0>>, #config{escaped_strings=true, dirty_strings=true}) + maybe_escape(16#0000, #config{escaped_strings=true, dirty_strings=true}) )} ]. From ceb7ce658679333d27e89a280ed88f26ae67b9f9 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 19 Feb 2013 22:51:54 -0800 Subject: [PATCH 44/61] shiny and new string cleaning for encoder and parser --- src/jsx_encoder.erl | 7 +- src/jsx_parser.erl | 23 +- src/jsx_utils.erl | 680 ++++++++++++-------------------------------- 3 files changed, 208 insertions(+), 502 deletions(-) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 4784b96..914b35a 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -53,7 +53,12 @@ start(Term, {Handler, State}, Config) -> value(String, {Handler, State}, Config) when is_binary(String) -> - Handler:handle_event({string, clean_string(String, Config)}, State); + case clean_string(String, Config) of + {error, badarg} -> + ?error([String, {Handler, State}, Config]); + CleanString -> + Handler:handle_event({string, CleanString}, State) + end; value(Float, {Handler, State}, _Config) when is_float(Float) -> Handler:handle_event({float, Float}, State); value(Int, {Handler, State}, _Config) when is_integer(Int) -> diff --git a/src/jsx_parser.erl b/src/jsx_parser.erl index 556d8ff..0cb146b 100644 --- a/src/jsx_parser.erl +++ b/src/jsx_parser.erl @@ -101,9 +101,19 @@ value([Number|Tokens], Handler, Stack, Config) when is_integer(Number) -> value([Number|Tokens], Handler, Stack, Config) when is_float(Number) -> value([{float, Number}] ++ Tokens, Handler, Stack, Config); value([{string, String}|Tokens], Handler, [], Config) when is_binary(String) -> - done(Tokens, handle_event({string, clean_string(String, Config)}, Handler, Config), [], Config); + case clean_string(String, Config) of + {error, badarg} -> + ?error([[{string, String}|Tokens], Handler, [], Config]); + CleanString -> + done(Tokens, handle_event({string, CleanString}, Handler, Config), [], Config) + end; value([{string, String}|Tokens], Handler, Stack, Config) when is_binary(String) -> - maybe_done(Tokens, handle_event({string, clean_string(String, Config)}, Handler, Config), Stack, Config); + case clean_string(String, Config) of + {error, badarg} -> + ?error([[{string, String}|Tokens], Handler, Stack, Config]); + CleanString -> + maybe_done(Tokens, handle_event({string, CleanString}, Handler, Config), Stack, Config) + end; value([String|Tokens], Handler, Stack, Config) when is_binary(String) -> value([{string, String}] ++ Tokens, Handler, Stack, Config); value([], Handler, Stack, Config) -> @@ -116,9 +126,14 @@ value(Token, Handler, Stack, Config) -> object([end_object|Tokens], Handler, [object|Stack], Config) -> maybe_done(Tokens, handle_event(end_object, Handler, Config), Stack, Config); object([{key, Key}|Tokens], Handler, Stack, Config) when is_atom(Key); is_binary(Key) -> - value(Tokens, handle_event({key, clean_string(fix_key(Key), Config)}, Handler, Config), Stack, Config); + case clean_string(fix_key(Key), Config) of + {error, badarg} -> + ?error([[{key, Key}|Tokens], Handler, Stack, Config]); + CleanString -> + value(Tokens, handle_event({key, CleanString}, Handler, Config), Stack, Config) + end; object([Key|Tokens], Handler, Stack, Config) when is_atom(Key); is_binary(Key) -> - value(Tokens, handle_event({key, clean_string(fix_key(Key), Config)}, Handler, Config), Stack, Config); + object([{key, Key}] ++ Tokens, Handler, Stack, Config); object([], Handler, Stack, Config) -> ?incomplete(object, Handler, Stack, Config); object(BadTokens, Handler, Stack, Config) when is_list(BadTokens) -> diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index 105082d..391d890 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -145,367 +145,80 @@ to_hex(X) -> X + 48. %% ascii "1" is [49], "2" is [50], etc... clean_string(Bin, #config{dirty_strings=true}) -> Bin; -clean_string(Bin, Config) -> - case Config#config.replaced_bad_utf8 orelse Config#config.escaped_strings of - true -> clean(Bin, [], Config) - ; false -> ensure_clean(Bin), Bin +clean_string(Bin, Config) -> clean_string(Bin, <<>>, Config). + + +clean_string(Bin, Acc, Config) -> + case cut(Bin, 0) of + {_, finished} -> <>; + {X, escape} -> + <> = Bin, + Escaped = maybe_escape(Codepoint, Config), + clean_string(Rest, <>, Config); + {X, replace, Y} -> + <> = Bin, + case maybe_replace(Bad, Config) of + {error, badarg} -> {error, badarg}; + Replaced -> clean_string(Rest, <>, Config) + end end. -%% fast path for no escaping and no correcting, throws error if string is 'bad' -ensure_clean(<<>>) -> ok; -ensure_clean(<<0, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<1, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<2, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<3, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<4, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<5, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<6, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<7, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<8, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<9, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<10, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<11, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<12, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<13, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<14, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<15, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<16, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<17, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<18, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<19, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<20, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<21, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<22, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<23, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<24, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<25, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<26, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<27, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<28, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<29, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<30, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<31, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<32, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<33, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<34, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<35, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<36, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<37, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<38, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<39, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<40, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<41, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<42, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<43, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<44, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<45, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<46, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<47, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<48, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<49, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<50, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<51, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<52, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<53, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<54, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<55, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<56, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<57, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<58, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<59, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<60, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<61, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<62, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<63, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<64, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<65, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<66, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<67, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<68, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<69, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<70, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<71, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<72, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<73, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<74, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<75, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<76, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<77, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<78, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<79, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<80, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<81, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<82, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<83, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<84, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<85, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<86, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<87, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<88, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<89, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<90, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<91, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<92, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<93, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<94, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<95, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<96, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<97, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<98, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<99, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<100, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<101, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<102, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<103, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<104, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<105, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<106, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<107, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<108, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<109, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<110, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<111, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<112, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<113, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<114, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<115, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<116, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<117, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<118, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<119, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<120, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<121, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<122, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<123, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<124, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<125, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<126, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<<127, Rest/binary>>) -> ensure_clean(Rest); -ensure_clean(<>) when X < 16#d800 -> ensure_clean(Rest); -ensure_clean(<>) when X > 16#dfff, X < 16#fdd0 -> ensure_clean(Rest); -ensure_clean(<>) when X > 16#fdef, X < 16#fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#10000, X < 16#1fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#20000, X < 16#2fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#30000, X < 16#3fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#40000, X < 16#4fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#50000, X < 16#5fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#60000, X < 16#6fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#70000, X < 16#7fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#80000, X < 16#8fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#90000, X < 16#9fffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#a0000, X < 16#afffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#b0000, X < 16#bfffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#c0000, X < 16#cfffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#d0000, X < 16#dfffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#e0000, X < 16#efffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#f0000, X < 16#ffffe -> ensure_clean(Rest); -ensure_clean(<>) when X >= 16#100000, X < 16#10fffe -> ensure_clean(Rest); -ensure_clean(Bin) -> erlang:error(badarg, [Bin]). - - -%% escape and/or replace bad codepoints if requested -clean(<<>>, Acc, _Config) -> unicode:characters_to_binary(lists:reverse(Acc)); -clean(<<0, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(0, Config) ++ Acc, Config); -clean(<<1, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(1, Config) ++ Acc, Config); -clean(<<2, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(2, Config) ++ Acc, Config); -clean(<<3, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(3, Config) ++ Acc, Config); -clean(<<4, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(4, Config) ++ Acc, Config); -clean(<<5, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(5, Config) ++ Acc, Config); -clean(<<6, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(6, Config) ++ Acc, Config); -clean(<<7, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(7, Config) ++ Acc, Config); -clean(<<8, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(8, Config) ++ Acc, Config); -clean(<<9, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(9, Config) ++ Acc, Config); -clean(<<10, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(10, Config) ++ Acc, Config); -clean(<<11, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(11, Config) ++ Acc, Config); -clean(<<12, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(12, Config) ++ Acc, Config); -clean(<<13, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(13, Config) ++ Acc, Config); -clean(<<14, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(14, Config) ++ Acc, Config); -clean(<<15, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(15, Config) ++ Acc, Config); -clean(<<16, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(16, Config) ++ Acc, Config); -clean(<<17, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(17, Config) ++ Acc, Config); -clean(<<18, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(18, Config) ++ Acc, Config); -clean(<<19, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(19, Config) ++ Acc, Config); -clean(<<20, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(20, Config) ++ Acc, Config); -clean(<<21, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(21, Config) ++ Acc, Config); -clean(<<22, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(22, Config) ++ Acc, Config); -clean(<<23, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(23, Config) ++ Acc, Config); -clean(<<24, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(24, Config) ++ Acc, Config); -clean(<<25, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(25, Config) ++ Acc, Config); -clean(<<26, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(26, Config) ++ Acc, Config); -clean(<<27, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(27, Config) ++ Acc, Config); -clean(<<28, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(28, Config) ++ Acc, Config); -clean(<<29, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(29, Config) ++ Acc, Config); -clean(<<30, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(30, Config) ++ Acc, Config); -clean(<<31, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(31, Config) ++ Acc, Config); -clean(<<32, Rest/binary>>, Acc, Config) -> clean(Rest, [32] ++ Acc, Config); -clean(<<33, Rest/binary>>, Acc, Config) -> clean(Rest, [33] ++ Acc, Config); -clean(<<34, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(34, Config) ++ Acc, Config); -clean(<<35, Rest/binary>>, Acc, Config) -> clean(Rest, [35] ++ Acc, Config); -clean(<<36, Rest/binary>>, Acc, Config) -> clean(Rest, [36] ++ Acc, Config); -clean(<<37, Rest/binary>>, Acc, Config) -> clean(Rest, [37] ++ Acc, Config); -clean(<<38, Rest/binary>>, Acc, Config) -> clean(Rest, [38] ++ Acc, Config); -clean(<<39, Rest/binary>>, Acc, Config) -> clean(Rest, [39] ++ Acc, Config); -clean(<<40, Rest/binary>>, Acc, Config) -> clean(Rest, [40] ++ Acc, Config); -clean(<<41, Rest/binary>>, Acc, Config) -> clean(Rest, [41] ++ Acc, Config); -clean(<<42, Rest/binary>>, Acc, Config) -> clean(Rest, [42] ++ Acc, Config); -clean(<<43, Rest/binary>>, Acc, Config) -> clean(Rest, [43] ++ Acc, Config); -clean(<<44, Rest/binary>>, Acc, Config) -> clean(Rest, [44] ++ Acc, Config); -clean(<<45, Rest/binary>>, Acc, Config) -> clean(Rest, [45] ++ Acc, Config); -clean(<<46, Rest/binary>>, Acc, Config) -> clean(Rest, [46] ++ Acc, Config); -clean(<<47, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(47, Config) ++ Acc, Config); -clean(<<48, Rest/binary>>, Acc, Config) -> clean(Rest, [48] ++ Acc, Config); -clean(<<49, Rest/binary>>, Acc, Config) -> clean(Rest, [49] ++ Acc, Config); -clean(<<50, Rest/binary>>, Acc, Config) -> clean(Rest, [50] ++ Acc, Config); -clean(<<51, Rest/binary>>, Acc, Config) -> clean(Rest, [51] ++ Acc, Config); -clean(<<52, Rest/binary>>, Acc, Config) -> clean(Rest, [52] ++ Acc, Config); -clean(<<53, Rest/binary>>, Acc, Config) -> clean(Rest, [53] ++ Acc, Config); -clean(<<54, Rest/binary>>, Acc, Config) -> clean(Rest, [54] ++ Acc, Config); -clean(<<55, Rest/binary>>, Acc, Config) -> clean(Rest, [55] ++ Acc, Config); -clean(<<56, Rest/binary>>, Acc, Config) -> clean(Rest, [56] ++ Acc, Config); -clean(<<57, Rest/binary>>, Acc, Config) -> clean(Rest, [57] ++ Acc, Config); -clean(<<58, Rest/binary>>, Acc, Config) -> clean(Rest, [58] ++ Acc, Config); -clean(<<59, Rest/binary>>, Acc, Config) -> clean(Rest, [59] ++ Acc, Config); -clean(<<60, Rest/binary>>, Acc, Config) -> clean(Rest, [60] ++ Acc, Config); -clean(<<61, Rest/binary>>, Acc, Config) -> clean(Rest, [61] ++ Acc, Config); -clean(<<62, Rest/binary>>, Acc, Config) -> clean(Rest, [62] ++ Acc, Config); -clean(<<63, Rest/binary>>, Acc, Config) -> clean(Rest, [63] ++ Acc, Config); -clean(<<64, Rest/binary>>, Acc, Config) -> clean(Rest, [64] ++ Acc, Config); -clean(<<65, Rest/binary>>, Acc, Config) -> clean(Rest, [65] ++ Acc, Config); -clean(<<66, Rest/binary>>, Acc, Config) -> clean(Rest, [66] ++ Acc, Config); -clean(<<67, Rest/binary>>, Acc, Config) -> clean(Rest, [67] ++ Acc, Config); -clean(<<68, Rest/binary>>, Acc, Config) -> clean(Rest, [68] ++ Acc, Config); -clean(<<69, Rest/binary>>, Acc, Config) -> clean(Rest, [69] ++ Acc, Config); -clean(<<70, Rest/binary>>, Acc, Config) -> clean(Rest, [70] ++ Acc, Config); -clean(<<71, Rest/binary>>, Acc, Config) -> clean(Rest, [71] ++ Acc, Config); -clean(<<72, Rest/binary>>, Acc, Config) -> clean(Rest, [72] ++ Acc, Config); -clean(<<73, Rest/binary>>, Acc, Config) -> clean(Rest, [73] ++ Acc, Config); -clean(<<74, Rest/binary>>, Acc, Config) -> clean(Rest, [74] ++ Acc, Config); -clean(<<75, Rest/binary>>, Acc, Config) -> clean(Rest, [75] ++ Acc, Config); -clean(<<76, Rest/binary>>, Acc, Config) -> clean(Rest, [76] ++ Acc, Config); -clean(<<77, Rest/binary>>, Acc, Config) -> clean(Rest, [77] ++ Acc, Config); -clean(<<78, Rest/binary>>, Acc, Config) -> clean(Rest, [78] ++ Acc, Config); -clean(<<79, Rest/binary>>, Acc, Config) -> clean(Rest, [79] ++ Acc, Config); -clean(<<80, Rest/binary>>, Acc, Config) -> clean(Rest, [80] ++ Acc, Config); -clean(<<81, Rest/binary>>, Acc, Config) -> clean(Rest, [81] ++ Acc, Config); -clean(<<82, Rest/binary>>, Acc, Config) -> clean(Rest, [82] ++ Acc, Config); -clean(<<83, Rest/binary>>, Acc, Config) -> clean(Rest, [83] ++ Acc, Config); -clean(<<84, Rest/binary>>, Acc, Config) -> clean(Rest, [84] ++ Acc, Config); -clean(<<85, Rest/binary>>, Acc, Config) -> clean(Rest, [85] ++ Acc, Config); -clean(<<86, Rest/binary>>, Acc, Config) -> clean(Rest, [86] ++ Acc, Config); -clean(<<87, Rest/binary>>, Acc, Config) -> clean(Rest, [87] ++ Acc, Config); -clean(<<88, Rest/binary>>, Acc, Config) -> clean(Rest, [88] ++ Acc, Config); -clean(<<89, Rest/binary>>, Acc, Config) -> clean(Rest, [89] ++ Acc, Config); -clean(<<90, Rest/binary>>, Acc, Config) -> clean(Rest, [90] ++ Acc, Config); -clean(<<91, Rest/binary>>, Acc, Config) -> clean(Rest, [91] ++ Acc, Config); -clean(<<92, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(92, Config) ++ Acc, Config); -clean(<<93, Rest/binary>>, Acc, Config) -> clean(Rest, [93] ++ Acc, Config); -clean(<<94, Rest/binary>>, Acc, Config) -> clean(Rest, [94] ++ Acc, Config); -clean(<<95, Rest/binary>>, Acc, Config) -> clean(Rest, [95] ++ Acc, Config); -clean(<<96, Rest/binary>>, Acc, Config) -> clean(Rest, [96] ++ Acc, Config); -clean(<<97, Rest/binary>>, Acc, Config) -> clean(Rest, [97] ++ Acc, Config); -clean(<<98, Rest/binary>>, Acc, Config) -> clean(Rest, [98] ++ Acc, Config); -clean(<<99, Rest/binary>>, Acc, Config) -> clean(Rest, [99] ++ Acc, Config); -clean(<<100, Rest/binary>>, Acc, Config) -> clean(Rest, [100] ++ Acc, Config); -clean(<<101, Rest/binary>>, Acc, Config) -> clean(Rest, [101] ++ Acc, Config); -clean(<<102, Rest/binary>>, Acc, Config) -> clean(Rest, [102] ++ Acc, Config); -clean(<<103, Rest/binary>>, Acc, Config) -> clean(Rest, [103] ++ Acc, Config); -clean(<<104, Rest/binary>>, Acc, Config) -> clean(Rest, [104] ++ Acc, Config); -clean(<<105, Rest/binary>>, Acc, Config) -> clean(Rest, [105] ++ Acc, Config); -clean(<<106, Rest/binary>>, Acc, Config) -> clean(Rest, [106] ++ Acc, Config); -clean(<<107, Rest/binary>>, Acc, Config) -> clean(Rest, [107] ++ Acc, Config); -clean(<<108, Rest/binary>>, Acc, Config) -> clean(Rest, [108] ++ Acc, Config); -clean(<<109, Rest/binary>>, Acc, Config) -> clean(Rest, [109] ++ Acc, Config); -clean(<<110, Rest/binary>>, Acc, Config) -> clean(Rest, [110] ++ Acc, Config); -clean(<<111, Rest/binary>>, Acc, Config) -> clean(Rest, [111] ++ Acc, Config); -clean(<<112, Rest/binary>>, Acc, Config) -> clean(Rest, [112] ++ Acc, Config); -clean(<<113, Rest/binary>>, Acc, Config) -> clean(Rest, [113] ++ Acc, Config); -clean(<<114, Rest/binary>>, Acc, Config) -> clean(Rest, [114] ++ Acc, Config); -clean(<<115, Rest/binary>>, Acc, Config) -> clean(Rest, [115] ++ Acc, Config); -clean(<<116, Rest/binary>>, Acc, Config) -> clean(Rest, [116] ++ Acc, Config); -clean(<<117, Rest/binary>>, Acc, Config) -> clean(Rest, [117] ++ Acc, Config); -clean(<<118, Rest/binary>>, Acc, Config) -> clean(Rest, [118] ++ Acc, Config); -clean(<<119, Rest/binary>>, Acc, Config) -> clean(Rest, [119] ++ Acc, Config); -clean(<<120, Rest/binary>>, Acc, Config) -> clean(Rest, [120] ++ Acc, Config); -clean(<<121, Rest/binary>>, Acc, Config) -> clean(Rest, [121] ++ Acc, Config); -clean(<<122, Rest/binary>>, Acc, Config) -> clean(Rest, [122] ++ Acc, Config); -clean(<<123, Rest/binary>>, Acc, Config) -> clean(Rest, [123] ++ Acc, Config); -clean(<<124, Rest/binary>>, Acc, Config) -> clean(Rest, [124] ++ Acc, Config); -clean(<<125, Rest/binary>>, Acc, Config) -> clean(Rest, [125] ++ Acc, Config); -clean(<<126, Rest/binary>>, Acc, Config) -> clean(Rest, [126] ++ Acc, Config); -clean(<<127, Rest/binary>>, Acc, Config) -> clean(Rest, [127] ++ Acc, Config); -clean(<>, Acc, Config) when X < 16#800 -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X == 16#2028; X == 16#2029 -> - clean(Rest, maybe_replace(X, Config) ++ Acc, Config); -clean(<>, Acc, Config) when X < 16#d800 -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X > 16#dfff, X < 16#fdd0 -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X > 16#fdef, X < 16#fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#10000, X < 16#1fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#20000, X < 16#2fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#30000, X < 16#3fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#40000, X < 16#4fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#50000, X < 16#5fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#60000, X < 16#6fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#70000, X < 16#7fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#80000, X < 16#8fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#90000, X < 16#9fffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#a0000, X < 16#afffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#b0000, X < 16#bfffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#c0000, X < 16#cfffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#d0000, X < 16#dfffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#e0000, X < 16#efffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#f0000, X < 16#ffffe -> - clean(Rest, [X] ++ Acc, Config); -clean(<>, Acc, Config) when X >= 16#100000, X < 16#10fffe -> - clean(Rest, [X] ++ Acc, Config); -%% noncharacters -clean(<<_/utf8, Rest/binary>>, Acc, Config) -> - clean(Rest, maybe_replace(noncharacter, Config) ++ Acc, Config); +cut(<<>>, N) -> {N, finished}; +cut(<>, N) when X < 32 -> {N, escape}; +cut(<<$\"/utf8, _/binary>>, N) -> {N, escape}; +cut(<<$//utf8, _/binary>>, N) -> {N, escape}; +cut(<<$\\/utf8, _/binary>>, N) -> {N, escape}; +cut(<>, N) when X < 128 -> cut(Rest, N + 1); +cut(<>, N) when X < 16#0800 -> cut(Rest, N + 2); +cut(<>, N) when X == 16#2028; X == 16#2029 -> {N, escape}; +cut(<>, N) when X < 16#d800 -> cut(Rest, N + 3); +cut(<>, N) when X > 16#dfff, X < 16#fdd0 -> cut(Rest, N + 3); +cut(<>, N) when X > 16#fdef, X < 16#fffe -> cut(Rest, N + 3); +cut(<>, N) when X >= 16#10000, X < 16#1fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#20000, X < 16#2fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#30000, X < 16#3fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#40000, X < 16#4fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#50000, X < 16#5fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#60000, X < 16#6fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#70000, X < 16#7fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#80000, X < 16#8fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#90000, X < 16#9fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#a0000, X < 16#afffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#b0000, X < 16#bfffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#c0000, X < 16#cfffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#d0000, X < 16#dfffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#e0000, X < 16#efffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#f0000, X < 16#ffffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#100000, X < 16#10fffe -> cut(Rest, N + 4); +%% noncharacters and reserved space +cut(<>, N) -> + {N, replace, case X of Y when Y < 16#10000 -> 3; _ -> 4 end}; %% surrogates -clean(<<237, X, _, Rest/binary>>, Acc, Config) when X >= 160 -> - clean(Rest, maybe_replace(surrogate, Config) ++ Acc, Config); +cut(<<237, X, _, _/binary>>, N) when X >= 160 -> {N, replace, 3}; %% u+fffe and u+ffff for R14BXX -clean(<<239, 191, X, Rest/binary>>, Acc, Config) when X == 190; X == 191 -> - clean(Rest, maybe_replace(noncharacter, Config) ++ Acc, Config); +cut(<<239, 191, X, _/binary>>, N) when X == 190; X == 191 -> {N, replace, 3}; %% overlong encodings and missing continuations of a 2 byte sequence -clean(<>, Acc, Config) when X >= 192, X =< 223 -> - clean(strip_continuations(Rest, 1), maybe_replace(badutf, Config) ++ Acc, Config); +cut(<>, N) when X >= 192, X =< 223 -> + {N, replace, 1 + count_continuations(Rest, 1)}; %% overlong encodings and missing continuations of a 3 byte sequence -clean(<>, Acc, Config) when X >= 224, X =< 239 -> - clean(strip_continuations(Rest, 2), maybe_replace(badutf, Config) ++ Acc, Config); +cut(<>, N) when X >= 224, X =< 239 -> + {N, replace, 1 + count_continuations(Rest, 2)}; %% overlong encodings and missing continuations of a 4 byte sequence -clean(<>, Acc, Config) when X >= 240, X =< 247 -> - clean(strip_continuations(Rest, 3), maybe_replace(badutf, Config) ++ Acc, Config); -clean(<<_, Rest/binary>>, Acc, Config) -> - clean(Rest, maybe_replace(badutf, Config) ++ Acc, Config). +cut(<>, N) when X >= 240, X =< 247 -> + {N, replace, 1 + count_continuations(Rest, 3)}; +cut(<<_, _/binary>>, N) -> {N, replace, 1}. -strip_continuations(Bin, 0) -> Bin; -strip_continuations(<>, N) when X >= 128, X =< 191 -> - strip_continuations(Rest, N - 1); +count_continuations(Bin, N) -> count_continuations(Bin, N, 0). + +count_continuations(_Bin, 0, Acc) -> Acc; +count_continuations(<>, N, Acc) when X >= 128, X =< 191 -> + count_continuations(Rest, N - 1, Acc + 1); %% not a continuation byte -strip_continuations(Bin, _) -> Bin. +count_continuations(_Bin, _, Acc) -> Acc. -maybe_escape(Escaped, #config{dirty_strings=true}) -> <>; maybe_escape(Escaped, #config{escaped_strings=true} = Config) -> escape(Escaped, Config); maybe_escape(Escaped, _Config) -> <>. @@ -527,29 +240,9 @@ escape(X, _) when X < 32 -> <<"\\u", (to_hex(A)), (to_hex(B)), (to_hex(C)), (to_hex(D))>>. -maybe_replace($\b, #config{escaped_strings=true}) -> [$b, $\\]; -maybe_replace($\t, #config{escaped_strings=true}) -> [$t, $\\]; -maybe_replace($\n, #config{escaped_strings=true}) -> [$n, $\\]; -maybe_replace($\f, #config{escaped_strings=true}) -> [$f, $\\]; -maybe_replace($\r, #config{escaped_strings=true}) -> [$r, $\\]; -maybe_replace($\", #config{escaped_strings=true}) -> [$\", $\\]; -maybe_replace($\\, #config{escaped_strings=true}) -> [$\\, $\\]; -maybe_replace($/, Config) -> - case Config#config.escaped_forward_slashes of - true -> [$/, $\\] - ; false -> [$/] - end; -maybe_replace(X, Config=#config{escaped_strings=true}) when X == 16#2028; X == 16#2029 -> - case Config#config.unescaped_jsonp of - true -> [X] - ; false -> lists:reverse(jsx_utils:json_escape_sequence(X)) - end; -maybe_replace(X, #config{escaped_strings=true}) when X < 32 -> - lists:reverse(jsx_utils:json_escape_sequence(X)); -maybe_replace(noncharacter, #config{replaced_bad_utf8=true}) -> [16#fffd]; -maybe_replace(surrogate, #config{replaced_bad_utf8=true}) -> [16#fffd]; -maybe_replace(badutf, #config{replaced_bad_utf8=true}) -> [16#fffd]; -maybe_replace(_, _) -> erlang:error(badarg). +maybe_replace(_, #config{replaced_bad_utf8=true}) -> <<16#fffd/utf8>>; +maybe_replace(_, _Config) -> {error, badarg}. + @@ -661,11 +354,6 @@ codepoints() -> ++ lists:seq(16#fdf0, 16#fffd) ). -escapables() -> - [ to_fake_utf8(N) || N <- - lists:seq(0, 31) ++ [34, 92, 16#2028, 16#2029] - ]. - extended_codepoints() -> unicode:characters_to_binary( lists:seq(16#10000, 16#1fffd) ++ [ @@ -675,6 +363,10 @@ extended_codepoints() -> ] ). +reserved_space() -> [ to_fake_utf8(N) || N <- lists:seq(16#fdd0, 16#fdef) ]. + +surrogates() -> [ to_fake_utf8(N) || N <- lists:seq(16#d800, 16#dfff) ]. + noncharacters() -> [ to_fake_utf8(N) || N <- lists:seq(16#fffe, 16#ffff) ]. extended_noncharacters() -> @@ -688,83 +380,81 @@ extended_noncharacters() -> ++ [16#ffffe, 16#fffff, 16#10fffe, 16#10ffff] ]. -surrogates() -> [ to_fake_utf8(N) || N <- lists:seq(16#d800, 16#dfff) ]. -reserved_space() -> [ to_fake_utf8(N) || N <- lists:seq(16#fdd0, 16#fdef) ]. - - -fail_ensure(Codepoints, Title) -> - {generator, - fun() -> case Codepoints of - [N|Rest] -> - [ {Title, ?_assertError(badarg, ensure_clean(N))} - | fail_ensure(Rest, Title) - ] - ; [] -> [] - end end - }. - -fail_clean(Codepoints, Title) -> - {generator, - fun() -> case Codepoints of - [N|Rest] -> - [ {Title, ?_assertError(badarg, clean(N, [], #config{}))} - | fail_clean(Rest, Title) - ] - ; [] -> [] - end end - }. - -fail_bad(Codepoints, Title) -> - {generator, - fun() -> case Codepoints of - [N|Rest] -> - [ {Title, ?_assertError(badarg, clean(N, [], #config{}))} - | fail_bad(Rest, Title) - ] - ; [] -> [] - end end - }. - -replace_bad(Codepoints, Title) -> - {generator, - fun() -> case Codepoints of - [N|Rest] -> - [ {Title, ?_assertEqual(<<16#fffd/utf8>>, clean(N, [], #config{replaced_bad_utf8=true}))} - | replace_bad(Rest, Title) - ] - ; [] -> [] - end end - }. - - -ensure_clean_test_() -> +clean_string_test_() -> [ - {"basic codepoints", ?_assertEqual(ok, ensure_clean(codepoints()))}, - {"escapables", ?_assertEqual(ok, ensure_clean(unicode:characters_to_binary(escapables())))}, - {"extended codepoints", ?_assertEqual(ok, ensure_clean(extended_codepoints()))}, - fail_ensure(noncharacters(), "noncharacters"), - fail_ensure(extended_noncharacters(), "extended noncharacters"), - fail_ensure(surrogates(), "surrogates"), - fail_ensure(reserved_space(), "reserved space") - ]. - - -clean_test_() -> - [ - {"basic codepoints", ?_assertEqual( + {"clean codepoints test", ?_assertEqual( codepoints(), - clean(codepoints(), [], #config{}) + clean_string(codepoints(), #config{}) )}, - fail_clean(escapables(), "escapables"), - {"extended codepoints", ?_assertEqual( + {"clean extended codepoints test", ?_assertEqual( extended_codepoints(), - clean(extended_codepoints(), [], #config{}) - )}, - fail_clean(noncharacters(), "noncharacters"), - fail_clean(extended_noncharacters(), "extended noncharacters"), - fail_clean(surrogates(), "surrogates"), - fail_clean(reserved_space(), "reserved space") + clean_string(extended_codepoints(), #config{}) + )} + ] ++ [ + { + "reserved character: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), + ?_assertEqual( + {error, badarg}, + clean_string(Codepoint, #config{}) + ) + } || Codepoint <- reserved_space() + ] ++ [ + { + "reserved character: " ++ lists:flatten(io_lib:format("~p", [Codepoint])) ++ " (replaced)", + ?_assertEqual( + <<16#fffd/utf8>>, + clean_string(Codepoint, #config{replaced_bad_utf8=true}) + ) + } || Codepoint <- reserved_space() + ] ++ [ + { + "surrogate: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), + ?_assertEqual( + {error, badarg}, + clean_string(Codepoint, #config{}) + ) + } || Codepoint <- surrogates() + ] ++ [ + { + "surrogate: " ++ lists:flatten(io_lib:format("~p", [Codepoint])) ++ " (replaced)", + ?_assertEqual( + <<16#fffd/utf8>>, + clean_string(Codepoint, #config{replaced_bad_utf8=true}) + ) + } || Codepoint <- surrogates() + ] ++ [ + { + "noncharacter: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), + ?_assertEqual( + {error, badarg}, + clean_string(Codepoint, #config{}) + ) + } || Codepoint <- noncharacters() + ] ++ [ + { + "noncharacter: " ++ lists:flatten(io_lib:format("~p", [Codepoint])) ++ " (replaced)", + ?_assertEqual( + <<16#fffd/utf8>>, + clean_string(Codepoint, #config{replaced_bad_utf8=true}) + ) + } || Codepoint <- noncharacters() + ] ++ [ + { + "extended noncharacter: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), + ?_assertEqual( + {error, badarg}, + clean_string(Codepoint, #config{}) + ) + } || Codepoint <- extended_noncharacters() + ] ++ [ + { + "extended noncharacter: " ++ lists:flatten(io_lib:format("~p", [Codepoint])) ++ " (replaced)", + ?_assertEqual( + <<16#fffd/utf8>>, + clean_string(Codepoint, #config{replaced_bad_utf8=true}) + ) + } || Codepoint <- extended_noncharacters() ]. @@ -933,50 +623,46 @@ escape_test_() -> {"maybe_escape u001f", ?_assertEqual( <<"\\u001f">>, maybe_escape(16#001f, #config{escaped_strings=true}) - )}, - {"dirty strings", ?_assertEqual( - <<0>>, - maybe_escape(16#0000, #config{escaped_strings=true, dirty_strings=true}) )} ]. bad_utf8_test_() -> [ - {"noncharacter u+fffe", ?_assertError(badarg, clean_string(to_fake_utf8(16#fffe), #config{}))}, + {"noncharacter u+fffe", ?_assertEqual( + {error, badarg}, + clean_string(to_fake_utf8(16#fffe), #config{}) + )}, {"noncharacter u+fffe replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(to_fake_utf8(16#fffe), #config{replaced_bad_utf8=true}) )}, - {"noncharacter u+ffff", ?_assertError(badarg, clean_string(to_fake_utf8(16#ffff), #config{}))}, + {"noncharacter u+ffff", ?_assertEqual( + {error, badarg}, + clean_string(to_fake_utf8(16#ffff), #config{}) + )}, {"noncharacter u+ffff replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(to_fake_utf8(16#ffff), #config{replaced_bad_utf8=true}) )}, - fail_bad(extended_noncharacters(), "extended noncharacters"), - replace_bad(extended_noncharacters(), "extended noncharacters replaced"), - fail_bad(surrogates(), "surrogates"), - replace_bad(surrogates(), "surrogates replaced"), - fail_bad(reserved_space(), "reserved space"), - replace_bad(reserved_space(), "reserved space replaced"), - {"orphan continuation byte u+0080", ?_assertError( - badarg, + {"orphan continuation byte u+0080", ?_assertEqual( + {error, badarg}, clean_string(<<16#0080>>, #config{}) )}, {"orphan continuation byte u+0080 replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(<<16#0080>>, #config{replaced_bad_utf8=true}) )}, - {"orphan continuation byte u+00bf", ?_assertError( - badarg, + {"orphan continuation byte u+00bf", ?_assertEqual( + {error, badarg}, clean_string(<<16#00bf>>, #config{}) )}, {"orphan continuation byte u+00bf replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(<<16#00bf>>, #config{replaced_bad_utf8=true}) )}, - {"2 continuation bytes", ?_assertError( - badarg, + {"2 continuation bytes", ?_assertEqual( + {error, badarg}, clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #config{}) )}, {"2 continuation bytes replaced", ?_assertEqual( @@ -984,35 +670,35 @@ bad_utf8_test_() -> clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #config{replaced_bad_utf8=true}) )}, {"3 continuation bytes", - ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{})) + ?_assertEqual({error, badarg}, clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{})) }, {"3 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 3), clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{replaced_bad_utf8=true}) )}, {"4 continuation bytes", - ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{})) + ?_assertEqual({error, badarg}, clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{})) }, {"4 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 4), clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{replaced_bad_utf8=true}) )}, {"5 continuation bytes", - ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{})) + ?_assertEqual({error, badarg}, clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{})) }, {"5 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 5), clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{replaced_bad_utf8=true}) )}, {"6 continuation bytes", - ?_assertError(badarg, clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{})) + ?_assertEqual({error, badarg}, clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{})) }, {"6 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 6), clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{replaced_bad_utf8=true}) )}, - {"all continuation bytes", ?_assertError( - badarg, + {"all continuation bytes", ?_assertEqual( + {error, badarg}, clean_string(<<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, #config{}) )}, {"all continuation bytes replaced", ?_assertEqual( @@ -1022,101 +708,101 @@ bad_utf8_test_() -> #config{replaced_bad_utf8=true} ) )}, - {"lonely start byte", ?_assertError(badarg, clean_string(<<16#00c0>>, #config{}))}, + {"lonely start byte", ?_assertEqual({error, badarg}, clean_string(<<16#00c0>>, #config{}))}, {"lonely start byte replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(<<16#00c0>>, #config{replaced_bad_utf8=true}) )}, - {"lonely start bytes (2 byte)", ?_assertError( - badarg, + {"lonely start bytes (2 byte)", ?_assertEqual( + {error, badarg}, clean_string(<<16#00c0, 32, 16#00df>>, #config{}) )}, {"lonely start bytes (2 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, clean_string(<<16#00c0, 32, 16#00df>>, #config{replaced_bad_utf8=true}) )}, - {"lonely start bytes (3 byte)", ?_assertError( - badarg, + {"lonely start bytes (3 byte)", ?_assertEqual( + {error, badarg}, clean_string(<<16#00e0, 32, 16#00ef>>, #config{}) )}, {"lonely start bytes (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, clean_string(<<16#00e0, 32, 16#00ef>>, #config{replaced_bad_utf8=true}) )}, - {"lonely start bytes (4 byte)", ?_assertError( - badarg, + {"lonely start bytes (4 byte)", ?_assertEqual( + {error, badarg}, clean_string(<<16#00f0, 32, 16#00f7>>, #config{}) )}, {"lonely start bytes (4 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, clean_string(<<16#00f0, 32, 16#00f7>>, #config{replaced_bad_utf8=true}) )}, - {"missing continuation byte (3 byte)", ?_assertError( - badarg, + {"missing continuation byte (3 byte)", ?_assertEqual( + {error, badarg}, clean_string(<<224, 160, 32>>, #config{}) )}, {"missing continuation byte (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<224, 160, 32>>, #config{replaced_bad_utf8=true}) )}, - {"missing continuation byte (4 byte missing one)", ?_assertError( - badarg, + {"missing continuation byte (4 byte missing one)", ?_assertEqual( + {error, badarg}, clean_string(<<240, 144, 128, 32>>, #config{}) )}, {"missing continuation byte (4 byte missing one) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<240, 144, 128, 32>>, #config{replaced_bad_utf8=true}) )}, - {"missing continuation byte (4 byte missing two)", ?_assertError( - badarg, + {"missing continuation byte (4 byte missing two)", ?_assertEqual( + {error, badarg}, clean_string(<<240, 144, 32>>, #config{}) )}, {"missing continuation byte (4 byte missing two) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<240, 144, 32>>, #config{replaced_bad_utf8=true}) )}, - {"overlong encoding of u+002f (2 byte)", ?_assertError( - badarg, + {"overlong encoding of u+002f (2 byte)", ?_assertEqual( + {error, badarg}, clean_string(<<16#c0, 16#af, 32>>, #config{}) )}, {"overlong encoding of u+002f (2 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#c0, 16#af, 32>>, #config{replaced_bad_utf8=true}) )}, - {"overlong encoding of u+002f (3 byte)", ?_assertError( - badarg, + {"overlong encoding of u+002f (3 byte)", ?_assertEqual( + {error, badarg}, clean_string(<<16#e0, 16#80, 16#af, 32>>, #config{}) )}, {"overlong encoding of u+002f (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#e0, 16#80, 16#af, 32>>, #config{replaced_bad_utf8=true}) )}, - {"overlong encoding of u+002f (4 byte)", ?_assertError( - badarg, + {"overlong encoding of u+002f (4 byte)", ?_assertEqual( + {error, badarg}, clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #config{}) )}, {"overlong encoding of u+002f (4 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #config{replaced_bad_utf8=true}) )}, - {"highest overlong 2 byte sequence", ?_assertError( - badarg, + {"highest overlong 2 byte sequence", ?_assertEqual( + {error, badarg}, clean_string(<<16#c1, 16#bf, 32>>, #config{}) )}, {"highest overlong 2 byte sequence replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#c1, 16#bf, 32>>, #config{replaced_bad_utf8=true}) )}, - {"highest overlong 3 byte sequence", ?_assertError( - badarg, + {"highest overlong 3 byte sequence", ?_assertEqual( + {error, badarg}, clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #config{}) )}, {"highest overlong 3 byte sequence replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #config{replaced_bad_utf8=true}) )}, - {"highest overlong 4 byte sequence", ?_assertError( - badarg, + {"highest overlong 4 byte sequence", ?_assertEqual( + {error, badarg}, clean_string(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, #config{}) )}, {"highest overlong 4 byte sequence replaced", ?_assertEqual( From de76cf48a7190c48a0369b2d01086c785c3b48ac Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Wed, 20 Feb 2013 20:00:03 -0800 Subject: [PATCH 45/61] streamline string parsing implementation in encoders --- src/jsx_utils.erl | 283 +++++++++++++++++++++++++++++++--------------- 1 file changed, 191 insertions(+), 92 deletions(-) diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index 391d890..f15edb2 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -147,32 +147,113 @@ to_hex(X) -> X + 48. %% ascii "1" is [49], "2" is [50], etc... clean_string(Bin, #config{dirty_strings=true}) -> Bin; clean_string(Bin, Config) -> clean_string(Bin, <<>>, Config). - clean_string(Bin, Acc, Config) -> - case cut(Bin, 0) of - {_, finished} -> <>; - {X, escape} -> - <> = Bin, - Escaped = maybe_escape(Codepoint, Config), - clean_string(Rest, <>, Config); - {X, replace, Y} -> - <> = Bin, - case maybe_replace(Bad, Config) of - {error, badarg} -> {error, badarg}; - Replaced -> clean_string(Rest, <>, Config) - end + Length = cut(Bin), + <> = Bin, + case Rest of + <<>> -> <>; + _ -> maybe_escape(Rest, <>, Config) end. -cut(<<>>, N) -> {N, finished}; -cut(<>, N) when X < 32 -> {N, escape}; -cut(<<$\"/utf8, _/binary>>, N) -> {N, escape}; -cut(<<$//utf8, _/binary>>, N) -> {N, escape}; -cut(<<$\\/utf8, _/binary>>, N) -> {N, escape}; -cut(<>, N) when X < 128 -> cut(Rest, N + 1); -cut(<>, N) when X < 16#0800 -> cut(Rest, N + 2); -cut(<>, N) when X == 16#2028; X == 16#2029 -> {N, escape}; -cut(<>, N) when X < 16#d800 -> cut(Rest, N + 3); +cut(Bin) -> cut(Bin, 0). + +cut(<<32, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<33, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<35, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<36, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<37, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<38, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<39, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<40, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<41, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<42, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<43, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<44, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<45, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<46, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<48, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<49, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<50, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<51, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<52, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<53, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<54, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<55, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<56, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<57, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<58, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<59, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<60, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<61, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<62, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<63, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<64, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<65, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<66, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<67, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<68, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<69, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<70, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<71, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<72, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<73, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<74, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<75, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<76, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<77, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<78, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<79, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<80, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<81, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<82, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<83, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<84, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<85, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<86, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<87, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<88, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<89, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<90, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<91, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<93, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<94, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<95, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<96, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<97, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<98, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<99, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<100, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<101, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<102, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<103, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<104, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<105, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<106, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<107, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<108, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<109, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<110, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<111, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<112, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<113, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<114, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<115, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<116, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<117, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<118, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<119, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<120, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<121, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<122, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<123, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<124, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<125, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<126, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<127, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<>, N) when X >= 128, X < 16#0800 -> cut(Rest, N + 2); +cut(<>, N) when X >= 16#0800, X < 16#2028 -> cut(Rest, N + 3); +cut(<>, N) when X >= 16#202a, X < 16#d800 -> cut(Rest, N + 3); cut(<>, N) when X > 16#dfff, X < 16#fdd0 -> cut(Rest, N + 3); cut(<>, N) when X > 16#fdef, X < 16#fffe -> cut(Rest, N + 3); cut(<>, N) when X >= 16#10000, X < 16#1fffe -> cut(Rest, N + 4); @@ -191,36 +272,28 @@ cut(<>, N) when X >= 16#d0000, X < 16#dfffe -> cut(Rest, N cut(<>, N) when X >= 16#e0000, X < 16#efffe -> cut(Rest, N + 4); cut(<>, N) when X >= 16#f0000, X < 16#ffffe -> cut(Rest, N + 4); cut(<>, N) when X >= 16#100000, X < 16#10fffe -> cut(Rest, N + 4); -%% noncharacters and reserved space -cut(<>, N) -> - {N, replace, case X of Y when Y < 16#10000 -> 3; _ -> 4 end}; -%% surrogates -cut(<<237, X, _, _/binary>>, N) when X >= 160 -> {N, replace, 3}; -%% u+fffe and u+ffff for R14BXX -cut(<<239, 191, X, _/binary>>, N) when X == 190; X == 191 -> {N, replace, 3}; -%% overlong encodings and missing continuations of a 2 byte sequence -cut(<>, N) when X >= 192, X =< 223 -> - {N, replace, 1 + count_continuations(Rest, 1)}; -%% overlong encodings and missing continuations of a 3 byte sequence -cut(<>, N) when X >= 224, X =< 239 -> - {N, replace, 1 + count_continuations(Rest, 2)}; -%% overlong encodings and missing continuations of a 4 byte sequence -cut(<>, N) when X >= 240, X =< 247 -> - {N, replace, 1 + count_continuations(Rest, 3)}; -cut(<<_, _/binary>>, N) -> {N, replace, 1}. +cut(_, N) -> N. -count_continuations(Bin, N) -> count_continuations(Bin, N, 0). - -count_continuations(_Bin, 0, Acc) -> Acc; -count_continuations(<>, N, Acc) when X >= 128, X =< 191 -> - count_continuations(Rest, N - 1, Acc + 1); +strip_continuations(Bin, 0) -> Bin; +strip_continuations(<>, N) when X >= 128, X =< 191 -> + strip_continuations(Rest, N - 1); %% not a continuation byte -count_continuations(_Bin, _, Acc) -> Acc. +strip_continuations(Bin, _) -> Bin. -maybe_escape(Escaped, #config{escaped_strings=true} = Config) -> escape(Escaped, Config); -maybe_escape(Escaped, _Config) -> <>. +maybe_escape(<>, Acc, #config{escaped_strings=true} = Config) -> + case escape(Codepoint, Config) of + inescapable -> noncharacter(<>, Acc, Config); + Escaped -> clean_string(Rest, <>, Config) + end; +maybe_escape(<>, Acc, Config) -> + case escape(Codepoint, Config) of + inescapable -> noncharacter(<>, Acc, Config); + _ -> clean_string(Rest, <>, Config) + end; +maybe_escape(Bin, Acc, Config) -> noncharacter(Bin, Acc, Config). + escape($\b, _) -> <<"\\b">>; escape($\t, _) -> <<"\\t">>; @@ -237,13 +310,36 @@ escape(16#2029, #config{unescaped_jsonp=true}) -> <<16#2029/utf8>>; escape(16#2029, _) -> <<"\\u2029">>; escape(X, _) when X < 32 -> <> = <>, - <<"\\u", (to_hex(A)), (to_hex(B)), (to_hex(C)), (to_hex(D))>>. + <<"\\u", (to_hex(A)), (to_hex(B)), (to_hex(C)), (to_hex(D))>>; +escape(_, _) -> inescapable. -maybe_replace(_, #config{replaced_bad_utf8=true}) -> <<16#fffd/utf8>>; -maybe_replace(_, _Config) -> {error, badarg}. +%% noncharacters and reserved space +noncharacter(<<_/utf8, Rest/binary>>, Acc, Config) -> + maybe_replace(Rest, Acc, Config); +%% surrogates +noncharacter(<<237, X, _, Rest/binary>>, Acc, Config) when X >= 160 -> + maybe_replace(Rest, Acc, Config); +%% u+fffe and u+ffff for R14BXX +noncharacter(<<239, 191, X, Rest/binary>>, Acc, Config) when X == 190; X == 191 -> + maybe_replace(Rest, Acc, Config); +%% overlong encodings and missing continuations of a 2 byte sequence +noncharacter(<>, Acc, Config) when X >= 192, X =< 223 -> + maybe_replace(strip_continuations(Rest, 1), Acc, Config); +%% overlong encodings and missing continuations of a 3 byte sequence +noncharacter(<>, Acc, Config) when X >= 224, X =< 239 -> + maybe_replace(strip_continuations(Rest, 2), Acc, Config); +%% overlong encodings and missing continuations of a 4 byte sequence +noncharacter(<>, Acc, Config) when X >= 240, X =< 247 -> + maybe_replace(strip_continuations(Rest, 3), Acc, Config); +noncharacter(<<_, Rest/binary>>, Acc, Config) -> + maybe_replace(Rest, Acc, Config). +maybe_replace(Bin, Acc, #config{replaced_bad_utf8=true} = Config) -> + clean_string(Bin, <>, Config); +maybe_replace(_, _, _) -> {error, badarg}. + %% eunit tests @@ -347,7 +443,8 @@ to_fake_utf8(N) -> codepoints() -> unicode:characters_to_binary( [32, 33] - ++ lists:seq(35, 91) + ++ lists:seq(35, 46) + ++ lists:seq(48, 91) ++ lists:seq(93, 16#2027) ++ lists:seq(16#202a, 16#d7ff) ++ lists:seq(16#e000, 16#fdcf) @@ -458,171 +555,173 @@ clean_string_test_() -> ]. +maybe_escape(Bin, Config) -> clean_string(Bin, Config). + escape_test_() -> [ {"maybe_escape backspace", ?_assertEqual( <<"\\b">>, - maybe_escape(16#0008, #config{escaped_strings=true}) + maybe_escape(<<16#0008/utf8>>, #config{escaped_strings=true}) )}, {"don't escape backspace", ?_assertEqual( <<"\b">>, - maybe_escape(16#0008, #config{}) + maybe_escape(<<16#0008/utf8>>, #config{}) )}, {"maybe_escape tab", ?_assertEqual( <<"\\t">>, - maybe_escape(16#0009, #config{escaped_strings=true}) + maybe_escape(<<16#0009/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape newline", ?_assertEqual( <<"\\n">>, - maybe_escape(16#000a, #config{escaped_strings=true}) + maybe_escape(<<16#000a/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape formfeed", ?_assertEqual( <<"\\f">>, - maybe_escape(16#000c, #config{escaped_strings=true}) + maybe_escape(<<16#000c/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape carriage return", ?_assertEqual( <<"\\r">>, - maybe_escape(16#000d, #config{escaped_strings=true}) + maybe_escape(<<16#000d/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape quote", ?_assertEqual( <<"\\\"">>, - maybe_escape(16#0022, #config{escaped_strings=true}) + maybe_escape(<<16#0022/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape forward slash", ?_assertEqual( <<"\\/">>, - maybe_escape(16#002f, #config{escaped_strings=true, escaped_forward_slashes=true}) + maybe_escape(<<16#002f/utf8>>, #config{escaped_strings=true, escaped_forward_slashes=true}) )}, {"do not maybe_escape forward slash", ?_assertEqual( <<"/">>, - maybe_escape(16#002f, #config{escaped_strings=true}) + maybe_escape(<<16#002f/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape backslash", ?_assertEqual( <<"\\\\">>, - maybe_escape(16#005c, #config{escaped_strings=true}) + maybe_escape(<<16#005c/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape jsonp (u2028)", ?_assertEqual( <<"\\u2028">>, - maybe_escape(16#2028, #config{escaped_strings=true}) + maybe_escape(<<16#2028/utf8>>, #config{escaped_strings=true}) )}, {"do not maybe_escape jsonp (u2028)", ?_assertEqual( <<16#2028/utf8>>, - maybe_escape(16#2028, #config{escaped_strings=true, unescaped_jsonp=true}) + maybe_escape(<<16#2028/utf8>>, #config{escaped_strings=true, unescaped_jsonp=true}) )}, {"maybe_escape jsonp (u2029)", ?_assertEqual( <<"\\u2029">>, - maybe_escape(16#2029, #config{escaped_strings=true}) + maybe_escape(<<16#2029/utf8>>, #config{escaped_strings=true}) )}, {"do not maybe_escape jsonp (u2029)", ?_assertEqual( <<16#2029/utf8>>, - maybe_escape(16#2029, #config{escaped_strings=true, unescaped_jsonp=true}) + maybe_escape(<<16#2029/utf8>>, #config{escaped_strings=true, unescaped_jsonp=true}) )}, {"maybe_escape u0000", ?_assertEqual( <<"\\u0000">>, - maybe_escape(16#0000, #config{escaped_strings=true}) + maybe_escape(<<16#0000/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u0001", ?_assertEqual( <<"\\u0001">>, - maybe_escape(16#0001, #config{escaped_strings=true}) + maybe_escape(<<16#0001/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u0002", ?_assertEqual( <<"\\u0002">>, - maybe_escape(16#0002, #config{escaped_strings=true}) + maybe_escape(<<16#0002/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u0003", ?_assertEqual( <<"\\u0003">>, - maybe_escape(16#0003, #config{escaped_strings=true}) + maybe_escape(<<16#0003/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u0004", ?_assertEqual( <<"\\u0004">>, - maybe_escape(16#0004, #config{escaped_strings=true}) + maybe_escape(<<16#0004/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u0005", ?_assertEqual( <<"\\u0005">>, - maybe_escape(16#0005, #config{escaped_strings=true}) + maybe_escape(<<16#0005/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u0006", ?_assertEqual( <<"\\u0006">>, - maybe_escape(16#0006, #config{escaped_strings=true}) + maybe_escape(<<16#0006/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u0007", ?_assertEqual( <<"\\u0007">>, - maybe_escape(16#0007, #config{escaped_strings=true}) + maybe_escape(<<16#0007/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u000b", ?_assertEqual( <<"\\u000b">>, - maybe_escape(16#000b, #config{escaped_strings=true}) + maybe_escape(<<16#000b/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u000e", ?_assertEqual( <<"\\u000e">>, - maybe_escape(16#000e, #config{escaped_strings=true}) + maybe_escape(<<16#000e/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u000f", ?_assertEqual( <<"\\u000f">>, - maybe_escape(16#000f, #config{escaped_strings=true}) + maybe_escape(<<16#000f/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u0010", ?_assertEqual( <<"\\u0010">>, - maybe_escape(16#0010, #config{escaped_strings=true}) + maybe_escape(<<16#0010/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u0011", ?_assertEqual( <<"\\u0011">>, - maybe_escape(16#0011, #config{escaped_strings=true}) + maybe_escape(<<16#0011/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u0012", ?_assertEqual( <<"\\u0012">>, - maybe_escape(16#0012, #config{escaped_strings=true}) + maybe_escape(<<16#0012/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u0013", ?_assertEqual( <<"\\u0013">>, - maybe_escape(16#0013, #config{escaped_strings=true}) + maybe_escape(<<16#0013/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u0014", ?_assertEqual( <<"\\u0014">>, - maybe_escape(16#0014, #config{escaped_strings=true}) + maybe_escape(<<16#0014/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u0015", ?_assertEqual( <<"\\u0015">>, - maybe_escape(16#0015, #config{escaped_strings=true}) + maybe_escape(<<16#0015/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u0016", ?_assertEqual( <<"\\u0016">>, - maybe_escape(16#0016, #config{escaped_strings=true}) + maybe_escape(<<16#0016/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u0017", ?_assertEqual( <<"\\u0017">>, - maybe_escape(16#0017, #config{escaped_strings=true}) + maybe_escape(<<16#0017/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u0018", ?_assertEqual( <<"\\u0018">>, - maybe_escape(16#0018, #config{escaped_strings=true}) + maybe_escape(<<16#0018/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u0019", ?_assertEqual( <<"\\u0019">>, - maybe_escape(16#0019, #config{escaped_strings=true}) + maybe_escape(<<16#0019/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u001a", ?_assertEqual( <<"\\u001a">>, - maybe_escape(16#001a, #config{escaped_strings=true}) + maybe_escape(<<16#001a/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u001b", ?_assertEqual( <<"\\u001b">>, - maybe_escape(16#001b, #config{escaped_strings=true}) + maybe_escape(<<16#001b/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u001c", ?_assertEqual( <<"\\u001c">>, - maybe_escape(16#001c, #config{escaped_strings=true}) + maybe_escape(<<16#001c/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u001d", ?_assertEqual( <<"\\u001d">>, - maybe_escape(16#001d, #config{escaped_strings=true}) + maybe_escape(<<16#001d/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u001e", ?_assertEqual( <<"\\u001e">>, - maybe_escape(16#001e, #config{escaped_strings=true}) + maybe_escape(<<16#001e/utf8>>, #config{escaped_strings=true}) )}, {"maybe_escape u001f", ?_assertEqual( <<"\\u001f">>, - maybe_escape(16#001f, #config{escaped_strings=true}) + maybe_escape(<<16#001f/utf8>>, #config{escaped_strings=true}) )} ]. From 488132aecbae4262609abeecb126a00c7ec7d348 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Wed, 20 Feb 2013 23:57:06 -0800 Subject: [PATCH 46/61] basic strings now FAST in decoder, escapes and bad utf nyi --- src/jsx_decoder.erl | 1039 ++++++++----------------------------------- 1 file changed, 178 insertions(+), 861 deletions(-) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 8b67904..634f596 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -158,9 +158,9 @@ definitely_bom(Bin, Handler, Stack, Config) -> value(<>, Handler, Stack, Config) -> - string(Rest, Handler, [?new_seq()|Stack], Config); + string(Rest, Handler, [string|Stack], Config); value(<>, Handler, Stack, Config = #config{single_quoted_strings=true}) -> - string(Rest, Handler, [?new_seq(), single_quote|Stack], Config); + string(Rest, Handler, [single_quoted_string|Stack], Config); value(<<$t, Rest/binary>>, Handler, Stack, Config) -> tr(Rest, Handler, Stack, Config); value(<<$f, Rest/binary>>, Handler, Stack, Config) -> @@ -188,9 +188,9 @@ value(Bin, Handler, Stack, Config) -> object(<>, Handler, Stack, Config) -> - string(Rest, Handler, [?new_seq()|Stack], Config); + string(Rest, Handler, [string|Stack], Config); object(<>, Handler, Stack, Config = #config{single_quoted_strings=true}) -> - string(Rest, Handler, [?new_seq(), single_quote|Stack], Config); + string(Rest, Handler, [single_quoted_string|Stack], Config); object(<>, Handler, [key|Stack], Config) -> maybe_done(Rest, handle_event(end_object, Handler, Config), Stack, Config); object(<>, Handler, Stack, Config) when ?is_whitespace(S) -> @@ -228,9 +228,9 @@ colon(Bin, Handler, Stack, Config) -> key(<>, Handler, Stack, Config) -> - string(Rest, Handler, [?new_seq()|Stack], Config); + string(Rest, Handler, [string|Stack], Config); key(<>, Handler, Stack, Config = #config{single_quoted_strings=true}) -> - string(Rest, Handler, [?new_seq(), single_quote|Stack], Config); + string(Rest, Handler, [single_quoted_string|Stack], Config); key(<>, Handler, Stack, Config) when ?is_whitespace(S) -> key(Rest, Handler, Stack, Config); key(<>, Handler, Stack, Config=#config{comments=true}) -> @@ -241,466 +241,145 @@ key(Bin, Handler, Stack, Config) -> ?error([Bin, Handler, Stack, Config]). -%% string appends it's output to the term at the top of the stack. for -%% efficiency the strings are build in reverse order and reversed before -%% being added to the output stream -%% when parsing strings, the naive detection of partial codepoints is -%% insufficient. this incredibly anal function should detect all badly formed -%% utf sequences -partial_utf(<<>>) -> true; -partial_utf(<>) when X >= 16#c2, X =< 16#f4 -> true; -partial_utf(<>) when X >= 16#e0, X =< 16#f4, Y >= 16#80, Y =< 16#bf -> true; -partial_utf(<>) - when X >= 16#f0, X =< 16#f4, - Y >= 16#80, Y =< 16#bf, - Z >= 16#80, Z =< 16#bf -> - true; -partial_utf(_) -> false. +string(Bin, Handler, Stack, Config) -> string(Bin, Handler, Stack, Config, <<>>). - -%% explicitly whitelist ascii set for better efficiency (seriously, it's worth -%% almost a 20% increase) -string(<<32, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 32)|Stack], Config); -string(<<33, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 33)|Stack], Config); -string(<>, Handler, S, Config) -> - case S of - [Acc, key|Stack] -> - colon(Rest, handle_event({key, ?end_seq(Acc)}, Handler, Config), [key|Stack], Config); - [_Acc, single_quote|_Stack] -> - ?error([<>, Handler, S, Config]); - [Acc|Stack] -> - maybe_done(Rest, handle_event({string, ?end_seq(Acc)}, Handler, Config), Stack, Config) - end; -string(<<35, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 35)|Stack], Config); -string(<<36, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 36)|Stack], Config); -string(<<37, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 37)|Stack], Config); -string(<<38, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 38)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) -> - case Config#config.single_quoted_strings of - true -> - case Stack of - [single_quote, key|S] -> - colon(Rest, handle_event({key, ?end_seq(Acc)}, Handler, Config), [key|S], Config) - ; [single_quote|S] -> - maybe_done(Rest, handle_event({string, ?end_seq(Acc)}, Handler, Config), S, Config) - ; _ -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace(?singlequote, Config))|Stack], Config) - end - ; false -> - string(Rest, Handler, [?acc_seq(Acc, ?singlequote)|Stack], Config) - end; -string(<<40, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 40)|Stack], Config); -string(<<41, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 41)|Stack], Config); -string(<<42, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 42)|Stack], Config); -string(<<43, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 43)|Stack], Config); -string(<<44, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 44)|Stack], Config); -string(<<45, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 45)|Stack], Config); -string(<<46, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 46)|Stack], Config); -string(<<$/, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace($/, Config))|Stack], Config); -string(<<48, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 48)|Stack], Config); -string(<<49, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 49)|Stack], Config); -string(<<50, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 50)|Stack], Config); -string(<<51, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 51)|Stack], Config); -string(<<52, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 52)|Stack], Config); -string(<<53, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 53)|Stack], Config); -string(<<54, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 54)|Stack], Config); -string(<<55, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 55)|Stack], Config); -string(<<56, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 56)|Stack], Config); -string(<<57, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 57)|Stack], Config); -string(<<58, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 58)|Stack], Config); -string(<<59, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 59)|Stack], Config); -string(<<60, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 60)|Stack], Config); -string(<<61, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 61)|Stack], Config); -string(<<62, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 62)|Stack], Config); -string(<<63, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 63)|Stack], Config); -string(<<64, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 64)|Stack], Config); -string(<<65, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 65)|Stack], Config); -string(<<66, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 66)|Stack], Config); -string(<<67, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 67)|Stack], Config); -string(<<68, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 68)|Stack], Config); -string(<<69, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 69)|Stack], Config); -string(<<70, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 70)|Stack], Config); -string(<<71, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 71)|Stack], Config); -string(<<72, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 72)|Stack], Config); -string(<<73, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 73)|Stack], Config); -string(<<74, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 74)|Stack], Config); -string(<<75, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 75)|Stack], Config); -string(<<76, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 76)|Stack], Config); -string(<<77, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 77)|Stack], Config); -string(<<78, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 78)|Stack], Config); -string(<<79, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 79)|Stack], Config); -string(<<80, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 80)|Stack], Config); -string(<<81, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 81)|Stack], Config); -string(<<82, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 82)|Stack], Config); -string(<<83, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 83)|Stack], Config); -string(<<84, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 84)|Stack], Config); -string(<<85, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 85)|Stack], Config); -string(<<86, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 86)|Stack], Config); -string(<<87, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 87)|Stack], Config); -string(<<88, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 88)|Stack], Config); -string(<<89, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 89)|Stack], Config); -string(<<90, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 90)|Stack], Config); -string(<<91, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 91)|Stack], Config); -string(<>, Handler, Stack, Config) -> - escape(Rest, Handler, Stack, Config); -string(<<93, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 93)|Stack], Config); -string(<<94, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 94)|Stack], Config); -string(<<95, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 95)|Stack], Config); -string(<<96, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 96)|Stack], Config); -string(<<97, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 97)|Stack], Config); -string(<<98, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 98)|Stack], Config); -string(<<99, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 99)|Stack], Config); -string(<<100, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 100)|Stack], Config); -string(<<101, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 101)|Stack], Config); -string(<<102, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 102)|Stack], Config); -string(<<103, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 103)|Stack], Config); -string(<<104, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 104)|Stack], Config); -string(<<105, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 105)|Stack], Config); -string(<<106, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 106)|Stack], Config); -string(<<107, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 107)|Stack], Config); -string(<<108, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 108)|Stack], Config); -string(<<109, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 109)|Stack], Config); -string(<<110, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 110)|Stack], Config); -string(<<111, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 111)|Stack], Config); -string(<<112, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 112)|Stack], Config); -string(<<113, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 113)|Stack], Config); -string(<<114, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 114)|Stack], Config); -string(<<115, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 115)|Stack], Config); -string(<<116, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 116)|Stack], Config); -string(<<117, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 117)|Stack], Config); -string(<<118, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 118)|Stack], Config); -string(<<119, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 119)|Stack], Config); -string(<<120, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 120)|Stack], Config); -string(<<121, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 121)|Stack], Config); -string(<<122, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 122)|Stack], Config); -string(<<123, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 123)|Stack], Config); -string(<<124, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 124)|Stack], Config); -string(<<125, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 125)|Stack], Config); -string(<<126, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 126)|Stack], Config); -string(<<127, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 127)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X >= 16#20, X < 16#2028 -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X == 16#2028; X == 16#2029 -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace(X, Config))|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X > 16#2029, X < 16#d800 -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X > 16#dfff, X < 16#fdd0 -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X > 16#fdef, X < 16#fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X >= 16#10000, X < 16#1fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X >= 16#20000, X < 16#2fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X >= 16#30000, X < 16#3fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X >= 16#40000, X < 16#4fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X >= 16#50000, X < 16#5fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X >= 16#60000, X < 16#6fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X >= 16#70000, X < 16#7fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X >= 16#80000, X < 16#8fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X >= 16#90000, X < 16#9fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X >= 16#a0000, X < 16#afffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X >= 16#b0000, X < 16#bfffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X >= 16#c0000, X < 16#cfffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X >= 16#d0000, X < 16#dfffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X >= 16#e0000, X < 16#efffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X >= 16#f0000, X < 16#ffffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) when X >= 16#100000, X < 16#10fffe -> - string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); -string(<>, Handler, [Acc|Stack], Config) -> - case Config#config.replaced_bad_utf8 of - true -> noncharacter(<>, Handler, [Acc|Stack], Config) - ; false -> ?error([<>, Handler, [Acc|Stack], Config]) - end; -string(Bin, Handler, Stack, Config) -> - case partial_utf(Bin) of - true -> ?incomplete(string, Bin, Handler, Stack, Config) - ; false -> - case Config#config.replaced_bad_utf8 of - true -> noncharacter(Bin, Handler, Stack, Config) - ; false -> ?error([Bin, Handler, Stack, Config]) - end +string(Bin, Handler, [StringType|Stack], Config, Acc) -> + Length = cut(Bin), + <> = Bin, + case Rest of + <> when StringType == string -> + finish_string(Rem, Handler, Stack, Config, <>); + <> when StringType == single_quoted_string -> + finish_string(Rem, Handler, Stack, Config, <>); + <> when Codepoint == ?doublequote; Codepoint == ?singlequote -> + string(Rem, Handler, [StringType|Stack], Config, <>) end. -%% we don't need to guard against partial utf here, because it's already taken -%% care of in string -%% surrogates -noncharacter(<<237, X, _, Rest/binary>>, Handler, [Acc|Stack], Config) when X >= 160 -> - string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config); -%% u+fffe and u+ffff for R14BXX -noncharacter(<<239, 191, X, Rest/binary>>, Handler, [Acc|Stack], Config) when X == 190; X == 191 -> - string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config); -%% u+xfffe, u+xffff and other noncharacters -noncharacter(<<_/utf8, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config); -%% overlong encodings and missing continuations of a 2 byte sequence -noncharacter(<>, Handler, Stack, Config) when X >= 192, X =< 223 -> - strip_continuations(Rest, Handler, [1|Stack], Config); -%% overlong encodings and missing continuations of a 3 byte sequence -noncharacter(<>, Handler, Stack, Config) when X >= 224, X =< 239 -> - strip_continuations(Rest, Handler, [2|Stack], Config); -%% overlong encodings and missing continuations of a 4 byte sequence -noncharacter(<>, Handler, Stack, Config) when X >= 240, X =< 247 -> - strip_continuations(Rest, Handler, [3|Stack], Config); -%% unexpected bytes, including orphan continuations -noncharacter(<<_, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config); -noncharacter(<<>>, Handler, Stack, Config) -> - ?incomplete(noncharacter, <<>>, Handler, Stack, Config). +finish_string(Rest, Handler, [key|_] = Stack, Config, Acc) -> + State = handle_event({key, Acc}, Handler, Config), + colon(Rest, State, Stack, Config); +finish_string(Rest, Handler, Stack, Config, Acc) -> + State = handle_event({string, Acc}, Handler, Config), + maybe_done(Rest, State, Stack, Config). -%% strips continuation bytes after bad utf bytes, guards against both too short -%% and overlong sequences. N is the maximum number of bytes to strip -strip_continuations(Rest, Handler, [0, Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config); -strip_continuations(<>, Handler, [N|Stack], Config) when X >= 128, X =< 191 -> - strip_continuations(Rest, Handler, [N - 1|Stack], Config); -%% incomplete -strip_continuations(<<>>, Handler, Stack, Config) -> - ?incomplete(strip_continuations, <<>>, Handler, Stack, Config); -%% not a continuation byte, dispatch back to string -strip_continuations(Rest, Handler, [_, Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config). +cut(Bin) -> cut(Bin, 0). - -escape(<<$b, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\b, Config))|Stack], Config); -escape(<<$f, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\f, Config))|Stack], Config); -escape(<<$n, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\n, Config))|Stack], Config); -escape(<<$r, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\r, Config))|Stack], Config); -escape(<<$t, Rest/binary>>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\t, Config))|Stack], Config); -escape(<>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\\, Config))|Stack], Config); -escape(<>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace($/, Config))|Stack], Config); -escape(<>, Handler, [Acc|Stack], Config) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\", Config))|Stack], Config); -escape(<>, Handler, [Acc|Stack], Config = #config{single_quoted_strings=true}) -> - string(Rest, Handler, [?acc_seq(Acc, maybe_replace(?singlequote, Config))|Stack], Config); -escape(<<$u, Rest/binary>>, Handler, Stack, Config) -> - escaped_unicode(Rest, Handler, Stack, Config); -escape(<<>>, Handler, Stack, Config) -> - ?incomplete(escape, <<>>, Handler, Stack, Config); -escape(Bin, Handler, [Acc|Stack], Config=#config{ignored_bad_escapes=true}) -> - string(Bin, Handler, [?acc_seq(Acc, ?rsolidus)|Stack], Config); -escape(Bin, Handler, Stack, Config) -> - ?error([Bin, Handler, Stack, Config]). - - -%% this code is ugly and unfortunate, but so is json's handling of escaped -%% unicode codepoint sequences. -escaped_unicode(<>, Handler, [Acc|Stack], Config) - when ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D) -> - case erlang:list_to_integer([A, B, C, D], 16) of - %% high surrogate, dispatch to low surrogate - X when X >= 16#d800, X =< 16#dbff -> - low_surrogate(Rest, Handler, [X, Acc|Stack], Config) - %% low surrogate, illegal in this position - ; X when X >= 16#dc00, X =< 16#dfff -> - case Config#config.replaced_bad_utf8 of - true -> string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config) - ; false -> ?error([<>, Handler, [Acc|Stack], Config]) - end - %% anything else - ; X -> string(Rest, Handler, [?acc_seq(Acc, maybe_replace(X, Config))|Stack], Config) - end; -escaped_unicode(Bin, Handler, Stack, Config) -> - case is_partial_escape(Bin) of - true -> ?incomplete(escaped_unicode, Bin, Handler, Stack, Config) - ; false -> ?error([Bin, Handler, Stack, Config]) - end. - - -is_partial_escape(<>) when ?is_hex(A), ?is_hex(B), ?is_hex(C) -> true; -is_partial_escape(<>) when ?is_hex(A), ?is_hex(B) -> true; -is_partial_escape(<>) when ?is_hex(A) -> true; -is_partial_escape(<<>>) -> true; -is_partial_escape(_) -> false. - - -low_surrogate(<>, Handler, [High, Acc|Stack], Config) - when ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D) -> - case erlang:list_to_integer([A, B, C, D], 16) of - X when X >= 16#dc00, X =< 16#dfff -> - Y = surrogate_to_codepoint(High, X), - case (Y =< 16#d800 orelse Y >= 16#e000) of - true -> string(Rest, Handler, [?acc_seq(Acc, Y)|Stack], Config) - ; false -> - case Config#config.replaced_bad_utf8 of - true -> - string(Rest, Handler, [?acc_seq(Acc, 16#fffd, 16#fffd)|Stack], Config) - ; false -> - ?error([<>, Handler, [High, Acc|Stack], Config]) - end - end - ; _ -> - case Config#config.replaced_bad_utf8 of - true -> string(Rest, Handler, [?acc_seq(Acc, 16#fffd, 16#fffd)|Stack], Config) - ; false -> ?error([<>, Handler, [High, Acc|Stack], Config]) - end - end; -low_surrogate(Bin, Handler, [High, Acc|Stack], Config) -> - case is_partial_low(Bin) of - true -> ?incomplete(low_surrogate, Bin, Handler, [High, Acc|Stack], Config) - ; false -> - case Config#config.replaced_bad_utf8 of - true -> string(Bin, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config) - ; false -> ?error([Bin, Handler, [High, Acc|Stack], Config]) - end - end. - - -is_partial_low(<>) when ?is_hex(A), ?is_hex(B), ?is_hex(C) -> true; -is_partial_low(<>) when ?is_hex(A), ?is_hex(B) -> true; -is_partial_low(<>) when ?is_hex(A) -> true; -is_partial_low(<>) -> true; -is_partial_low(<>) -> true; -is_partial_low(<<>>) -> true; -is_partial_low(_) -> false. - - -%% stole this from the unicode spec -surrogate_to_codepoint(High, Low) -> - (High - 16#d800) * 16#400 + (Low - 16#dc00) + 16#10000. - - -maybe_replace(X, #config{dirty_strings=true}) when is_integer(X) -> [X]; -maybe_replace($\b, #config{escaped_strings=true}) -> [$\\, $b]; -maybe_replace($\t, #config{escaped_strings=true}) -> [$\\, $t]; -maybe_replace($\n, #config{escaped_strings=true}) -> [$\\, $n]; -maybe_replace($\f, #config{escaped_strings=true}) -> [$\\, $f]; -maybe_replace($\r, #config{escaped_strings=true}) -> [$\\, $r]; -maybe_replace($\", #config{escaped_strings=true}) -> [$\\, $\"]; -maybe_replace($', Config=#config{escaped_strings=true}) -> - case Config#config.single_quoted_strings of - true -> [$\\, $'] - ; false -> [$'] - end; -maybe_replace($/, Config=#config{escaped_strings=true}) -> - case Config#config.escaped_forward_slashes of - true -> [$\\, $/] - ; false -> [$/] - end; -maybe_replace($\\, #config{escaped_strings=true}) -> [$\\, $\\]; -maybe_replace(X, Config=#config{escaped_strings=true}) when X == 16#2028; X == 16#2029 -> - case Config#config.unescaped_jsonp of - true -> [X] - ; false -> jsx_utils:json_escape_sequence(X) - end; -maybe_replace(X, #config{escaped_strings=true}) when X < 32 -> - jsx_utils:json_escape_sequence(X); -maybe_replace(X, _Config) -> [X]. +cut(<<32, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<33, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<35, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<36, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<37, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<38, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<40, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<41, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<42, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<43, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<44, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<45, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<46, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<48, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<49, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<50, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<51, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<52, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<53, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<54, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<55, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<56, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<57, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<58, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<59, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<60, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<61, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<62, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<63, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<64, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<65, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<66, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<67, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<68, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<69, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<70, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<71, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<72, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<73, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<74, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<75, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<76, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<77, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<78, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<79, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<80, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<81, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<82, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<83, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<84, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<85, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<86, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<87, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<88, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<89, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<90, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<91, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<93, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<94, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<95, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<96, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<97, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<98, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<99, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<100, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<101, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<102, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<103, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<104, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<105, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<106, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<107, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<108, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<109, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<110, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<111, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<112, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<113, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<114, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<115, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<116, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<117, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<118, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<119, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<120, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<121, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<122, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<123, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<124, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<125, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<126, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<<127, Rest/binary>>, N) -> cut(Rest, N + 1); +cut(<>, N) when X >= 128, X < 16#0800 -> cut(Rest, N + 2); +cut(<>, N) when X >= 16#0800, X < 16#2028 -> cut(Rest, N + 3); +cut(<>, N) when X >= 16#202a, X < 16#d800 -> cut(Rest, N + 3); +cut(<>, N) when X > 16#dfff, X < 16#fdd0 -> cut(Rest, N + 3); +cut(<>, N) when X > 16#fdef, X < 16#fffe -> cut(Rest, N + 3); +cut(<>, N) when X >= 16#10000, X < 16#1fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#20000, X < 16#2fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#30000, X < 16#3fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#40000, X < 16#4fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#50000, X < 16#5fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#60000, X < 16#6fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#70000, X < 16#7fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#80000, X < 16#8fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#90000, X < 16#9fffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#a0000, X < 16#afffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#b0000, X < 16#bfffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#c0000, X < 16#cfffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#d0000, X < 16#dfffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#e0000, X < 16#efffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#f0000, X < 16#ffffe -> cut(Rest, N + 4); +cut(<>, N) when X >= 16#100000, X < 16#10fffe -> cut(Rest, N + 4); +cut(_, N) -> N. @@ -1024,6 +703,18 @@ done(Bin, Handler, Stack, Config) -> ?error([Bin, Handler, Stack, Config]). -include_lib("eunit/include/eunit.hrl"). +decode_test_() -> + Data = jsx:test_cases(), + [ + { + Title, ?_assertEqual( + Events ++ [end_json], + start(JSON, {jsx, []}, [], #config{}) + ) + } || {Title, JSON, _, Events} <- Data + ]. + + %% all these numbers have different representation in erlang than in javascript and %% do not roundtrip like most integers/floats special_number_test_() -> @@ -1059,258 +750,6 @@ special_number_test_() -> ]. -unescape_test_() -> - [ - {"unescape backspace", ?_assertEqual( - [{string, <<"\b">>}, end_json], - start(<<"\"\\b\"">>, {jsx, []}, [], #config{}) - )}, - {"unescape tab", ?_assertEqual( - [{string, <<"\t">>}, end_json], - start(<<"\"\\t\"">>, {jsx, []}, [], #config{}) - )}, - {"unescape newline", ?_assertEqual( - [{string, <<"\n">>}, end_json], - start(<<"\"\\n\"">>, {jsx, []}, [], #config{}) - )}, - {"unescape formfeed", ?_assertEqual( - [{string, <<"\f">>}, end_json], - start(<<"\"\\f\"">>, {jsx, []}, [], #config{}) - )}, - {"unescape carriage return", ?_assertEqual( - [{string, <<"\r">>}, end_json], - start(<<"\"\\r\"">>, {jsx, []}, [], #config{}) - )}, - {"unescape quote", ?_assertEqual( - [{string, <<"\"">>}, end_json], - start(<<"\"\\\"\"">>, {jsx, []}, [], #config{}) - )}, - {"unescape forward slash", ?_assertEqual( - [{string, <<"/">>}, end_json], - start(<<"\"\\/\"">>, {jsx, []}, [], #config{}) - )}, - {"unescape back slash", ?_assertEqual( - [{string, <<"\\">>}, end_json], - start(<<"\"\\\\\"">>, {jsx, []}, [], #config{}) - )}, - {"unescape control code", ?_assertEqual( - [{string, <<0>>}, end_json], - start(<<"\"\\u0000\"">>, {jsx, []}, [], #config{}) - )} - ]. - - -ignored_bad_escapes_test_() -> - [ - {"ignore unrecognized escape sequence", ?_assertEqual( - [{string, <<"\\x25">>}, end_json], - start(<<"\"\\x25\"">>, {jsx, []}, [], #config{ignored_bad_escapes=true}) - )} - ]. - - - -xcode(Bin) -> xcode(Bin, []). - -xcode(Bin, Config) -> - Size = size(Bin), - try jsx:to_term(<<34, Bin:Size/binary, 34>>, Config) - catch error:badarg -> {error, badarg} - end. - - -is_bad({error, badarg}) -> true; -is_bad(_) -> false. - - -bad_utf8_test_() -> - [ - {"orphan continuation byte u+0080", - ?_assert(is_bad(xcode(<<16#0080>>))) - }, - {"orphan continuation byte u+0080 replaced", - ?_assertEqual(xcode(<<16#0080>>, [replaced_bad_utf8]), <<16#fffd/utf8>>) - }, - {"orphan continuation byte u+00bf", - ?_assert(is_bad(xcode(<<16#00bf>>))) - }, - {"orphan continuation byte u+00bf replaced", - ?_assertEqual(xcode(<<16#00bf>>, [replaced_bad_utf8]), <<16#fffd/utf8>>) - }, - {"2 continuation bytes", - ?_assert(is_bad(xcode(<<(binary:copy(<<16#0080>>, 2))/binary>>))) - }, - {"2 continuation bytes replaced", - ?_assertEqual( - xcode(<<(binary:copy(<<16#0080>>, 2))/binary>>, [replaced_bad_utf8]), - binary:copy(<<16#fffd/utf8>>, 2) - ) - }, - {"3 continuation bytes", - ?_assert(is_bad(xcode(<<(binary:copy(<<16#0080>>, 3))/binary>>))) - }, - {"3 continuation bytes replaced", - ?_assertEqual( - xcode(<<(binary:copy(<<16#0080>>, 3))/binary>>, [replaced_bad_utf8]), - binary:copy(<<16#fffd/utf8>>, 3) - ) - }, - {"4 continuation bytes", - ?_assert(is_bad(xcode(<<(binary:copy(<<16#0080>>, 4))/binary>>))) - }, - {"4 continuation bytes replaced", - ?_assertEqual( - xcode(<<(binary:copy(<<16#0080>>, 4))/binary>>, [replaced_bad_utf8]), - binary:copy(<<16#fffd/utf8>>, 4) - ) - }, - {"5 continuation bytes", - ?_assert(is_bad(xcode(<<(binary:copy(<<16#0080>>, 5))/binary>>))) - }, - {"5 continuation bytes replaced", - ?_assertEqual( - xcode(<<(binary:copy(<<16#0080>>, 5))/binary>>, [replaced_bad_utf8]), - binary:copy(<<16#fffd/utf8>>, 5) - ) - }, - {"6 continuation bytes", - ?_assert(is_bad(xcode(<<(binary:copy(<<16#0080>>, 6))/binary>>))) - }, - {"6 continuation bytes replaced", - ?_assertEqual( - xcode(<<(binary:copy(<<16#0080>>, 6))/binary>>, [replaced_bad_utf8]), - binary:copy(<<16#fffd/utf8>>, 6) - ) - }, - {"all continuation bytes", - ?_assert(is_bad(xcode(<<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>))) - }, - {"all continuation bytes replaced", - ?_assertEqual( - xcode(<<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, [replaced_bad_utf8]), - binary:copy(<<16#fffd/utf8>>, length(lists:seq(16#0080, 16#00bf))) - ) - }, - {"lonely start byte", - ?_assert(is_bad(xcode(<<16#00c0>>))) - }, - {"lonely start byte replaced", - ?_assertEqual( - xcode(<<16#00c0>>, [replaced_bad_utf8]), - <<16#fffd/utf8>> - ) - }, - {"lonely start bytes (2 byte)", - ?_assert(is_bad(xcode(<<16#00c0, 32, 16#00df>>))) - }, - {"lonely start bytes (2 byte) replaced", - ?_assertEqual( - xcode(<<16#00c0, 32, 16#00df>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32, 16#fffd/utf8>> - ) - }, - {"lonely start bytes (3 byte)", - ?_assert(is_bad(xcode(<<16#00e0, 32, 16#00ef>>))) - }, - {"lonely start bytes (3 byte) replaced", - ?_assertEqual( - xcode(<<16#00e0, 32, 16#00ef>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32, 16#fffd/utf8>> - ) - }, - {"lonely start bytes (4 byte)", - ?_assert(is_bad(xcode(<<16#00f0, 32, 16#00f7>>))) - }, - {"lonely start bytes (4 byte) replaced", - ?_assertEqual( - xcode(<<16#00f0, 32, 16#00f7>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32, 16#fffd/utf8>> - ) - }, - {"missing continuation byte (3 byte)", - ?_assert(is_bad(xcode(<<224, 160, 32>>))) - }, - {"missing continuation byte (3 byte) replaced", - ?_assertEqual( - xcode(<<224, 160, 32>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32>> - ) - }, - {"missing continuation byte (4 byte missing one)", - ?_assert(is_bad(xcode(<<240, 144, 128, 32>>))) - }, - {"missing continuation byte2 (4 byte missing one) replaced", - ?_assertEqual( - xcode(<<240, 144, 128, 32>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32>> - ) - }, - {"missing continuation byte (4 byte missing two)", - ?_assert(is_bad(xcode(<<240, 144, 32>>))) - }, - {"missing continuation byte2 (4 byte missing two) replaced", - ?_assertEqual( - xcode(<<240, 144, 32>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32>> - ) - }, - {"overlong encoding of u+002f (2 byte)", - ?_assert(is_bad(xcode(<<16#c0, 16#af, 32>>))) - }, - {"overlong encoding of u+002f (2 byte) replaced", - ?_assertEqual( - xcode(<<16#c0, 16#af, 32>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32>> - ) - }, - {"overlong encoding of u+002f (3 byte)", - ?_assert(is_bad(xcode(<<16#e0, 16#80, 16#af, 32>>))) - }, - {"overlong encoding of u+002f (3 byte) replaced", - ?_assertEqual( - xcode(<<16#e0, 16#80, 16#af, 32>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32>> - ) - }, - {"overlong encoding of u+002f (4 byte)", - ?_assert(is_bad(xcode(<<16#f0, 16#80, 16#80, 16#af, 32>>))) - }, - {"overlong encoding of u+002f (4 byte) replaced", - ?_assertEqual( - xcode(<<16#f0, 16#80, 16#80, 16#af, 32>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32>> - ) - }, - {"highest overlong 2 byte sequence", - ?_assert(is_bad(xcode(<<16#c1, 16#bf, 32>>))) - }, - {"highest overlong 2 byte sequence replaced", - ?_assertEqual( - xcode(<<16#c1, 16#bf, 32>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32>> - ) - }, - {"highest overlong 3 byte sequence", - ?_assert(is_bad(xcode(<<16#e0, 16#9f, 16#bf, 32>>))) - }, - {"highest overlong 3 byte sequence replaced", - ?_assertEqual( - xcode(<<16#e0, 16#9f, 16#bf, 32>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32>> - ) - }, - {"highest overlong 4 byte sequence", - ?_assert(is_bad(xcode(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>))) - }, - {"highest overlong 4 byte sequence replaced", - ?_assertEqual( - xcode(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, [replaced_bad_utf8]), - <<16#fffd/utf8, 32>> - ) - } - ]. - - decode(JSON, Config) -> try (decoder(jsx, [], Config))(JSON) @@ -1520,136 +959,6 @@ comments_test_() -> ]. - - - -escapes_test_() -> - [ - {"backspace escape", ?_assertEqual(decode(<<"\"\\b\"">>, [escaped_strings]), [{string, <<"\\b">>}, end_json])}, - {"formfeed escape", ?_assertEqual(decode(<<"\"\\f\"">>, [escaped_strings]), [{string, <<"\\f">>}, end_json])}, - {"newline escape", ?_assertEqual(decode(<<"\"\\n\"">>, [escaped_strings]), [{string, <<"\\n">>}, end_json])}, - {"carriage return escape", ?_assertEqual(decode(<<"\"\\r\"">>, [escaped_strings]), [{string, <<"\\r">>}, end_json])}, - {"tab escape", ?_assertEqual(decode(<<"\"\\t\"">>, [escaped_strings]), [{string, <<"\\t">>}, end_json])}, - {"quote escape", ?_assertEqual(decode(<<"\"\\\"\"">>, [escaped_strings]), [{string, <<"\\\"">>}, end_json])}, - {"single quote escape", ?_assertEqual(decode(<<"\"'\"">>, [escaped_strings, single_quoted_strings]), [{string, <<"\\'">>}, end_json])}, - {"naked single quote escape", ?_assertEqual(decode(<<"'\\''">>, [escaped_strings, single_quoted_strings]), [{string, <<"\\'">>}, end_json])}, - {"no single quote escape", ?_assertEqual(decode(<<"\"'\"">>, [escaped_strings]), [{string, <<"'">>}, end_json])}, - {"forward slash escape", ?_assertEqual(decode(<<"\"/\"">>, [escaped_strings, escaped_forward_slashes]), [{string, <<"\\/">>}, end_json])}, - {"no forward slash escape", ?_assertEqual(decode(<<"\"/\"">>, [escaped_strings]), [{string, <<"/">>}, end_json])}, - {"back slash escape", ?_assertEqual(decode(<<"\"\\\\\"">>, [escaped_strings]), [{string, <<"\\\\">>}, end_json])}, - {"jsonp escape", ?_assertEqual( - decode(<<$\", 16#2028/utf8, 16#2029/utf8, $\">>, [escaped_strings]), - [{string, <<"\\u2028\\u2029">>}, end_json] - )}, - {"no jsonp escape", ?_assertEqual( - decode(<<$\", 16#2028/utf8, 16#2029/utf8, $\">>, [escaped_strings, unescaped_jsonp]), - [{string, <<16#2028/utf8, 16#2029/utf8>>}, end_json] - )}, - {"control escape", ?_assertEqual(decode(<<$\", "\\u0000"/utf8, $\">>, [escaped_strings]), [{string, <<"\\u0000">>}, end_json])}, - {"dirty strings", ?_assertEqual(decode(<<"\"\\n\"">>, [escaped_strings, dirty_strings]), [{string, <<"\n">>}, end_json])}, - {"ignore bad escapes", ?_assertEqual(decode(<<"\"\\x25\"">>, [escaped_strings, ignored_bad_escapes]), [{string, <<"\\x25">>}, end_json])} - ]. - - -noncharacters_test_() -> - [ - {"noncharacters - badarg", - ?_assert(check_bad(noncharacters())) - }, - {"noncharacters - replaced", - ?_assert(check_replaced(noncharacters())) - } - ]. - - -extended_noncharacters_test_() -> - [ - {"extended noncharacters - badarg", - ?_assert(check_bad(extended_noncharacters())) - }, - {"extended noncharacters - replaced", - ?_assert(check_replaced(extended_noncharacters())) - } - ]. - - -surrogates_test_() -> - [ - {"surrogates - badarg", - ?_assert(check_bad(surrogates())) - }, - {"surrogates - replaced", - ?_assert(check_replaced(surrogates())) - } - ]. - - -control_test_() -> - [ - {"control characters - badarg", - ?_assert(check_bad(control_characters())) - } - ]. - - -reserved_test_() -> - [ - {"reserved noncharacters - badarg", - ?_assert(check_bad(reserved_space())) - }, - {"reserved noncharacters - replaced", - ?_assert(check_replaced(reserved_space())) - } - ]. - - -good_characters_test_() -> - [ - {"acceptable codepoints", - ?_assert(check_good(good())) - }, - {"acceptable codepoints - escaped_strings", - ?_assert(check_good(good(), [escaped_strings])) - }, - {"acceptable codepoints - replaced_bad_utf8", - ?_assert(check_good(good(), [replaced_bad_utf8])) - }, - {"acceptable codepoints - escaped_strings + replaced_bad_utf8", - ?_assert(check_good(good(), [escaped_strings, replaced_bad_utf8])) - }, - {"acceptable extended", - ?_assert(check_good(good_extended())) - } - ]. - - -check_bad(List) -> - [] == lists:dropwhile(fun({_, {error, badarg}}) -> true ; (_) -> false end, - check(List, [], []) - ). - - -check_replaced(List) -> - [] == lists:dropwhile(fun({_, [{string, <<16#fffd/utf8>>}|_]}) -> true ; (_) -> false - end, - check(List, [replaced_bad_utf8], []) - ). - - -check_good(List) -> check_good(List, []). - -check_good(List, Config) -> - [] == lists:dropwhile(fun({_, [{string, _}|_]}) -> true ; (_) -> false end, - check(List, Config, []) - ). - - -check([], _Config, Acc) -> Acc; -check([H|T], Config, Acc) -> - R = decode(to_fake_utf(H, utf8), Config), - check(T, Config, [{H, R}] ++ Acc). - - noncharacters() -> lists:seq(16#fffe, 16#ffff). extended_noncharacters() -> @@ -1670,20 +979,26 @@ control_characters() -> lists:seq(1, 31). reserved_space() -> lists:seq(16#fdd0, 16#fdef). -good() -> - [32, 33] - ++ lists:seq(16#23, 16#5b) - ++ lists:seq(16#5d, 16#d7ff) +codepoints() -> + unicode:characters_to_binary( + [32, 33] + ++ lists:seq(35, 38) + ++ lists:seq(40, 46) + ++ lists:seq(48, 91) + ++ lists:seq(93, 16#2027) + ++ lists:seq(16#202a, 16#d7ff) ++ lists:seq(16#e000, 16#fdcf) - ++ lists:seq(16#fdf0, 16#fffd). + ++ lists:seq(16#fdf0, 16#fffd) + ). -good_extended() -> - lists:seq(16#10000, 16#1fffd) ++ - [ - 16#10000, 16#20000, 16#30000, 16#40000, 16#50000, - 16#60000, 16#70000, 16#80000, 16#90000, 16#a0000, - 16#b0000, 16#c0000, 16#d0000, 16#e0000, 16#f0000 - ] ++ lists:seq(16#100000, 16#10fffd). +extended_codepoints() -> + unicode:characters_to_binary( + lists:seq(16#10000, 16#1fffd) ++ [ + 16#20000, 16#30000, 16#40000, 16#50000, 16#60000, + 16#70000, 16#80000, 16#90000, 16#a0000, 16#b0000, + 16#c0000, 16#d0000, 16#e0000, 16#f0000, 16#100000 + ] + ). %% erlang refuses to decode certain codepoints, so fake them all @@ -1699,15 +1014,17 @@ to_fake_utf(N, utf8) -> <<34/utf8, 2#11110:5, W:3, 2#10:2, Z:6, 2#10:2, Y:6, 2#10:2, X:6, 34/utf8>>. -decode_test_() -> - Data = jsx:test_cases(), +clean_string_test_() -> [ - { - Title, ?_assertEqual( - Events ++ [end_json], - start(JSON, {jsx, []}, [], #config{}) - ) - } || {Title, JSON, _, Events} <- Data + {"clean codepoints test", ?_assertEqual( + [{string, codepoints()}, end_json], + decode(<<34, (codepoints())/binary, 34>>, []) + )}, + {"clean extended codepoints test", ?_assertEqual( + [{string, extended_codepoints()}, end_json], + decode(<<34, (extended_codepoints())/binary, 34>>, []) + )} ]. + -endif. \ No newline at end of file From 6b6f713f96e0cc91729aced1b7a45164b3f792d8 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Thu, 21 Feb 2013 00:05:12 -0800 Subject: [PATCH 47/61] add some tests for single quoted strings --- src/jsx_decoder.erl | 61 +++++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 634f596..4be6de0 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -959,26 +959,6 @@ comments_test_() -> ]. -noncharacters() -> lists:seq(16#fffe, 16#ffff). - -extended_noncharacters() -> - [ - 16#1fffe, 16#1ffff, 16#2fffe, 16#2ffff, - 16#3fffe, 16#3ffff, 16#4fffe, 16#4ffff, - 16#5fffe, 16#5ffff, 16#6fffe, 16#6ffff, - 16#7fffe, 16#7ffff, 16#8fffe, 16#8ffff, - 16#9fffe, 16#9ffff, 16#afffe, 16#affff, - 16#bfffe, 16#bffff, 16#cfffe, 16#cffff, - 16#dfffe, 16#dffff, 16#efffe, 16#effff, - 16#ffffe, 16#fffff, 16#10fffe, 16#10ffff - ]. - -surrogates() -> lists:seq(16#d800, 16#dfff). - -control_characters() -> lists:seq(1, 31). - -reserved_space() -> lists:seq(16#fdd0, 16#fdef). - codepoints() -> unicode:characters_to_binary( [32, 33] @@ -1000,16 +980,32 @@ extended_codepoints() -> ] ). +reserved_space() -> [ to_fake_utf8(N) || N <- lists:seq(16#fdd0, 16#fdef) ]. + +surrogates() -> [ to_fake_utf8(N) || N <- lists:seq(16#d800, 16#dfff) ]. + +noncharacters() -> [ to_fake_utf8(N) || N <- lists:seq(16#fffe, 16#ffff) ]. + +extended_noncharacters() -> + [ to_fake_utf8(N) || N <- [16#1fffe, 16#1ffff, 16#2fffe, 16#2ffff] + ++ [16#3fffe, 16#3ffff, 16#4fffe, 16#4ffff] + ++ [16#5fffe, 16#5ffff, 16#6fffe, 16#6ffff] + ++ [16#7fffe, 16#7ffff, 16#8fffe, 16#8ffff] + ++ [16#9fffe, 16#9ffff, 16#afffe, 16#affff] + ++ [16#bfffe, 16#bffff, 16#cfffe, 16#cffff] + ++ [16#dfffe, 16#dffff, 16#efffe, 16#effff] + ++ [16#ffffe, 16#fffff, 16#10fffe, 16#10ffff] + ]. %% erlang refuses to decode certain codepoints, so fake them all -to_fake_utf(N, utf8) when N < 16#0080 -> <<34/utf8, N:8, 34/utf8>>; -to_fake_utf(N, utf8) when N < 16#0800 -> +to_fake_utf8(N) when N < 16#0080 -> <<34/utf8, N:8, 34/utf8>>; +to_fake_utf8(N) when N < 16#0800 -> <<0:5, Y:5, X:6>> = <>, <<34/utf8, 2#110:3, Y:5, 2#10:2, X:6, 34/utf8>>; -to_fake_utf(N, utf8) when N < 16#10000 -> +to_fake_utf8(N) when N < 16#10000 -> <> = <>, <<34/utf8, 2#1110:4, Z:4, 2#10:2, Y:6, 2#10:2, X:6, 34/utf8>>; -to_fake_utf(N, utf8) -> +to_fake_utf8(N) -> <<0:3, W:3, Z:6, Y:6, X:6>> = <>, <<34/utf8, 2#11110:5, W:3, 2#10:2, Z:6, 2#10:2, Y:6, 2#10:2, X:6, 34/utf8>>. @@ -1027,4 +1023,21 @@ clean_string_test_() -> ]. +single_quoted_string_test_() -> + [ + {"single quoted string", ?_assertEqual( + [{string, <<"hello world">>}, end_json], + decode(<<39, "hello world", 39>>, [single_quoted_strings]) + )}, + {"single quoted string with embedded double quotes", ?_assertEqual( + [{string, <<"quoth the raven, \"nevermore\"">>}, end_json], + decode(<<39, "quoth the raven, \"nevermore\"", 39>>, [single_quoted_strings]) + )}, + {"string with embedded single quotes", ?_assertEqual( + [{string, <<"quoth the raven, 'nevermore'">>}, end_json], + decode(<<34, "quoth the raven, 'nevermore'", 34>>, []) + )} + ]. + + -endif. \ No newline at end of file From 9244845c3784b079f4ff7cb5dfaafa2a5fe32a09 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Thu, 21 Feb 2013 00:13:07 -0800 Subject: [PATCH 48/61] fix order of comments test --- src/jsx_decoder.erl | 164 ++++++++++++++++++++++---------------------- 1 file changed, 82 insertions(+), 82 deletions(-) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 4be6de0..0ce2afd 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -761,91 +761,90 @@ decode(JSON, Config) -> comments_test_() -> [ {"preceeding // comment", ?_assertEqual( - decode(<<"// comment ", ?newline, "[]">>, [comments]), - [start_array, end_array, end_json] + [start_array, end_array, end_json], + decode(<<"// comment ", ?newline, "[]">>, [comments]) )}, {"preceeding /**/ comment", ?_assertEqual( - decode(<<"/* comment */[]">>, [comments]), - [start_array, end_array, end_json] + [start_array, end_array, end_json], + decode(<<"/* comment */[]">>, [comments]) )}, {"trailing // comment", ?_assertEqual( - decode(<<"[]// comment", ?newline>>, [comments]), - [start_array, end_array, end_json] + [start_array, end_array, end_json], + decode(<<"[]// comment", ?newline>>, [comments]) )}, {"trailing // comment (no newline)", ?_assertEqual( - decode(<<"[]// comment">>, [comments]), - [start_array, end_array, end_json] + [start_array, end_array, end_json], + decode(<<"[]// comment">>, [comments]) )}, {"trailing /**/ comment", ?_assertEqual( - decode(<<"[] /* comment */">>, [comments]), - [start_array, end_array, end_json] + [start_array, end_array, end_json], + decode(<<"[] /* comment */">>, [comments]) )}, {"// comment inside array", ?_assertEqual( - decode(<<"[ // comment", ?newline, "]">>, [comments]), - [start_array, end_array, end_json] + [start_array, end_array, end_json], + decode(<<"[ // comment", ?newline, "]">>, [comments]) )}, {"/**/ comment inside array", ?_assertEqual( - decode(<<"[ /* comment */ ]">>, [comments]), - [start_array, end_array, end_json] + [start_array, end_array, end_json], + decode(<<"[ /* comment */ ]">>, [comments]) )}, {"// comment at beginning of array", ?_assertEqual( - decode(<<"[ // comment", ?newline, "true", ?newline, "]">>, [comments]), - [start_array, {literal, true}, end_array, end_json] + [start_array, {literal, true}, end_array, end_json], + decode(<<"[ // comment", ?newline, "true", ?newline, "]">>, [comments]) )}, {"/**/ comment at beginning of array", ?_assertEqual( - decode(<<"[ /* comment */ true ]">>, [comments]), - [start_array, {literal, true}, end_array, end_json] + [start_array, {literal, true}, end_array, end_json], + decode(<<"[ /* comment */ true ]">>, [comments]) )}, {"// comment at end of array", ?_assertEqual( - decode(<<"[ true // comment", ?newline, "]">>, [comments]), - [start_array, {literal, true}, end_array, end_json] + [start_array, {literal, true}, end_array, end_json], + decode(<<"[ true // comment", ?newline, "]">>, [comments]) )}, {"/**/ comment at end of array", ?_assertEqual( - decode(<<"[ true /* comment */ ]">>, [comments]), - [start_array, {literal, true}, end_array, end_json] + [start_array, {literal, true}, end_array, end_json], + decode(<<"[ true /* comment */ ]">>, [comments]) )}, {"// comment midarray (post comma)", ?_assertEqual( - decode(<<"[ true, // comment", ?newline, "false ]">>, [comments]), - [start_array, {literal, true}, {literal, false}, end_array, end_json] + [start_array, {literal, true}, {literal, false}, end_array, end_json], + decode(<<"[ true, // comment", ?newline, "false ]">>, [comments]) )}, {"/**/ comment midarray (post comma)", ?_assertEqual( - decode(<<"[ true, /* comment */ false ]">>, [comments]), - [start_array, {literal, true}, {literal, false}, end_array, end_json] + [start_array, {literal, true}, {literal, false}, end_array, end_json], + decode(<<"[ true, /* comment */ false ]">>, [comments]) )}, {"// comment midarray (pre comma)", ?_assertEqual( - decode(<<"[ true// comment", ?newline, ", false ]">>, [comments]), - [start_array, {literal, true}, {literal, false}, end_array, end_json] + [start_array, {literal, true}, {literal, false}, end_array, end_json], + decode(<<"[ true// comment", ?newline, ", false ]">>, [comments]) )}, {"/**/ comment midarray (pre comma)", ?_assertEqual( - decode(<<"[ true/* comment */, false ]">>, [comments]), - [start_array, {literal, true}, {literal, false}, end_array, end_json] + [start_array, {literal, true}, {literal, false}, end_array, end_json], + decode(<<"[ true/* comment */, false ]">>, [comments]) )}, {"// comment inside object", ?_assertEqual( - decode(<<"{ // comment", ?newline, "}">>, [comments]), - [start_object, end_object, end_json] + [start_object, end_object, end_json], + decode(<<"{ // comment", ?newline, "}">>, [comments]) )}, {"/**/ comment inside object", ?_assertEqual( - decode(<<"{ /* comment */ }">>, [comments]), - [start_object, end_object, end_json] + [start_object, end_object, end_json], + decode(<<"{ /* comment */ }">>, [comments]) )}, {"// comment at beginning of object", ?_assertEqual( - decode(<<"{ // comment", ?newline, " \"key\": true", ?newline, "}">>, [comments]), - [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json] + [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json], + decode(<<"{ // comment", ?newline, " \"key\": true", ?newline, "}">>, [comments]) )}, {"/**/ comment at beginning of object", ?_assertEqual( - decode(<<"{ /* comment */ \"key\": true }">>, [comments]), - [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json] + [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json], + decode(<<"{ /* comment */ \"key\": true }">>, [comments]) )}, {"// comment at end of object", ?_assertEqual( - decode(<<"{ \"key\": true // comment", ?newline, "}">>, [comments]), - [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json] + [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json], + decode(<<"{ \"key\": true // comment", ?newline, "}">>, [comments]) )}, {"/**/ comment at end of object", ?_assertEqual( - decode(<<"{ \"key\": true /* comment */ }">>, [comments]), - [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json] + [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json], + decode(<<"{ \"key\": true /* comment */ }">>, [comments]) )}, {"// comment midobject (post comma)", ?_assertEqual( - decode(<<"{ \"x\": true, // comment", ?newline, "\"y\": false }">>, [comments]), [ start_object, {key, <<"x">>}, @@ -854,10 +853,10 @@ comments_test_() -> {literal, false}, end_object, end_json - ] + ], + decode(<<"{ \"x\": true, // comment", ?newline, "\"y\": false }">>, [comments]) )}, {"/**/ comment midobject (post comma)", ?_assertEqual( - decode(<<"{ \"x\": true, /* comment */", ?newline, "\"y\": false }">>, [comments]), [ start_object, {key, <<"x">>}, @@ -866,10 +865,10 @@ comments_test_() -> {literal, false}, end_object, end_json - ] + ], + decode(<<"{ \"x\": true, /* comment */", ?newline, "\"y\": false }">>, [comments]) )}, {"// comment midobject (pre comma)", ?_assertEqual( - decode(<<"{ \"x\": true// comment", ?newline, ", \"y\": false }">>, [comments]), [ start_object, {key, <<"x">>}, @@ -878,10 +877,10 @@ comments_test_() -> {literal, false}, end_object, end_json - ] + ], + decode(<<"{ \"x\": true// comment", ?newline, ", \"y\": false }">>, [comments]) )}, {"/**/ comment midobject (pre comma)", ?_assertEqual( - decode(<<"{ \"x\": true/* comment */", ?newline, ", \"y\": false }">>, [comments]), [ start_object, {key, <<"x">>}, @@ -890,71 +889,72 @@ comments_test_() -> {literal, false}, end_object, end_json - ] + ], + decode(<<"{ \"x\": true/* comment */", ?newline, ", \"y\": false }">>, [comments]) )}, {"// comment precolon", ?_assertEqual( - decode(<<"{ \"key\" // comment", ?newline, ": true }">>, [comments]), - [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json] + [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json], + decode(<<"{ \"key\" // comment", ?newline, ": true }">>, [comments]) )}, {"/**/ comment precolon", ?_assertEqual( - decode(<<"{ \"key\"/* comment */: true }">>, [comments]), - [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json] + [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json], + decode(<<"{ \"key\"/* comment */: true }">>, [comments]) )}, {"// comment postcolon", ?_assertEqual( - decode(<<"{ \"key\": // comment", ?newline, " true }">>, [comments]), - [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json] + [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json], + decode(<<"{ \"key\": // comment", ?newline, " true }">>, [comments]) )}, {"/**/ comment postcolon", ?_assertEqual( - decode(<<"{ \"key\":/* comment */ true }">>, [comments]), - [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json] + [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json], + decode(<<"{ \"key\":/* comment */ true }">>, [comments]) )}, {"// comment terminating zero", ?_assertEqual( - decode(<<"[ 0// comment", ?newline, "]">>, [comments]), - [start_array, {integer, 0}, end_array, end_json] + [start_array, {integer, 0}, end_array, end_json], + decode(<<"[ 0// comment", ?newline, "]">>, [comments]) )}, {"// comment terminating integer", ?_assertEqual( - decode(<<"[ 1// comment", ?newline, "]">>, [comments]), - [start_array, {integer, 1}, end_array, end_json] + [start_array, {integer, 1}, end_array, end_json], + decode(<<"[ 1// comment", ?newline, "]">>, [comments]) )}, {"// comment terminating float", ?_assertEqual( - decode(<<"[ 1.0// comment", ?newline, "]">>, [comments]), - [start_array, {float, 1.0}, end_array, end_json] + [start_array, {float, 1.0}, end_array, end_json], + decode(<<"[ 1.0// comment", ?newline, "]">>, [comments]) )}, {"// comment terminating exp", ?_assertEqual( - decode(<<"[ 1e1// comment", ?newline, "]">>, [comments]), - [start_array, {float, 1.0e1}, end_array, end_json] + [start_array, {float, 1.0e1}, end_array, end_json], + decode(<<"[ 1e1// comment", ?newline, "]">>, [comments]) )}, {"/**/ comment terminating zero", ?_assertEqual( - decode(<<"[ 0/* comment */ ]">>, [comments]), - [start_array, {integer, 0}, end_array, end_json] + [start_array, {integer, 0}, end_array, end_json], + decode(<<"[ 0/* comment */ ]">>, [comments]) )}, {"/**/ comment terminating integer", ?_assertEqual( - decode(<<"[ 1/* comment */ ]">>, [comments]), - [start_array, {integer, 1}, end_array, end_json] + [start_array, {integer, 1}, end_array, end_json], + decode(<<"[ 1/* comment */ ]">>, [comments]) )}, {"/**/ comment terminating float", ?_assertEqual( - decode(<<"[ 1.0/* comment */ ]">>, [comments]), - [start_array, {float, 1.0}, end_array, end_json] + [start_array, {float, 1.0}, end_array, end_json], + decode(<<"[ 1.0/* comment */ ]">>, [comments]) )}, {"/**/ comment terminating exp", ?_assertEqual( - decode(<<"[ 1e1/* comment */ ]">>, [comments]), - [start_array, {float, 1.0e1}, end_array, end_json] + [start_array, {float, 1.0e1}, end_array, end_json], + decode(<<"[ 1e1/* comment */ ]">>, [comments]) )}, {"/**/ comment following /**/ comment", ?_assertEqual( - decode(<<"[/* comment *//* comment */true]">>, [comments]), - [start_array, {literal, true}, end_array, end_json] + [start_array, {literal, true}, end_array, end_json], + decode(<<"[/* comment *//* comment */true]">>, [comments]) )}, {"/**/ comment following // comment", ?_assertEqual( - decode(<<"[// comment", ?newline, "/* comment */true]">>, [comments]), - [start_array, {literal, true}, end_array, end_json] + [start_array, {literal, true}, end_array, end_json], + decode(<<"[// comment", ?newline, "/* comment */true]">>, [comments]) )}, {"// comment following /**/ comment", ?_assertEqual( - decode(<<"[/* comment */// comment", ?newline, "true]">>, [comments]), - [start_array, {literal, true}, end_array, end_json] + [start_array, {literal, true}, end_array, end_json], + decode(<<"[/* comment */// comment", ?newline, "true]">>, [comments]) )}, {"// comment following // comment", ?_assertEqual( - decode(<<"[// comment", ?newline, "// comment", ?newline, "true]">>, [comments]), - [start_array, {literal, true}, end_array, end_json] + [start_array, {literal, true}, end_array, end_json], + decode(<<"[// comment", ?newline, "// comment", ?newline, "true]">>, [comments]) )} ]. From 74a688c82760c8b9960a675f634a2b30c5e6cef5 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sat, 23 Feb 2013 22:21:35 -0800 Subject: [PATCH 49/61] reformat clean_string tests --- src/jsx_utils.erl | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index f15edb2..bbfca93 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -768,30 +768,34 @@ bad_utf8_test_() -> binary:copy(<<16#fffd/utf8>>, 2), clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #config{replaced_bad_utf8=true}) )}, - {"3 continuation bytes", - ?_assertEqual({error, badarg}, clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{})) - }, + {"3 continuation bytes", ?_assertEqual( + {error, badarg}, + clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{}) + )}, {"3 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 3), clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{replaced_bad_utf8=true}) )}, - {"4 continuation bytes", - ?_assertEqual({error, badarg}, clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{})) - }, + {"4 continuation bytes", ?_assertEqual( + {error, badarg}, + clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{}) + )}, {"4 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 4), clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{replaced_bad_utf8=true}) )}, - {"5 continuation bytes", - ?_assertEqual({error, badarg}, clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{})) - }, + {"5 continuation bytes", ?_assertEqual( + {error, badarg}, + clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{}) + )}, {"5 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 5), clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{replaced_bad_utf8=true}) )}, - {"6 continuation bytes", - ?_assertEqual({error, badarg}, clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{})) - }, + {"6 continuation bytes", ?_assertEqual( + {error, badarg}, + clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{}) + )}, {"6 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 6), clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{replaced_bad_utf8=true}) @@ -807,7 +811,10 @@ bad_utf8_test_() -> #config{replaced_bad_utf8=true} ) )}, - {"lonely start byte", ?_assertEqual({error, badarg}, clean_string(<<16#00c0>>, #config{}))}, + {"lonely start byte", ?_assertEqual( + {error, badarg}, + clean_string(<<16#00c0>>, #config{}) + )}, {"lonely start byte replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(<<16#00c0>>, #config{replaced_bad_utf8=true}) From 230be8ebc2e341fac05dd3a61167c844b5df567a Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sat, 23 Feb 2013 22:36:35 -0800 Subject: [PATCH 50/61] import better tests --- src/jsx_utils.erl | 658 ++++++++++++++++++++++++++++++---------------- 1 file changed, 425 insertions(+), 233 deletions(-) diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index bbfca93..505a3b1 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -145,134 +145,355 @@ to_hex(X) -> X + 48. %% ascii "1" is [49], "2" is [50], etc... clean_string(Bin, #config{dirty_strings=true}) -> Bin; -clean_string(Bin, Config) -> clean_string(Bin, <<>>, Config). - -clean_string(Bin, Acc, Config) -> - Length = cut(Bin), - <> = Bin, - case Rest of - <<>> -> <>; - _ -> maybe_escape(Rest, <>, Config) +clean_string(Bin, Config) -> + case Config#config.replaced_bad_utf8 orelse Config#config.escaped_strings of + true -> clean(Bin, [], Config) + ; false -> ensure_clean(Bin), Bin end. -cut(Bin) -> cut(Bin, 0). +%% fast path for no escaping and no correcting, throws error if string is 'bad' +ensure_clean(<<>>) -> ok; +ensure_clean(<<0, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<1, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<2, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<3, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<4, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<5, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<6, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<7, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<8, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<9, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<10, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<11, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<12, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<13, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<14, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<15, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<16, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<17, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<18, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<19, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<20, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<21, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<22, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<23, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<24, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<25, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<26, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<27, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<28, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<29, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<30, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<31, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<32, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<33, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<34, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<35, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<36, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<37, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<38, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<39, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<40, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<41, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<42, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<43, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<44, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<45, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<46, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<47, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<48, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<49, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<50, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<51, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<52, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<53, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<54, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<55, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<56, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<57, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<58, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<59, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<60, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<61, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<62, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<63, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<64, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<65, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<66, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<67, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<68, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<69, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<70, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<71, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<72, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<73, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<74, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<75, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<76, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<77, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<78, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<79, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<80, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<81, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<82, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<83, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<84, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<85, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<86, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<87, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<88, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<89, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<90, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<91, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<92, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<93, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<94, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<95, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<96, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<97, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<98, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<99, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<100, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<101, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<102, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<103, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<104, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<105, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<106, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<107, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<108, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<109, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<110, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<111, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<112, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<113, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<114, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<115, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<116, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<117, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<118, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<119, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<120, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<121, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<122, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<123, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<124, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<125, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<126, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<<127, Rest/binary>>) -> ensure_clean(Rest); +ensure_clean(<>) when X < 16#dcff -> ensure_clean(Rest); +ensure_clean(<>) when X > 16#dfff, X < 16#fdd0 -> ensure_clean(Rest); +ensure_clean(<>) when X > 16#fdef, X < 16#fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#10000, X < 16#1fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#20000, X < 16#2fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#30000, X < 16#3fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#40000, X < 16#4fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#50000, X < 16#5fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#60000, X < 16#6fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#70000, X < 16#7fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#80000, X < 16#8fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#90000, X < 16#9fffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#a0000, X < 16#afffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#b0000, X < 16#bfffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#c0000, X < 16#cfffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#d0000, X < 16#dfffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#e0000, X < 16#efffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#f0000, X < 16#ffffe -> ensure_clean(Rest); +ensure_clean(<>) when X >= 16#100000, X < 16#10fffe -> ensure_clean(Rest); +ensure_clean(Bin) -> erlang:error(badarg, [Bin]). -cut(<<32, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<33, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<35, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<36, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<37, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<38, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<39, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<40, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<41, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<42, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<43, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<44, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<45, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<46, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<48, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<49, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<50, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<51, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<52, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<53, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<54, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<55, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<56, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<57, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<58, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<59, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<60, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<61, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<62, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<63, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<64, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<65, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<66, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<67, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<68, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<69, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<70, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<71, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<72, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<73, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<74, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<75, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<76, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<77, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<78, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<79, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<80, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<81, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<82, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<83, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<84, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<85, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<86, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<87, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<88, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<89, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<90, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<91, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<93, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<94, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<95, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<96, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<97, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<98, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<99, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<100, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<101, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<102, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<103, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<104, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<105, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<106, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<107, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<108, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<109, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<110, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<111, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<112, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<113, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<114, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<115, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<116, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<117, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<118, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<119, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<120, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<121, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<122, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<123, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<124, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<125, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<126, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<127, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<>, N) when X >= 128, X < 16#0800 -> cut(Rest, N + 2); -cut(<>, N) when X >= 16#0800, X < 16#2028 -> cut(Rest, N + 3); -cut(<>, N) when X >= 16#202a, X < 16#d800 -> cut(Rest, N + 3); -cut(<>, N) when X > 16#dfff, X < 16#fdd0 -> cut(Rest, N + 3); -cut(<>, N) when X > 16#fdef, X < 16#fffe -> cut(Rest, N + 3); -cut(<>, N) when X >= 16#10000, X < 16#1fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#20000, X < 16#2fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#30000, X < 16#3fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#40000, X < 16#4fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#50000, X < 16#5fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#60000, X < 16#6fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#70000, X < 16#7fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#80000, X < 16#8fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#90000, X < 16#9fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#a0000, X < 16#afffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#b0000, X < 16#bfffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#c0000, X < 16#cfffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#d0000, X < 16#dfffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#e0000, X < 16#efffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#f0000, X < 16#ffffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#100000, X < 16#10fffe -> cut(Rest, N + 4); -cut(_, N) -> N. + +%% escape and/or replace bad codepoints if requested +clean(<<>>, Acc, _Config) -> unicode:characters_to_binary(lists:reverse(Acc)); +clean(<<0, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(0, Config) ++ Acc, Config); +clean(<<1, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(1, Config) ++ Acc, Config); +clean(<<2, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(2, Config) ++ Acc, Config); +clean(<<3, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(3, Config) ++ Acc, Config); +clean(<<4, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(4, Config) ++ Acc, Config); +clean(<<5, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(5, Config) ++ Acc, Config); +clean(<<6, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(6, Config) ++ Acc, Config); +clean(<<7, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(7, Config) ++ Acc, Config); +clean(<<8, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(8, Config) ++ Acc, Config); +clean(<<9, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(9, Config) ++ Acc, Config); +clean(<<10, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(10, Config) ++ Acc, Config); +clean(<<11, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(11, Config) ++ Acc, Config); +clean(<<12, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(12, Config) ++ Acc, Config); +clean(<<13, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(13, Config) ++ Acc, Config); +clean(<<14, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(14, Config) ++ Acc, Config); +clean(<<15, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(15, Config) ++ Acc, Config); +clean(<<16, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(16, Config) ++ Acc, Config); +clean(<<17, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(17, Config) ++ Acc, Config); +clean(<<18, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(18, Config) ++ Acc, Config); +clean(<<19, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(19, Config) ++ Acc, Config); +clean(<<20, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(20, Config) ++ Acc, Config); +clean(<<21, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(21, Config) ++ Acc, Config); +clean(<<22, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(22, Config) ++ Acc, Config); +clean(<<23, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(23, Config) ++ Acc, Config); +clean(<<24, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(24, Config) ++ Acc, Config); +clean(<<25, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(25, Config) ++ Acc, Config); +clean(<<26, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(26, Config) ++ Acc, Config); +clean(<<27, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(27, Config) ++ Acc, Config); +clean(<<28, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(28, Config) ++ Acc, Config); +clean(<<29, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(29, Config) ++ Acc, Config); +clean(<<30, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(30, Config) ++ Acc, Config); +clean(<<31, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(31, Config) ++ Acc, Config); +clean(<<32, Rest/binary>>, Acc, Config) -> clean(Rest, [32] ++ Acc, Config); +clean(<<33, Rest/binary>>, Acc, Config) -> clean(Rest, [33] ++ Acc, Config); +clean(<<34, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(34, Config) ++ Acc, Config); +clean(<<35, Rest/binary>>, Acc, Config) -> clean(Rest, [35] ++ Acc, Config); +clean(<<36, Rest/binary>>, Acc, Config) -> clean(Rest, [36] ++ Acc, Config); +clean(<<37, Rest/binary>>, Acc, Config) -> clean(Rest, [37] ++ Acc, Config); +clean(<<38, Rest/binary>>, Acc, Config) -> clean(Rest, [38] ++ Acc, Config); +clean(<<39, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(39, Config) ++ Acc, Config); +clean(<<40, Rest/binary>>, Acc, Config) -> clean(Rest, [40] ++ Acc, Config); +clean(<<41, Rest/binary>>, Acc, Config) -> clean(Rest, [41] ++ Acc, Config); +clean(<<42, Rest/binary>>, Acc, Config) -> clean(Rest, [42] ++ Acc, Config); +clean(<<43, Rest/binary>>, Acc, Config) -> clean(Rest, [43] ++ Acc, Config); +clean(<<44, Rest/binary>>, Acc, Config) -> clean(Rest, [44] ++ Acc, Config); +clean(<<45, Rest/binary>>, Acc, Config) -> clean(Rest, [45] ++ Acc, Config); +clean(<<46, Rest/binary>>, Acc, Config) -> clean(Rest, [46] ++ Acc, Config); +clean(<<47, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(47, Config) ++ Acc, Config); +clean(<<48, Rest/binary>>, Acc, Config) -> clean(Rest, [48] ++ Acc, Config); +clean(<<49, Rest/binary>>, Acc, Config) -> clean(Rest, [49] ++ Acc, Config); +clean(<<50, Rest/binary>>, Acc, Config) -> clean(Rest, [50] ++ Acc, Config); +clean(<<51, Rest/binary>>, Acc, Config) -> clean(Rest, [51] ++ Acc, Config); +clean(<<52, Rest/binary>>, Acc, Config) -> clean(Rest, [52] ++ Acc, Config); +clean(<<53, Rest/binary>>, Acc, Config) -> clean(Rest, [53] ++ Acc, Config); +clean(<<54, Rest/binary>>, Acc, Config) -> clean(Rest, [54] ++ Acc, Config); +clean(<<55, Rest/binary>>, Acc, Config) -> clean(Rest, [55] ++ Acc, Config); +clean(<<56, Rest/binary>>, Acc, Config) -> clean(Rest, [56] ++ Acc, Config); +clean(<<57, Rest/binary>>, Acc, Config) -> clean(Rest, [57] ++ Acc, Config); +clean(<<58, Rest/binary>>, Acc, Config) -> clean(Rest, [58] ++ Acc, Config); +clean(<<59, Rest/binary>>, Acc, Config) -> clean(Rest, [59] ++ Acc, Config); +clean(<<60, Rest/binary>>, Acc, Config) -> clean(Rest, [60] ++ Acc, Config); +clean(<<61, Rest/binary>>, Acc, Config) -> clean(Rest, [61] ++ Acc, Config); +clean(<<62, Rest/binary>>, Acc, Config) -> clean(Rest, [62] ++ Acc, Config); +clean(<<63, Rest/binary>>, Acc, Config) -> clean(Rest, [63] ++ Acc, Config); +clean(<<64, Rest/binary>>, Acc, Config) -> clean(Rest, [64] ++ Acc, Config); +clean(<<65, Rest/binary>>, Acc, Config) -> clean(Rest, [65] ++ Acc, Config); +clean(<<66, Rest/binary>>, Acc, Config) -> clean(Rest, [66] ++ Acc, Config); +clean(<<67, Rest/binary>>, Acc, Config) -> clean(Rest, [67] ++ Acc, Config); +clean(<<68, Rest/binary>>, Acc, Config) -> clean(Rest, [68] ++ Acc, Config); +clean(<<69, Rest/binary>>, Acc, Config) -> clean(Rest, [69] ++ Acc, Config); +clean(<<70, Rest/binary>>, Acc, Config) -> clean(Rest, [70] ++ Acc, Config); +clean(<<71, Rest/binary>>, Acc, Config) -> clean(Rest, [71] ++ Acc, Config); +clean(<<72, Rest/binary>>, Acc, Config) -> clean(Rest, [72] ++ Acc, Config); +clean(<<73, Rest/binary>>, Acc, Config) -> clean(Rest, [73] ++ Acc, Config); +clean(<<74, Rest/binary>>, Acc, Config) -> clean(Rest, [74] ++ Acc, Config); +clean(<<75, Rest/binary>>, Acc, Config) -> clean(Rest, [75] ++ Acc, Config); +clean(<<76, Rest/binary>>, Acc, Config) -> clean(Rest, [76] ++ Acc, Config); +clean(<<77, Rest/binary>>, Acc, Config) -> clean(Rest, [77] ++ Acc, Config); +clean(<<78, Rest/binary>>, Acc, Config) -> clean(Rest, [78] ++ Acc, Config); +clean(<<79, Rest/binary>>, Acc, Config) -> clean(Rest, [79] ++ Acc, Config); +clean(<<80, Rest/binary>>, Acc, Config) -> clean(Rest, [80] ++ Acc, Config); +clean(<<81, Rest/binary>>, Acc, Config) -> clean(Rest, [81] ++ Acc, Config); +clean(<<82, Rest/binary>>, Acc, Config) -> clean(Rest, [82] ++ Acc, Config); +clean(<<83, Rest/binary>>, Acc, Config) -> clean(Rest, [83] ++ Acc, Config); +clean(<<84, Rest/binary>>, Acc, Config) -> clean(Rest, [84] ++ Acc, Config); +clean(<<85, Rest/binary>>, Acc, Config) -> clean(Rest, [85] ++ Acc, Config); +clean(<<86, Rest/binary>>, Acc, Config) -> clean(Rest, [86] ++ Acc, Config); +clean(<<87, Rest/binary>>, Acc, Config) -> clean(Rest, [87] ++ Acc, Config); +clean(<<88, Rest/binary>>, Acc, Config) -> clean(Rest, [88] ++ Acc, Config); +clean(<<89, Rest/binary>>, Acc, Config) -> clean(Rest, [89] ++ Acc, Config); +clean(<<90, Rest/binary>>, Acc, Config) -> clean(Rest, [90] ++ Acc, Config); +clean(<<91, Rest/binary>>, Acc, Config) -> clean(Rest, [91] ++ Acc, Config); +clean(<<92, Rest/binary>>, Acc, Config) -> clean(Rest, maybe_replace(92, Config) ++ Acc, Config); +clean(<<93, Rest/binary>>, Acc, Config) -> clean(Rest, [93] ++ Acc, Config); +clean(<<94, Rest/binary>>, Acc, Config) -> clean(Rest, [94] ++ Acc, Config); +clean(<<95, Rest/binary>>, Acc, Config) -> clean(Rest, [95] ++ Acc, Config); +clean(<<96, Rest/binary>>, Acc, Config) -> clean(Rest, [96] ++ Acc, Config); +clean(<<97, Rest/binary>>, Acc, Config) -> clean(Rest, [97] ++ Acc, Config); +clean(<<98, Rest/binary>>, Acc, Config) -> clean(Rest, [98] ++ Acc, Config); +clean(<<99, Rest/binary>>, Acc, Config) -> clean(Rest, [99] ++ Acc, Config); +clean(<<100, Rest/binary>>, Acc, Config) -> clean(Rest, [100] ++ Acc, Config); +clean(<<101, Rest/binary>>, Acc, Config) -> clean(Rest, [101] ++ Acc, Config); +clean(<<102, Rest/binary>>, Acc, Config) -> clean(Rest, [102] ++ Acc, Config); +clean(<<103, Rest/binary>>, Acc, Config) -> clean(Rest, [103] ++ Acc, Config); +clean(<<104, Rest/binary>>, Acc, Config) -> clean(Rest, [104] ++ Acc, Config); +clean(<<105, Rest/binary>>, Acc, Config) -> clean(Rest, [105] ++ Acc, Config); +clean(<<106, Rest/binary>>, Acc, Config) -> clean(Rest, [106] ++ Acc, Config); +clean(<<107, Rest/binary>>, Acc, Config) -> clean(Rest, [107] ++ Acc, Config); +clean(<<108, Rest/binary>>, Acc, Config) -> clean(Rest, [108] ++ Acc, Config); +clean(<<109, Rest/binary>>, Acc, Config) -> clean(Rest, [109] ++ Acc, Config); +clean(<<110, Rest/binary>>, Acc, Config) -> clean(Rest, [110] ++ Acc, Config); +clean(<<111, Rest/binary>>, Acc, Config) -> clean(Rest, [111] ++ Acc, Config); +clean(<<112, Rest/binary>>, Acc, Config) -> clean(Rest, [112] ++ Acc, Config); +clean(<<113, Rest/binary>>, Acc, Config) -> clean(Rest, [113] ++ Acc, Config); +clean(<<114, Rest/binary>>, Acc, Config) -> clean(Rest, [114] ++ Acc, Config); +clean(<<115, Rest/binary>>, Acc, Config) -> clean(Rest, [115] ++ Acc, Config); +clean(<<116, Rest/binary>>, Acc, Config) -> clean(Rest, [116] ++ Acc, Config); +clean(<<117, Rest/binary>>, Acc, Config) -> clean(Rest, [117] ++ Acc, Config); +clean(<<118, Rest/binary>>, Acc, Config) -> clean(Rest, [118] ++ Acc, Config); +clean(<<119, Rest/binary>>, Acc, Config) -> clean(Rest, [119] ++ Acc, Config); +clean(<<120, Rest/binary>>, Acc, Config) -> clean(Rest, [120] ++ Acc, Config); +clean(<<121, Rest/binary>>, Acc, Config) -> clean(Rest, [121] ++ Acc, Config); +clean(<<122, Rest/binary>>, Acc, Config) -> clean(Rest, [122] ++ Acc, Config); +clean(<<123, Rest/binary>>, Acc, Config) -> clean(Rest, [123] ++ Acc, Config); +clean(<<124, Rest/binary>>, Acc, Config) -> clean(Rest, [124] ++ Acc, Config); +clean(<<125, Rest/binary>>, Acc, Config) -> clean(Rest, [125] ++ Acc, Config); +clean(<<126, Rest/binary>>, Acc, Config) -> clean(Rest, [126] ++ Acc, Config); +clean(<<127, Rest/binary>>, Acc, Config) -> clean(Rest, [127] ++ Acc, Config); +clean(<>, Acc, Config) when X == 16#2028; X == 16#2029 -> + clean(Rest, maybe_replace(X, Config) ++ Acc, Config); +clean(<>, Acc, Config) when X < 16#dcff -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X > 16#dfff, X < 16#fdd0 -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X > 16#fdef, X < 16#fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#10000, X < 16#1fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#20000, X < 16#2fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#30000, X < 16#3fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#40000, X < 16#4fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#50000, X < 16#5fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#60000, X < 16#6fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#70000, X < 16#7fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#80000, X < 16#8fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#90000, X < 16#9fffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#a0000, X < 16#afffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#b0000, X < 16#bfffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#c0000, X < 16#cfffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#d0000, X < 16#dfffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#e0000, X < 16#efffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#f0000, X < 16#ffffe -> + clean(Rest, [X] ++ Acc, Config); +clean(<>, Acc, Config) when X >= 16#100000, X < 16#10fffe -> + clean(Rest, [X] ++ Acc, Config); +%% noncharacters +clean(<<_/utf8, Rest/binary>>, Acc, Config) -> + clean(Rest, maybe_replace(noncharacter, Config) ++ Acc, Config); +%% surrogates +clean(<<237, X, _, Rest/binary>>, Acc, Config) when X >= 160 -> + clean(Rest, maybe_replace(surrogate, Config) ++ Acc, Config); +%% u+fffe and u+ffff for R14BXX +clean(<<239, 191, X, Rest/binary>>, Acc, Config) when X == 190; X == 191 -> + clean(Rest, maybe_replace(noncharacter, Config) ++ Acc, Config); +%% overlong encodings and missing continuations of a 2 byte sequence +clean(<>, Acc, Config) when X >= 192, X =< 223 -> + clean(strip_continuations(Rest, 1), maybe_replace(badutf, Config) ++ Acc, Config); +%% overlong encodings and missing continuations of a 3 byte sequence +clean(<>, Acc, Config) when X >= 224, X =< 239 -> + clean(strip_continuations(Rest, 2), maybe_replace(badutf, Config) ++ Acc, Config); +%% overlong encodings and missing continuations of a 4 byte sequence +clean(<>, Acc, Config) when X >= 240, X =< 247 -> + clean(strip_continuations(Rest, 3), maybe_replace(badutf, Config) ++ Acc, Config); +clean(<<_, Rest/binary>>, Acc, Config) -> + clean(Rest, maybe_replace(badutf, Config) ++ Acc, Config). strip_continuations(Bin, 0) -> Bin; @@ -282,63 +503,34 @@ strip_continuations(<>, N) when X >= 128, X =< 191 -> strip_continuations(Bin, _) -> Bin. -maybe_escape(<>, Acc, #config{escaped_strings=true} = Config) -> - case escape(Codepoint, Config) of - inescapable -> noncharacter(<>, Acc, Config); - Escaped -> clean_string(Rest, <>, Config) +maybe_replace($\b, #config{escaped_strings=true}) -> [$b, $\\]; +maybe_replace($\t, #config{escaped_strings=true}) -> [$t, $\\]; +maybe_replace($\n, #config{escaped_strings=true}) -> [$n, $\\]; +maybe_replace($\f, #config{escaped_strings=true}) -> [$f, $\\]; +maybe_replace($\r, #config{escaped_strings=true}) -> [$r, $\\]; +maybe_replace($\", #config{escaped_strings=true}) -> [$\", $\\]; +maybe_replace($', Config=#config{escaped_strings=true}) -> + case Config#config.single_quoted_strings of + true -> [$', $\\] + ; false -> [$'] end; -maybe_escape(<>, Acc, Config) -> - case escape(Codepoint, Config) of - inescapable -> noncharacter(<>, Acc, Config); - _ -> clean_string(Rest, <>, Config) +maybe_replace($/, Config=#config{escaped_strings=true}) -> + case Config#config.escaped_forward_slashes of + true -> [$/, $\\] + ; false -> [$/] end; -maybe_escape(Bin, Acc, Config) -> noncharacter(Bin, Acc, Config). - - -escape($\b, _) -> <<"\\b">>; -escape($\t, _) -> <<"\\t">>; -escape($\n, _) -> <<"\\n">>; -escape($\f, _) -> <<"\\f">>; -escape($\r, _) -> <<"\\r">>; -escape($\", _) -> <<"\\\"">>; -escape($\\, _) -> <<"\\\\">>; -escape($/, #config{escaped_forward_slashes=true}) -> <<"\\/">>; -escape($/, _) -> <<"/">>; -escape(16#2028, #config{unescaped_jsonp=true}) -> <<16#2028/utf8>>; -escape(16#2028, _) -> <<"\\u2028">>; -escape(16#2029, #config{unescaped_jsonp=true}) -> <<16#2029/utf8>>; -escape(16#2029, _) -> <<"\\u2029">>; -escape(X, _) when X < 32 -> - <> = <>, - <<"\\u", (to_hex(A)), (to_hex(B)), (to_hex(C)), (to_hex(D))>>; -escape(_, _) -> inescapable. - - -%% noncharacters and reserved space -noncharacter(<<_/utf8, Rest/binary>>, Acc, Config) -> - maybe_replace(Rest, Acc, Config); -%% surrogates -noncharacter(<<237, X, _, Rest/binary>>, Acc, Config) when X >= 160 -> - maybe_replace(Rest, Acc, Config); -%% u+fffe and u+ffff for R14BXX -noncharacter(<<239, 191, X, Rest/binary>>, Acc, Config) when X == 190; X == 191 -> - maybe_replace(Rest, Acc, Config); -%% overlong encodings and missing continuations of a 2 byte sequence -noncharacter(<>, Acc, Config) when X >= 192, X =< 223 -> - maybe_replace(strip_continuations(Rest, 1), Acc, Config); -%% overlong encodings and missing continuations of a 3 byte sequence -noncharacter(<>, Acc, Config) when X >= 224, X =< 239 -> - maybe_replace(strip_continuations(Rest, 2), Acc, Config); -%% overlong encodings and missing continuations of a 4 byte sequence -noncharacter(<>, Acc, Config) when X >= 240, X =< 247 -> - maybe_replace(strip_continuations(Rest, 3), Acc, Config); -noncharacter(<<_, Rest/binary>>, Acc, Config) -> - maybe_replace(Rest, Acc, Config). - - -maybe_replace(Bin, Acc, #config{replaced_bad_utf8=true} = Config) -> - clean_string(Bin, <>, Config); -maybe_replace(_, _, _) -> {error, badarg}. +maybe_replace($\\, #config{escaped_strings=true}) -> [$\\, $\\]; +maybe_replace(X, Config=#config{escaped_strings=true}) when X == 16#2028; X == 16#2029 -> + case Config#config.unescaped_jsonp of + true -> [X] + ; false -> lists:reverse(jsx_utils:json_escape_sequence(X)) + end; +maybe_replace(X, #config{escaped_strings=true}) when X < 32 -> + lists:reverse(jsx_utils:json_escape_sequence(X)); +maybe_replace(noncharacter, #config{replaced_bad_utf8=true}) -> [16#fffd]; +maybe_replace(surrogate, #config{replaced_bad_utf8=true}) -> [16#fffd]; +maybe_replace(badutf, #config{replaced_bad_utf8=true}) -> [16#fffd]; +maybe_replace(_, _) -> erlang:error(badarg). @@ -491,8 +683,8 @@ clean_string_test_() -> ] ++ [ { "reserved character: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), - ?_assertEqual( - {error, badarg}, + ?_assertError( + badarg, clean_string(Codepoint, #config{}) ) } || Codepoint <- reserved_space() @@ -507,8 +699,8 @@ clean_string_test_() -> ] ++ [ { "surrogate: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), - ?_assertEqual( - {error, badarg}, + ?_assertError( + badarg, clean_string(Codepoint, #config{}) ) } || Codepoint <- surrogates() @@ -523,8 +715,8 @@ clean_string_test_() -> ] ++ [ { "noncharacter: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), - ?_assertEqual( - {error, badarg}, + ?_assertError( + badarg, clean_string(Codepoint, #config{}) ) } || Codepoint <- noncharacters() @@ -539,8 +731,8 @@ clean_string_test_() -> ] ++ [ { "extended noncharacter: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), - ?_assertEqual( - {error, badarg}, + ?_assertError( + badarg, clean_string(Codepoint, #config{}) ) } || Codepoint <- extended_noncharacters() @@ -728,80 +920,80 @@ escape_test_() -> bad_utf8_test_() -> [ - {"noncharacter u+fffe", ?_assertEqual( - {error, badarg}, + {"noncharacter u+fffe", ?_assertError( + badarg, clean_string(to_fake_utf8(16#fffe), #config{}) )}, {"noncharacter u+fffe replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(to_fake_utf8(16#fffe), #config{replaced_bad_utf8=true}) )}, - {"noncharacter u+ffff", ?_assertEqual( - {error, badarg}, + {"noncharacter u+ffff", ?_assertError( + badarg, clean_string(to_fake_utf8(16#ffff), #config{}) )}, {"noncharacter u+ffff replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(to_fake_utf8(16#ffff), #config{replaced_bad_utf8=true}) )}, - {"orphan continuation byte u+0080", ?_assertEqual( - {error, badarg}, + {"orphan continuation byte u+0080", ?_assertError( + badarg, clean_string(<<16#0080>>, #config{}) )}, {"orphan continuation byte u+0080 replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(<<16#0080>>, #config{replaced_bad_utf8=true}) )}, - {"orphan continuation byte u+00bf", ?_assertEqual( - {error, badarg}, + {"orphan continuation byte u+00bf", ?_assertError( + badarg, clean_string(<<16#00bf>>, #config{}) )}, {"orphan continuation byte u+00bf replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(<<16#00bf>>, #config{replaced_bad_utf8=true}) )}, - {"2 continuation bytes", ?_assertEqual( - {error, badarg}, + {"2 continuation bytes", ?_assertError( + badarg, clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #config{}) )}, {"2 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 2), clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #config{replaced_bad_utf8=true}) )}, - {"3 continuation bytes", ?_assertEqual( - {error, badarg}, + {"3 continuation bytes", ?_assertError( + badarg, clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{}) )}, {"3 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 3), clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{replaced_bad_utf8=true}) )}, - {"4 continuation bytes", ?_assertEqual( - {error, badarg}, + {"4 continuation bytes", ?_assertError( + badarg, clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{}) )}, {"4 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 4), clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{replaced_bad_utf8=true}) )}, - {"5 continuation bytes", ?_assertEqual( - {error, badarg}, + {"5 continuation bytes", ?_assertError( + badarg, clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{}) )}, {"5 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 5), clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{replaced_bad_utf8=true}) )}, - {"6 continuation bytes", ?_assertEqual( - {error, badarg}, + {"6 continuation bytes", ?_assertError( + badarg, clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{}) )}, {"6 continuation bytes replaced", ?_assertEqual( binary:copy(<<16#fffd/utf8>>, 6), clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{replaced_bad_utf8=true}) )}, - {"all continuation bytes", ?_assertEqual( - {error, badarg}, + {"all continuation bytes", ?_assertError( + badarg, clean_string(<<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, #config{}) )}, {"all continuation bytes replaced", ?_assertEqual( @@ -811,104 +1003,104 @@ bad_utf8_test_() -> #config{replaced_bad_utf8=true} ) )}, - {"lonely start byte", ?_assertEqual( - {error, badarg}, + {"lonely start byte", ?_assertError( + badarg, clean_string(<<16#00c0>>, #config{}) )}, {"lonely start byte replaced", ?_assertEqual( <<16#fffd/utf8>>, clean_string(<<16#00c0>>, #config{replaced_bad_utf8=true}) )}, - {"lonely start bytes (2 byte)", ?_assertEqual( - {error, badarg}, + {"lonely start bytes (2 byte)", ?_assertError( + badarg, clean_string(<<16#00c0, 32, 16#00df>>, #config{}) )}, {"lonely start bytes (2 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, clean_string(<<16#00c0, 32, 16#00df>>, #config{replaced_bad_utf8=true}) )}, - {"lonely start bytes (3 byte)", ?_assertEqual( - {error, badarg}, + {"lonely start bytes (3 byte)", ?_assertError( + badarg, clean_string(<<16#00e0, 32, 16#00ef>>, #config{}) )}, {"lonely start bytes (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, clean_string(<<16#00e0, 32, 16#00ef>>, #config{replaced_bad_utf8=true}) )}, - {"lonely start bytes (4 byte)", ?_assertEqual( - {error, badarg}, + {"lonely start bytes (4 byte)", ?_assertError( + badarg, clean_string(<<16#00f0, 32, 16#00f7>>, #config{}) )}, {"lonely start bytes (4 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32, 16#fffd/utf8>>, clean_string(<<16#00f0, 32, 16#00f7>>, #config{replaced_bad_utf8=true}) )}, - {"missing continuation byte (3 byte)", ?_assertEqual( - {error, badarg}, + {"missing continuation byte (3 byte)", ?_assertError( + badarg, clean_string(<<224, 160, 32>>, #config{}) )}, {"missing continuation byte (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<224, 160, 32>>, #config{replaced_bad_utf8=true}) )}, - {"missing continuation byte (4 byte missing one)", ?_assertEqual( - {error, badarg}, + {"missing continuation byte (4 byte missing one)", ?_assertError( + badarg, clean_string(<<240, 144, 128, 32>>, #config{}) )}, {"missing continuation byte (4 byte missing one) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<240, 144, 128, 32>>, #config{replaced_bad_utf8=true}) )}, - {"missing continuation byte (4 byte missing two)", ?_assertEqual( - {error, badarg}, + {"missing continuation byte (4 byte missing two)", ?_assertError( + badarg, clean_string(<<240, 144, 32>>, #config{}) )}, {"missing continuation byte (4 byte missing two) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<240, 144, 32>>, #config{replaced_bad_utf8=true}) )}, - {"overlong encoding of u+002f (2 byte)", ?_assertEqual( - {error, badarg}, + {"overlong encoding of u+002f (2 byte)", ?_assertError( + badarg, clean_string(<<16#c0, 16#af, 32>>, #config{}) )}, {"overlong encoding of u+002f (2 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#c0, 16#af, 32>>, #config{replaced_bad_utf8=true}) )}, - {"overlong encoding of u+002f (3 byte)", ?_assertEqual( - {error, badarg}, + {"overlong encoding of u+002f (3 byte)", ?_assertError( + badarg, clean_string(<<16#e0, 16#80, 16#af, 32>>, #config{}) )}, {"overlong encoding of u+002f (3 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#e0, 16#80, 16#af, 32>>, #config{replaced_bad_utf8=true}) )}, - {"overlong encoding of u+002f (4 byte)", ?_assertEqual( - {error, badarg}, + {"overlong encoding of u+002f (4 byte)", ?_assertError( + badarg, clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #config{}) )}, {"overlong encoding of u+002f (4 byte) replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #config{replaced_bad_utf8=true}) )}, - {"highest overlong 2 byte sequence", ?_assertEqual( - {error, badarg}, + {"highest overlong 2 byte sequence", ?_assertError( + badarg, clean_string(<<16#c1, 16#bf, 32>>, #config{}) )}, {"highest overlong 2 byte sequence replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#c1, 16#bf, 32>>, #config{replaced_bad_utf8=true}) )}, - {"highest overlong 3 byte sequence", ?_assertEqual( - {error, badarg}, + {"highest overlong 3 byte sequence", ?_assertError( + badarg, clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #config{}) )}, {"highest overlong 3 byte sequence replaced", ?_assertEqual( <<16#fffd/utf8, 32>>, clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #config{replaced_bad_utf8=true}) )}, - {"highest overlong 4 byte sequence", ?_assertEqual( - {error, badarg}, + {"highest overlong 4 byte sequence", ?_assertError( + badarg, clean_string(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, #config{}) )}, {"highest overlong 4 byte sequence replaced", ?_assertEqual( From 6c51597f653d5163102b967488e4ac461dbf68e6 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sat, 23 Feb 2013 22:49:46 -0800 Subject: [PATCH 51/61] slightly generalize decoder for tests --- src/jsx_decoder.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 0ce2afd..72e8835 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -752,7 +752,7 @@ special_number_test_() -> decode(JSON, Config) -> try - (decoder(jsx, [], Config))(JSON) + start(JSON, {jsx, []}, [], jsx_utils:parse_config(Config)) catch error:badarg -> {error, badarg} end. From 257d7ee38724fa9fcd5d7e73d1dc48fa0a643d0b Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sun, 24 Feb 2013 00:54:16 -0800 Subject: [PATCH 52/61] make tests more consistent in decoder --- src/jsx_decoder.erl | 42 ++++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 72e8835..d1c30bc 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -703,15 +703,13 @@ done(Bin, Handler, Stack, Config) -> ?error([Bin, Handler, Stack, Config]). -include_lib("eunit/include/eunit.hrl"). +decode(JSON, Config) -> start(JSON, {jsx, []}, [], jsx_utils:parse_config(Config)). + + decode_test_() -> Data = jsx:test_cases(), - [ - { - Title, ?_assertEqual( - Events ++ [end_json], - start(JSON, {jsx, []}, [], #config{}) - ) - } || {Title, JSON, _, Events} <- Data + [{Title, ?_assertEqual(Events ++ [end_json], decode(JSON, []))} + || {Title, JSON, _, Events} <- Data ]. @@ -721,43 +719,35 @@ special_number_test_() -> [ {"-0", ?_assertEqual( [{integer, 0}, end_json], - start(<<"-0">>, {jsx, []}, [], #config{}) + decode(<<"-0">>, []) )}, {"-0.0", ?_assertEqual( [{float, 0.0}, end_json], - start(<<"-0.0">>, {jsx, []}, [], #config{}) + decode(<<"-0.0">>, []) )}, {"0e0", ?_assertEqual( [{float, 0.0}, end_json], - start(<<"0e0">>, {jsx, []}, [], #config{}) + decode(<<"0e0">>, []) )}, {"0e4", ?_assertEqual( [{float, 0.0}, end_json], - start(<<"0e4">>, {jsx, []}, [], #config{}) + decode(<<"0e4">>, []) )}, {"1e0", ?_assertEqual( [{float, 1.0}, end_json], - start(<<"1e0">>, {jsx, []}, [], #config{}) + decode(<<"1e0">>, []) )}, {"-1e0", ?_assertEqual( [{float, -1.0}, end_json], - start(<<"-1e0">>, {jsx, []}, [], #config{}) + decode(<<"-1e0">>, []) )}, {"1e4", ?_assertEqual( [{float, 1.0e4}, end_json], - start(<<"1e4">>, {jsx, []}, [], #config{}) + decode(<<"1e4">>, []) )} ]. -decode(JSON, Config) -> - try - start(JSON, {jsx, []}, [], jsx_utils:parse_config(Config)) - catch - error:badarg -> {error, badarg} - end. - - comments_test_() -> [ {"preceeding // comment", ?_assertEqual( @@ -1012,13 +1002,17 @@ to_fake_utf8(N) -> clean_string_test_() -> [ - {"clean codepoints test", ?_assertEqual( + {"clean codepoints", ?_assertEqual( [{string, codepoints()}, end_json], decode(<<34, (codepoints())/binary, 34>>, []) )}, - {"clean extended codepoints test", ?_assertEqual( + {"clean extended codepoints", ?_assertEqual( [{string, extended_codepoints()}, end_json], decode(<<34, (extended_codepoints())/binary, 34>>, []) + )}, + {"clean reserved space", ?_assertEqual( + lists:duplicate(length(reserved_space()), [{string, <<16#fffd/utf8>>}, end_json]), + lists:map(fun(Codepoint) -> decode(Codepoint, [replaced_bad_utf8]) end, reserved_space()) )} ]. From 5d40e559c98485249ce3ee63366a44be63fa4ebb Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sun, 24 Feb 2013 01:07:16 -0800 Subject: [PATCH 53/61] import old string parsing and fix parsing of single quoted strings --- src/jsx_decoder.erl | 596 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 458 insertions(+), 138 deletions(-) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index d1c30bc..d508887 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -158,9 +158,9 @@ definitely_bom(Bin, Handler, Stack, Config) -> value(<>, Handler, Stack, Config) -> - string(Rest, Handler, [string|Stack], Config); + string(Rest, Handler, [?new_seq()|Stack], Config); value(<>, Handler, Stack, Config = #config{single_quoted_strings=true}) -> - string(Rest, Handler, [single_quoted_string|Stack], Config); + string(Rest, Handler, [?new_seq(), single_quote|Stack], Config); value(<<$t, Rest/binary>>, Handler, Stack, Config) -> tr(Rest, Handler, Stack, Config); value(<<$f, Rest/binary>>, Handler, Stack, Config) -> @@ -188,9 +188,9 @@ value(Bin, Handler, Stack, Config) -> object(<>, Handler, Stack, Config) -> - string(Rest, Handler, [string|Stack], Config); + string(Rest, Handler, [?new_seq()|Stack], Config); object(<>, Handler, Stack, Config = #config{single_quoted_strings=true}) -> - string(Rest, Handler, [single_quoted_string|Stack], Config); + string(Rest, Handler, [?new_seq(), single_quote|Stack], Config); object(<>, Handler, [key|Stack], Config) -> maybe_done(Rest, handle_event(end_object, Handler, Config), Stack, Config); object(<>, Handler, Stack, Config) when ?is_whitespace(S) -> @@ -228,9 +228,9 @@ colon(Bin, Handler, Stack, Config) -> key(<>, Handler, Stack, Config) -> - string(Rest, Handler, [string|Stack], Config); + string(Rest, Handler, [?new_seq()|Stack], Config); key(<>, Handler, Stack, Config = #config{single_quoted_strings=true}) -> - string(Rest, Handler, [single_quoted_string|Stack], Config); + string(Rest, Handler, [?new_seq(), single_quote|Stack], Config); key(<>, Handler, Stack, Config) when ?is_whitespace(S) -> key(Rest, Handler, Stack, Config); key(<>, Handler, Stack, Config=#config{comments=true}) -> @@ -241,146 +241,466 @@ key(Bin, Handler, Stack, Config) -> ?error([Bin, Handler, Stack, Config]). -string(Bin, Handler, Stack, Config) -> string(Bin, Handler, Stack, Config, <<>>). +%% string appends it's output to the term at the top of the stack. for +%% efficiency the strings are build in reverse order and reversed before +%% being added to the output stream +%% when parsing strings, the naive detection of partial codepoints is +%% insufficient. this incredibly anal function should detect all badly formed +%% utf sequences +partial_utf(<<>>) -> true; +partial_utf(<>) when X >= 16#c2, X =< 16#f4 -> true; +partial_utf(<>) when X >= 16#e0, X =< 16#f4, Y >= 16#80, Y =< 16#bf -> true; +partial_utf(<>) + when X >= 16#f0, X =< 16#f4, + Y >= 16#80, Y =< 16#bf, + Z >= 16#80, Z =< 16#bf -> + true; +partial_utf(_) -> false. -string(Bin, Handler, [StringType|Stack], Config, Acc) -> - Length = cut(Bin), - <> = Bin, - case Rest of - <> when StringType == string -> - finish_string(Rem, Handler, Stack, Config, <>); - <> when StringType == single_quoted_string -> - finish_string(Rem, Handler, Stack, Config, <>); - <> when Codepoint == ?doublequote; Codepoint == ?singlequote -> - string(Rem, Handler, [StringType|Stack], Config, <>) + +%% explicitly whitelist ascii set for better efficiency (seriously, it's worth +%% almost a 20% increase) +string(<<32, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 32)|Stack], Config); +string(<<33, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 33)|Stack], Config); +string(<>, Handler, S, Config) -> + case S of + [Acc, key|Stack] -> + colon(Rest, handle_event({key, ?end_seq(Acc)}, Handler, Config), [key|Stack], Config); + [Acc, single_quote|Stack] -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace(?doublequote, Config)), single_quote|Stack], Config); + [Acc|Stack] -> + maybe_done(Rest, handle_event({string, ?end_seq(Acc)}, Handler, Config), Stack, Config) + end; +string(<<35, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 35)|Stack], Config); +string(<<36, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 36)|Stack], Config); +string(<<37, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 37)|Stack], Config); +string(<<38, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 38)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) -> + case Config#config.single_quoted_strings of + true -> + case Stack of + [single_quote, key|S] -> + colon(Rest, handle_event({key, ?end_seq(Acc)}, Handler, Config), [key|S], Config) + ; [single_quote|S] -> + maybe_done(Rest, handle_event({string, ?end_seq(Acc)}, Handler, Config), S, Config) + ; _ -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace(?singlequote, Config))|Stack], Config) + end + ; false -> + string(Rest, Handler, [?acc_seq(Acc, ?singlequote)|Stack], Config) + end; +string(<<40, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 40)|Stack], Config); +string(<<41, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 41)|Stack], Config); +string(<<42, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 42)|Stack], Config); +string(<<43, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 43)|Stack], Config); +string(<<44, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 44)|Stack], Config); +string(<<45, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 45)|Stack], Config); +string(<<46, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 46)|Stack], Config); +string(<<$/, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace($/, Config))|Stack], Config); +string(<<48, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 48)|Stack], Config); +string(<<49, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 49)|Stack], Config); +string(<<50, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 50)|Stack], Config); +string(<<51, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 51)|Stack], Config); +string(<<52, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 52)|Stack], Config); +string(<<53, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 53)|Stack], Config); +string(<<54, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 54)|Stack], Config); +string(<<55, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 55)|Stack], Config); +string(<<56, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 56)|Stack], Config); +string(<<57, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 57)|Stack], Config); +string(<<58, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 58)|Stack], Config); +string(<<59, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 59)|Stack], Config); +string(<<60, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 60)|Stack], Config); +string(<<61, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 61)|Stack], Config); +string(<<62, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 62)|Stack], Config); +string(<<63, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 63)|Stack], Config); +string(<<64, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 64)|Stack], Config); +string(<<65, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 65)|Stack], Config); +string(<<66, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 66)|Stack], Config); +string(<<67, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 67)|Stack], Config); +string(<<68, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 68)|Stack], Config); +string(<<69, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 69)|Stack], Config); +string(<<70, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 70)|Stack], Config); +string(<<71, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 71)|Stack], Config); +string(<<72, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 72)|Stack], Config); +string(<<73, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 73)|Stack], Config); +string(<<74, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 74)|Stack], Config); +string(<<75, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 75)|Stack], Config); +string(<<76, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 76)|Stack], Config); +string(<<77, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 77)|Stack], Config); +string(<<78, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 78)|Stack], Config); +string(<<79, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 79)|Stack], Config); +string(<<80, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 80)|Stack], Config); +string(<<81, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 81)|Stack], Config); +string(<<82, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 82)|Stack], Config); +string(<<83, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 83)|Stack], Config); +string(<<84, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 84)|Stack], Config); +string(<<85, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 85)|Stack], Config); +string(<<86, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 86)|Stack], Config); +string(<<87, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 87)|Stack], Config); +string(<<88, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 88)|Stack], Config); +string(<<89, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 89)|Stack], Config); +string(<<90, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 90)|Stack], Config); +string(<<91, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 91)|Stack], Config); +string(<>, Handler, Stack, Config) -> + escape(Rest, Handler, Stack, Config); +string(<<93, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 93)|Stack], Config); +string(<<94, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 94)|Stack], Config); +string(<<95, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 95)|Stack], Config); +string(<<96, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 96)|Stack], Config); +string(<<97, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 97)|Stack], Config); +string(<<98, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 98)|Stack], Config); +string(<<99, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 99)|Stack], Config); +string(<<100, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 100)|Stack], Config); +string(<<101, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 101)|Stack], Config); +string(<<102, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 102)|Stack], Config); +string(<<103, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 103)|Stack], Config); +string(<<104, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 104)|Stack], Config); +string(<<105, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 105)|Stack], Config); +string(<<106, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 106)|Stack], Config); +string(<<107, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 107)|Stack], Config); +string(<<108, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 108)|Stack], Config); +string(<<109, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 109)|Stack], Config); +string(<<110, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 110)|Stack], Config); +string(<<111, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 111)|Stack], Config); +string(<<112, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 112)|Stack], Config); +string(<<113, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 113)|Stack], Config); +string(<<114, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 114)|Stack], Config); +string(<<115, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 115)|Stack], Config); +string(<<116, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 116)|Stack], Config); +string(<<117, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 117)|Stack], Config); +string(<<118, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 118)|Stack], Config); +string(<<119, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 119)|Stack], Config); +string(<<120, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 120)|Stack], Config); +string(<<121, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 121)|Stack], Config); +string(<<122, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 122)|Stack], Config); +string(<<123, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 123)|Stack], Config); +string(<<124, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 124)|Stack], Config); +string(<<125, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 125)|Stack], Config); +string(<<126, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 126)|Stack], Config); +string(<<127, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 127)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#20, X < 16#2028 -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X == 16#2028; X == 16#2029 -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace(X, Config))|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X > 16#2029, X < 16#d800 -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X > 16#dfff, X < 16#fdd0 -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X > 16#fdef, X < 16#fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#10000, X < 16#1fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#20000, X < 16#2fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#30000, X < 16#3fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#40000, X < 16#4fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#50000, X < 16#5fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#60000, X < 16#6fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#70000, X < 16#7fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#80000, X < 16#8fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#90000, X < 16#9fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#a0000, X < 16#afffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#b0000, X < 16#bfffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#c0000, X < 16#cfffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#d0000, X < 16#dfffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#e0000, X < 16#efffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#f0000, X < 16#ffffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) when X >= 16#100000, X < 16#10fffe -> + string(Rest, Handler, [?acc_seq(Acc, X)|Stack], Config); +string(<>, Handler, [Acc|Stack], Config) -> + case Config#config.replaced_bad_utf8 of + true -> noncharacter(<>, Handler, [Acc|Stack], Config) + ; false -> ?error([<>, Handler, [Acc|Stack], Config]) + end; +string(Bin, Handler, Stack, Config) -> + case partial_utf(Bin) of + true -> ?incomplete(string, Bin, Handler, Stack, Config) + ; false -> + case Config#config.replaced_bad_utf8 of + true -> noncharacter(Bin, Handler, Stack, Config) + ; false -> ?error([Bin, Handler, Stack, Config]) + end end. -finish_string(Rest, Handler, [key|_] = Stack, Config, Acc) -> - State = handle_event({key, Acc}, Handler, Config), - colon(Rest, State, Stack, Config); -finish_string(Rest, Handler, Stack, Config, Acc) -> - State = handle_event({string, Acc}, Handler, Config), - maybe_done(Rest, State, Stack, Config). +%% we don't need to guard against partial utf here, because it's already taken +%% care of in string +%% surrogates +noncharacter(<<237, X, _, Rest/binary>>, Handler, [Acc|Stack], Config) when X >= 160 -> + string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config); +%% u+fffe and u+ffff for R14BXX +noncharacter(<<239, 191, X, Rest/binary>>, Handler, [Acc|Stack], Config) when X == 190; X == 191 -> + string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config); +%% u+xfffe, u+xffff and other noncharacters +noncharacter(<<_/utf8, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config); +%% overlong encodings and missing continuations of a 2 byte sequence +noncharacter(<>, Handler, Stack, Config) when X >= 192, X =< 223 -> + strip_continuations(Rest, Handler, [1|Stack], Config); +%% overlong encodings and missing continuations of a 3 byte sequence +noncharacter(<>, Handler, Stack, Config) when X >= 224, X =< 239 -> + strip_continuations(Rest, Handler, [2|Stack], Config); +%% overlong encodings and missing continuations of a 4 byte sequence +noncharacter(<>, Handler, Stack, Config) when X >= 240, X =< 247 -> + strip_continuations(Rest, Handler, [3|Stack], Config); +%% unexpected bytes, including orphan continuations +noncharacter(<<_, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config); +noncharacter(<<>>, Handler, Stack, Config) -> + ?incomplete(noncharacter, <<>>, Handler, Stack, Config). -cut(Bin) -> cut(Bin, 0). +%% strips continuation bytes after bad utf bytes, guards against both too short +%% and overlong sequences. N is the maximum number of bytes to strip +strip_continuations(Rest, Handler, [0, Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config); +strip_continuations(<>, Handler, [N|Stack], Config) when X >= 128, X =< 191 -> + strip_continuations(Rest, Handler, [N - 1|Stack], Config); +%% incomplete +strip_continuations(<<>>, Handler, Stack, Config) -> + ?incomplete(strip_continuations, <<>>, Handler, Stack, Config); +%% not a continuation byte, dispatch back to string +strip_continuations(Rest, Handler, [_, Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config). -cut(<<32, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<33, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<35, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<36, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<37, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<38, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<40, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<41, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<42, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<43, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<44, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<45, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<46, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<48, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<49, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<50, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<51, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<52, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<53, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<54, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<55, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<56, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<57, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<58, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<59, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<60, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<61, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<62, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<63, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<64, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<65, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<66, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<67, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<68, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<69, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<70, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<71, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<72, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<73, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<74, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<75, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<76, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<77, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<78, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<79, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<80, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<81, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<82, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<83, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<84, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<85, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<86, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<87, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<88, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<89, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<90, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<91, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<93, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<94, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<95, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<96, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<97, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<98, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<99, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<100, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<101, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<102, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<103, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<104, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<105, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<106, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<107, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<108, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<109, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<110, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<111, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<112, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<113, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<114, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<115, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<116, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<117, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<118, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<119, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<120, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<121, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<122, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<123, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<124, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<125, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<126, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<<127, Rest/binary>>, N) -> cut(Rest, N + 1); -cut(<>, N) when X >= 128, X < 16#0800 -> cut(Rest, N + 2); -cut(<>, N) when X >= 16#0800, X < 16#2028 -> cut(Rest, N + 3); -cut(<>, N) when X >= 16#202a, X < 16#d800 -> cut(Rest, N + 3); -cut(<>, N) when X > 16#dfff, X < 16#fdd0 -> cut(Rest, N + 3); -cut(<>, N) when X > 16#fdef, X < 16#fffe -> cut(Rest, N + 3); -cut(<>, N) when X >= 16#10000, X < 16#1fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#20000, X < 16#2fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#30000, X < 16#3fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#40000, X < 16#4fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#50000, X < 16#5fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#60000, X < 16#6fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#70000, X < 16#7fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#80000, X < 16#8fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#90000, X < 16#9fffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#a0000, X < 16#afffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#b0000, X < 16#bfffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#c0000, X < 16#cfffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#d0000, X < 16#dfffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#e0000, X < 16#efffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#f0000, X < 16#ffffe -> cut(Rest, N + 4); -cut(<>, N) when X >= 16#100000, X < 16#10fffe -> cut(Rest, N + 4); -cut(_, N) -> N. +escape(<<$b, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\b, Config))|Stack], Config); +escape(<<$f, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\f, Config))|Stack], Config); +escape(<<$n, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\n, Config))|Stack], Config); +escape(<<$r, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\r, Config))|Stack], Config); +escape(<<$t, Rest/binary>>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\t, Config))|Stack], Config); +escape(<>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\\, Config))|Stack], Config); +escape(<>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace($/, Config))|Stack], Config); +escape(<>, Handler, [Acc|Stack], Config) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace($\", Config))|Stack], Config); +escape(<>, Handler, [Acc|Stack], Config = #config{single_quoted_strings=true}) -> + string(Rest, Handler, [?acc_seq(Acc, maybe_replace(?singlequote, Config))|Stack], Config); +escape(<<$u, Rest/binary>>, Handler, Stack, Config) -> + escaped_unicode(Rest, Handler, Stack, Config); +escape(<<>>, Handler, Stack, Config) -> + ?incomplete(escape, <<>>, Handler, Stack, Config); +escape(Bin, Handler, [Acc|Stack], Config=#config{ignored_bad_escapes=true}) -> + string(Bin, Handler, [?acc_seq(Acc, ?rsolidus)|Stack], Config); +escape(Bin, Handler, Stack, Config) -> + ?error([Bin, Handler, Stack, Config]). + + +%% this code is ugly and unfortunate, but so is json's handling of escaped +%% unicode codepoint sequences. +escaped_unicode(<>, Handler, [Acc|Stack], Config) + when ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D) -> + case erlang:list_to_integer([A, B, C, D], 16) of + %% high surrogate, dispatch to low surrogate + X when X >= 16#d800, X =< 16#dbff -> + low_surrogate(Rest, Handler, [X, Acc|Stack], Config) + %% low surrogate, illegal in this position + ; X when X >= 16#dc00, X =< 16#dfff -> + case Config#config.replaced_bad_utf8 of + true -> string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config) + ; false -> ?error([<>, Handler, [Acc|Stack], Config]) + end + %% anything else + ; X -> string(Rest, Handler, [?acc_seq(Acc, maybe_replace(X, Config))|Stack], Config) + end; +escaped_unicode(Bin, Handler, Stack, Config) -> + case is_partial_escape(Bin) of + true -> ?incomplete(escaped_unicode, Bin, Handler, Stack, Config) + ; false -> ?error([Bin, Handler, Stack, Config]) + end. + + +is_partial_escape(<>) when ?is_hex(A), ?is_hex(B), ?is_hex(C) -> true; +is_partial_escape(<>) when ?is_hex(A), ?is_hex(B) -> true; +is_partial_escape(<>) when ?is_hex(A) -> true; +is_partial_escape(<<>>) -> true; +is_partial_escape(_) -> false. + + +low_surrogate(<>, Handler, [High, Acc|Stack], Config) + when ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D) -> + case erlang:list_to_integer([A, B, C, D], 16) of + X when X >= 16#dc00, X =< 16#dfff -> + Y = surrogate_to_codepoint(High, X), + case (Y =< 16#d800 orelse Y >= 16#e000) of + true -> string(Rest, Handler, [?acc_seq(Acc, Y)|Stack], Config) + ; false -> + case Config#config.replaced_bad_utf8 of + true -> + string(Rest, Handler, [?acc_seq(Acc, 16#fffd, 16#fffd)|Stack], Config) + ; false -> + ?error([<>, Handler, [High, Acc|Stack], Config]) + end + end + ; _ -> + case Config#config.replaced_bad_utf8 of + true -> string(Rest, Handler, [?acc_seq(Acc, 16#fffd, 16#fffd)|Stack], Config) + ; false -> ?error([<>, Handler, [High, Acc|Stack], Config]) + end + end; +low_surrogate(Bin, Handler, [High, Acc|Stack], Config) -> + case is_partial_low(Bin) of + true -> ?incomplete(low_surrogate, Bin, Handler, [High, Acc|Stack], Config) + ; false -> + case Config#config.replaced_bad_utf8 of + true -> string(Bin, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Config) + ; false -> ?error([Bin, Handler, [High, Acc|Stack], Config]) + end + end. + + +is_partial_low(<>) when ?is_hex(A), ?is_hex(B), ?is_hex(C) -> true; +is_partial_low(<>) when ?is_hex(A), ?is_hex(B) -> true; +is_partial_low(<>) when ?is_hex(A) -> true; +is_partial_low(<>) -> true; +is_partial_low(<>) -> true; +is_partial_low(<<>>) -> true; +is_partial_low(_) -> false. + + +%% stole this from the unicode spec +surrogate_to_codepoint(High, Low) -> + (High - 16#d800) * 16#400 + (Low - 16#dc00) + 16#10000. + + +maybe_replace(X, #config{dirty_strings=true}) when is_integer(X) -> [X]; +maybe_replace($\b, #config{escaped_strings=true}) -> [$\\, $b]; +maybe_replace($\t, #config{escaped_strings=true}) -> [$\\, $t]; +maybe_replace($\n, #config{escaped_strings=true}) -> [$\\, $n]; +maybe_replace($\f, #config{escaped_strings=true}) -> [$\\, $f]; +maybe_replace($\r, #config{escaped_strings=true}) -> [$\\, $r]; +maybe_replace($\", #config{escaped_strings=true}) -> [$\\, $\"]; +maybe_replace($', Config=#config{escaped_strings=true}) -> + case Config#config.single_quoted_strings of + true -> [$\\, $'] + ; false -> [$'] + end; +maybe_replace($/, Config=#config{escaped_strings=true}) -> + case Config#config.escaped_forward_slashes of + true -> [$\\, $/] + ; false -> [$/] + end; +maybe_replace($\\, #config{escaped_strings=true}) -> [$\\, $\\]; +maybe_replace(X, Config=#config{escaped_strings=true}) when X == 16#2028; X == 16#2029 -> + case Config#config.unescaped_jsonp of + true -> [X] + ; false -> jsx_utils:json_escape_sequence(X) + end; +maybe_replace(X, #config{escaped_strings=true}) when X < 32 -> + jsx_utils:json_escape_sequence(X); +maybe_replace(X, _Config) -> [X]. %% like strings, numbers are collected in an intermediate accumulator before From 7b31bef0f630a09df378de20dd8399b8d0db481b Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sun, 24 Feb 2013 01:32:17 -0800 Subject: [PATCH 54/61] more comprehensive string checking --- src/jsx_decoder.erl | 35 ++++++++++++- src/jsx_utils.erl | 116 ++++++++++++++++++-------------------------- 2 files changed, 80 insertions(+), 71 deletions(-) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index d508887..b8a3ff9 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -1023,7 +1023,12 @@ done(Bin, Handler, Stack, Config) -> ?error([Bin, Handler, Stack, Config]). -include_lib("eunit/include/eunit.hrl"). -decode(JSON, Config) -> start(JSON, {jsx, []}, [], jsx_utils:parse_config(Config)). +decode(JSON, Config) -> + try + start(JSON, {jsx, []}, [], jsx_utils:parse_config(Config)) + catch + error:badarg -> {error, badarg} + end. decode_test_() -> @@ -1330,9 +1335,37 @@ clean_string_test_() -> [{string, extended_codepoints()}, end_json], decode(<<34, (extended_codepoints())/binary, 34>>, []) )}, + {"error reserved space", ?_assertEqual( + lists:duplicate(length(reserved_space()), {error, badarg}), + lists:map(fun(Codepoint) -> decode(Codepoint, []) end, reserved_space()) + )}, + {"error surrogates", ?_assertEqual( + lists:duplicate(length(surrogates()), {error, badarg}), + lists:map(fun(Codepoint) -> decode(Codepoint, []) end, surrogates()) + )}, + {"error noncharacters", ?_assertEqual( + lists:duplicate(length(noncharacters()), {error, badarg}), + lists:map(fun(Codepoint) -> decode(Codepoint, []) end, noncharacters()) + )}, + {"error extended noncharacters", ?_assertEqual( + lists:duplicate(length(extended_noncharacters()), {error, badarg}), + lists:map(fun(Codepoint) -> decode(Codepoint, []) end, extended_noncharacters()) + )}, {"clean reserved space", ?_assertEqual( lists:duplicate(length(reserved_space()), [{string, <<16#fffd/utf8>>}, end_json]), lists:map(fun(Codepoint) -> decode(Codepoint, [replaced_bad_utf8]) end, reserved_space()) + )}, + {"clean surrogates", ?_assertEqual( + lists:duplicate(length(surrogates()), [{string, <<16#fffd/utf8>>}, end_json]), + lists:map(fun(Codepoint) -> decode(Codepoint, [replaced_bad_utf8]) end, surrogates()) + )}, + {"clean noncharacters", ?_assertEqual( + lists:duplicate(length(noncharacters()), [{string, <<16#fffd/utf8>>}, end_json]), + lists:map(fun(Codepoint) -> decode(Codepoint, [replaced_bad_utf8]) end, noncharacters()) + )}, + {"clean extended noncharacters", ?_assertEqual( + lists:duplicate(length(extended_noncharacters()), [{string, <<16#fffd/utf8>>}, end_json]), + lists:map(fun(Codepoint) -> decode(Codepoint, [replaced_bad_utf8]) end, extended_noncharacters()) )} ]. diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index 505a3b1..5556643 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -670,80 +670,56 @@ extended_noncharacters() -> ]. +decode(String, Config) -> + try + [{string, clean_string(String, jsx_utils:parse_config(Config))}, end_json] + catch + error:badarg -> {error, badarg} + end. + + clean_string_test_() -> [ - {"clean codepoints test", ?_assertEqual( - codepoints(), - clean_string(codepoints(), #config{}) + {"clean codepoints", ?_assertEqual( + [{string, codepoints()}, end_json], + decode(codepoints(), []) )}, - {"clean extended codepoints test", ?_assertEqual( - extended_codepoints(), - clean_string(extended_codepoints(), #config{}) + {"clean extended codepoints", ?_assertEqual( + [{string, extended_codepoints()}, end_json], + decode(extended_codepoints(), []) + )}, + {"error reserved space", ?_assertEqual( + lists:duplicate(length(reserved_space()), {error, badarg}), + lists:map(fun(Codepoint) -> decode(Codepoint, []) end, reserved_space()) + )}, + {"error surrogates", ?_assertEqual( + lists:duplicate(length(surrogates()), {error, badarg}), + lists:map(fun(Codepoint) -> decode(Codepoint, []) end, surrogates()) + )}, + {"error noncharacters", ?_assertEqual( + lists:duplicate(length(noncharacters()), {error, badarg}), + lists:map(fun(Codepoint) -> decode(Codepoint, []) end, noncharacters()) + )}, + {"error extended noncharacters", ?_assertEqual( + lists:duplicate(length(extended_noncharacters()), {error, badarg}), + lists:map(fun(Codepoint) -> decode(Codepoint, []) end, extended_noncharacters()) + )}, + {"clean reserved space", ?_assertEqual( + lists:duplicate(length(reserved_space()), [{string, <<16#fffd/utf8>>}, end_json]), + lists:map(fun(Codepoint) -> decode(Codepoint, [replaced_bad_utf8]) end, reserved_space()) + )}, + {"clean surrogates", ?_assertEqual( + lists:duplicate(length(surrogates()), [{string, <<16#fffd/utf8>>}, end_json]), + lists:map(fun(Codepoint) -> decode(Codepoint, [replaced_bad_utf8]) end, surrogates()) + )}, + {"clean noncharacters", ?_assertEqual( + lists:duplicate(length(noncharacters()), [{string, <<16#fffd/utf8>>}, end_json]), + lists:map(fun(Codepoint) -> decode(Codepoint, [replaced_bad_utf8]) end, noncharacters()) + )}, + {"clean extended noncharacters", ?_assertEqual( + lists:duplicate(length(extended_noncharacters()), [{string, <<16#fffd/utf8>>}, end_json]), + lists:map(fun(Codepoint) -> decode(Codepoint, [replaced_bad_utf8]) end, extended_noncharacters()) )} - ] ++ [ - { - "reserved character: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), - ?_assertError( - badarg, - clean_string(Codepoint, #config{}) - ) - } || Codepoint <- reserved_space() - ] ++ [ - { - "reserved character: " ++ lists:flatten(io_lib:format("~p", [Codepoint])) ++ " (replaced)", - ?_assertEqual( - <<16#fffd/utf8>>, - clean_string(Codepoint, #config{replaced_bad_utf8=true}) - ) - } || Codepoint <- reserved_space() - ] ++ [ - { - "surrogate: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), - ?_assertError( - badarg, - clean_string(Codepoint, #config{}) - ) - } || Codepoint <- surrogates() - ] ++ [ - { - "surrogate: " ++ lists:flatten(io_lib:format("~p", [Codepoint])) ++ " (replaced)", - ?_assertEqual( - <<16#fffd/utf8>>, - clean_string(Codepoint, #config{replaced_bad_utf8=true}) - ) - } || Codepoint <- surrogates() - ] ++ [ - { - "noncharacter: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), - ?_assertError( - badarg, - clean_string(Codepoint, #config{}) - ) - } || Codepoint <- noncharacters() - ] ++ [ - { - "noncharacter: " ++ lists:flatten(io_lib:format("~p", [Codepoint])) ++ " (replaced)", - ?_assertEqual( - <<16#fffd/utf8>>, - clean_string(Codepoint, #config{replaced_bad_utf8=true}) - ) - } || Codepoint <- noncharacters() - ] ++ [ - { - "extended noncharacter: " ++ lists:flatten(io_lib:format("~p", [Codepoint])), - ?_assertError( - badarg, - clean_string(Codepoint, #config{}) - ) - } || Codepoint <- extended_noncharacters() - ] ++ [ - { - "extended noncharacter: " ++ lists:flatten(io_lib:format("~p", [Codepoint])) ++ " (replaced)", - ?_assertEqual( - <<16#fffd/utf8>>, - clean_string(Codepoint, #config{replaced_bad_utf8=true}) - ) - } || Codepoint <- extended_noncharacters() ]. From 00b593c6ea9c61474988bdc5129e4b2e91e6d812 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sun, 24 Feb 2013 01:50:17 -0800 Subject: [PATCH 55/61] add escape tests to decoder --- src/jsx_decoder.erl | 173 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index b8a3ff9..a018d1d 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -1370,6 +1370,179 @@ clean_string_test_() -> ]. +maybe_escape(Bin, Config) -> + [{string, String}, end_json] = decode(Bin, Config), + String. + +escape_test_() -> + [ + {"maybe_escape backspace", ?_assertEqual( + <<"\\b">>, + maybe_escape(<<34, "\\b"/utf8, 34>>, [escaped_strings]) + )}, + {"don't escape backspace", ?_assertEqual( + <<"\b">>, + maybe_escape(<<34, "\\b"/utf8, 34>>, []) + )}, + {"maybe_escape tab", ?_assertEqual( + <<"\\t">>, + maybe_escape(<<34, "\\t"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape newline", ?_assertEqual( + <<"\\n">>, + maybe_escape(<<34, "\\n"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape formfeed", ?_assertEqual( + <<"\\f">>, + maybe_escape(<<34, "\\f"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape carriage return", ?_assertEqual( + <<"\\r">>, + maybe_escape(<<34, "\\r"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape quote", ?_assertEqual( + <<"\\\"">>, + maybe_escape(<<34, "\\\""/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape forward slash", ?_assertEqual( + <<"\\/">>, + maybe_escape(<<34, "/"/utf8, 34>>, [escaped_strings, escaped_forward_slashes]) + )}, + {"do not maybe_escape forward slash", ?_assertEqual( + <<"/">>, + maybe_escape(<<34, "/"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape backslash", ?_assertEqual( + <<"\\\\">>, + maybe_escape(<<34, "\\\\"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape jsonp (u2028)", ?_assertEqual( + <<"\\u2028">>, + maybe_escape(<<34, 16#2028/utf8, 34>>, [escaped_strings]) + )}, + {"do not maybe_escape jsonp (u2028)", ?_assertEqual( + <<16#2028/utf8>>, + maybe_escape(<<34, 16#2028/utf8, 34>>, [escaped_strings, unescaped_jsonp]) + )}, + {"maybe_escape jsonp (u2029)", ?_assertEqual( + <<"\\u2029">>, + maybe_escape(<<34, 16#2029/utf8, 34>>, [escaped_strings]) + )}, + {"do not maybe_escape jsonp (u2029)", ?_assertEqual( + <<16#2029/utf8>>, + maybe_escape(<<34, 16#2029/utf8, 34>>, [escaped_strings, unescaped_jsonp]) + )}, + {"maybe_escape u0000", ?_assertEqual( + <<"\\u0000">>, + maybe_escape(<<34, "\\u0000"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u0001", ?_assertEqual( + <<"\\u0001">>, + maybe_escape(<<34, "\\u0001"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u0002", ?_assertEqual( + <<"\\u0002">>, + maybe_escape(<<34, "\\u0002"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u0003", ?_assertEqual( + <<"\\u0003">>, + maybe_escape(<<34, "\\u0003"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u0004", ?_assertEqual( + <<"\\u0004">>, + maybe_escape(<<34, "\\u0004"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u0005", ?_assertEqual( + <<"\\u0005">>, + maybe_escape(<<34, "\\u0005"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u0006", ?_assertEqual( + <<"\\u0006">>, + maybe_escape(<<34, "\\u0006"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u0007", ?_assertEqual( + <<"\\u0007">>, + maybe_escape(<<34, "\\u0007"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u000b", ?_assertEqual( + <<"\\u000b">>, + maybe_escape(<<34, "\\u000b"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u000e", ?_assertEqual( + <<"\\u000e">>, + maybe_escape(<<34, "\\u000e"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u000f", ?_assertEqual( + <<"\\u000f">>, + maybe_escape(<<34, "\\u000f"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u0010", ?_assertEqual( + <<"\\u0010">>, + maybe_escape(<<34, "\\u0010"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u0011", ?_assertEqual( + <<"\\u0011">>, + maybe_escape(<<34, "\\u0011"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u0012", ?_assertEqual( + <<"\\u0012">>, + maybe_escape(<<34, "\\u0012"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u0013", ?_assertEqual( + <<"\\u0013">>, + maybe_escape(<<34, "\\u0013"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u0014", ?_assertEqual( + <<"\\u0014">>, + maybe_escape(<<34, "\\u0014"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u0015", ?_assertEqual( + <<"\\u0015">>, + maybe_escape(<<34, "\\u0015"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u0016", ?_assertEqual( + <<"\\u0016">>, + maybe_escape(<<34, "\\u0016"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u0017", ?_assertEqual( + <<"\\u0017">>, + maybe_escape(<<34, "\\u0017"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u0018", ?_assertEqual( + <<"\\u0018">>, + maybe_escape(<<34, "\\u0018"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u0019", ?_assertEqual( + <<"\\u0019">>, + maybe_escape(<<34, "\\u0019"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u001a", ?_assertEqual( + <<"\\u001a">>, + maybe_escape(<<34, "\\u001a"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u001b", ?_assertEqual( + <<"\\u001b">>, + maybe_escape(<<34, "\\u001b"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u001c", ?_assertEqual( + <<"\\u001c">>, + maybe_escape(<<34, "\\u001c"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u001d", ?_assertEqual( + <<"\\u001d">>, + maybe_escape(<<34, "\\u001d"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u001e", ?_assertEqual( + <<"\\u001e">>, + maybe_escape(<<34, "\\u001e"/utf8, 34>>, [escaped_strings]) + )}, + {"maybe_escape u001f", ?_assertEqual( + <<"\\u001f">>, + maybe_escape(<<34, "\\u001f"/utf8, 34>>, [escaped_strings]) + )} + ]. + + single_quoted_string_test_() -> [ {"single quoted string", ?_assertEqual( From 57e10dfca6dbc1208c595924c1896eccd00568c7 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sun, 24 Feb 2013 01:59:54 -0800 Subject: [PATCH 56/61] additional decoder tests --- src/jsx_decoder.erl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index a018d1d..38ce372 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -1560,4 +1560,14 @@ single_quoted_string_test_() -> ]. +ignored_bad_escapes_test_() -> + [ + {"ignore unrecognized escape sequence", ?_assertEqual( + [{string, <<"\\x25">>}, end_json], + decode(<<"\"\\x25\"">>, [ignored_bad_escapes]) + )} + ]. + + + -endif. \ No newline at end of file From fd9856341f642aef868535060a9ae417c150ba1c Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sun, 24 Feb 2013 02:14:47 -0800 Subject: [PATCH 57/61] bad utf tests for decoder --- src/jsx_decoder.erl | 196 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 38ce372..53b7b1a 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -1370,6 +1370,202 @@ clean_string_test_() -> ]. +clean_string(String, Config) -> + [{string, S}, end_json] = start(<<34, String/binary, 34>>, {jsx, []}, [], Config), + S. + +bad_utf8_test_() -> + [ + {"noncharacter u+fffe", ?_assertError( + badarg, + clean_string(<<239, 191, 190>>, #config{}) + )}, + {"noncharacter u+fffe replaced", ?_assertEqual( + <<16#fffd/utf8>>, + clean_string(<<239, 191, 190>>, #config{replaced_bad_utf8=true}) + )}, + {"noncharacter u+ffff", ?_assertError( + badarg, + clean_string(<<239, 191, 191>>, #config{}) + )}, + {"noncharacter u+ffff replaced", ?_assertEqual( + <<16#fffd/utf8>>, + clean_string(<<239, 191, 191>>, #config{replaced_bad_utf8=true}) + )}, + {"orphan continuation byte u+0080", ?_assertError( + badarg, + clean_string(<<16#0080>>, #config{}) + )}, + {"orphan continuation byte u+0080 replaced", ?_assertEqual( + <<16#fffd/utf8>>, + clean_string(<<16#0080>>, #config{replaced_bad_utf8=true}) + )}, + {"orphan continuation byte u+00bf", ?_assertError( + badarg, + clean_string(<<16#00bf>>, #config{}) + )}, + {"orphan continuation byte u+00bf replaced", ?_assertEqual( + <<16#fffd/utf8>>, + clean_string(<<16#00bf>>, #config{replaced_bad_utf8=true}) + )}, + {"2 continuation bytes", ?_assertError( + badarg, + clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #config{}) + )}, + {"2 continuation bytes replaced", ?_assertEqual( + binary:copy(<<16#fffd/utf8>>, 2), + clean_string(<<(binary:copy(<<16#0080>>, 2))/binary>>, #config{replaced_bad_utf8=true}) + )}, + {"3 continuation bytes", ?_assertError( + badarg, + clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{}) + )}, + {"3 continuation bytes replaced", ?_assertEqual( + binary:copy(<<16#fffd/utf8>>, 3), + clean_string(<<(binary:copy(<<16#0080>>, 3))/binary>>, #config{replaced_bad_utf8=true}) + )}, + {"4 continuation bytes", ?_assertError( + badarg, + clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{}) + )}, + {"4 continuation bytes replaced", ?_assertEqual( + binary:copy(<<16#fffd/utf8>>, 4), + clean_string(<<(binary:copy(<<16#0080>>, 4))/binary>>, #config{replaced_bad_utf8=true}) + )}, + {"5 continuation bytes", ?_assertError( + badarg, + clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{}) + )}, + {"5 continuation bytes replaced", ?_assertEqual( + binary:copy(<<16#fffd/utf8>>, 5), + clean_string(<<(binary:copy(<<16#0080>>, 5))/binary>>, #config{replaced_bad_utf8=true}) + )}, + {"6 continuation bytes", ?_assertError( + badarg, + clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{}) + )}, + {"6 continuation bytes replaced", ?_assertEqual( + binary:copy(<<16#fffd/utf8>>, 6), + clean_string(<<(binary:copy(<<16#0080>>, 6))/binary>>, #config{replaced_bad_utf8=true}) + )}, + {"all continuation bytes", ?_assertError( + badarg, + clean_string(<<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, #config{}) + )}, + {"all continuation bytes replaced", ?_assertEqual( + binary:copy(<<16#fffd/utf8>>, length(lists:seq(16#0080, 16#00bf))), + clean_string( + <<(list_to_binary(lists:seq(16#0080, 16#00bf)))/binary>>, + #config{replaced_bad_utf8=true} + ) + )}, + {"lonely start byte", ?_assertError( + badarg, + clean_string(<<16#00c0>>, #config{}) + )}, + {"lonely start byte replaced", ?_assertEqual( + <<16#fffd/utf8>>, + clean_string(<<16#00c0>>, #config{replaced_bad_utf8=true}) + )}, + {"lonely start bytes (2 byte)", ?_assertError( + badarg, + clean_string(<<16#00c0, 32, 16#00df>>, #config{}) + )}, + {"lonely start bytes (2 byte) replaced", ?_assertEqual( + <<16#fffd/utf8, 32, 16#fffd/utf8>>, + clean_string(<<16#00c0, 32, 16#00df>>, #config{replaced_bad_utf8=true}) + )}, + {"lonely start bytes (3 byte)", ?_assertError( + badarg, + clean_string(<<16#00e0, 32, 16#00ef>>, #config{}) + )}, + {"lonely start bytes (3 byte) replaced", ?_assertEqual( + <<16#fffd/utf8, 32, 16#fffd/utf8>>, + clean_string(<<16#00e0, 32, 16#00ef>>, #config{replaced_bad_utf8=true}) + )}, + {"lonely start bytes (4 byte)", ?_assertError( + badarg, + clean_string(<<16#00f0, 32, 16#00f7>>, #config{}) + )}, + {"lonely start bytes (4 byte) replaced", ?_assertEqual( + <<16#fffd/utf8, 32, 16#fffd/utf8>>, + clean_string(<<16#00f0, 32, 16#00f7>>, #config{replaced_bad_utf8=true}) + )}, + {"missing continuation byte (3 byte)", ?_assertError( + badarg, + clean_string(<<224, 160, 32>>, #config{}) + )}, + {"missing continuation byte (3 byte) replaced", ?_assertEqual( + <<16#fffd/utf8, 32>>, + clean_string(<<224, 160, 32>>, #config{replaced_bad_utf8=true}) + )}, + {"missing continuation byte (4 byte missing one)", ?_assertError( + badarg, + clean_string(<<240, 144, 128, 32>>, #config{}) + )}, + {"missing continuation byte (4 byte missing one) replaced", ?_assertEqual( + <<16#fffd/utf8, 32>>, + clean_string(<<240, 144, 128, 32>>, #config{replaced_bad_utf8=true}) + )}, + {"missing continuation byte (4 byte missing two)", ?_assertError( + badarg, + clean_string(<<240, 144, 32>>, #config{}) + )}, + {"missing continuation byte (4 byte missing two) replaced", ?_assertEqual( + <<16#fffd/utf8, 32>>, + clean_string(<<240, 144, 32>>, #config{replaced_bad_utf8=true}) + )}, + {"overlong encoding of u+002f (2 byte)", ?_assertError( + badarg, + clean_string(<<16#c0, 16#af, 32>>, #config{}) + )}, + {"overlong encoding of u+002f (2 byte) replaced", ?_assertEqual( + <<16#fffd/utf8, 32>>, + clean_string(<<16#c0, 16#af, 32>>, #config{replaced_bad_utf8=true}) + )}, + {"overlong encoding of u+002f (3 byte)", ?_assertError( + badarg, + clean_string(<<16#e0, 16#80, 16#af, 32>>, #config{}) + )}, + {"overlong encoding of u+002f (3 byte) replaced", ?_assertEqual( + <<16#fffd/utf8, 32>>, + clean_string(<<16#e0, 16#80, 16#af, 32>>, #config{replaced_bad_utf8=true}) + )}, + {"overlong encoding of u+002f (4 byte)", ?_assertError( + badarg, + clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #config{}) + )}, + {"overlong encoding of u+002f (4 byte) replaced", ?_assertEqual( + <<16#fffd/utf8, 32>>, + clean_string(<<16#f0, 16#80, 16#80, 16#af, 32>>, #config{replaced_bad_utf8=true}) + )}, + {"highest overlong 2 byte sequence", ?_assertError( + badarg, + clean_string(<<16#c1, 16#bf, 32>>, #config{}) + )}, + {"highest overlong 2 byte sequence replaced", ?_assertEqual( + <<16#fffd/utf8, 32>>, + clean_string(<<16#c1, 16#bf, 32>>, #config{replaced_bad_utf8=true}) + )}, + {"highest overlong 3 byte sequence", ?_assertError( + badarg, + clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #config{}) + )}, + {"highest overlong 3 byte sequence replaced", ?_assertEqual( + <<16#fffd/utf8, 32>>, + clean_string(<<16#e0, 16#9f, 16#bf, 32>>, #config{replaced_bad_utf8=true}) + )}, + {"highest overlong 4 byte sequence", ?_assertError( + badarg, + clean_string(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, #config{}) + )}, + {"highest overlong 4 byte sequence replaced", ?_assertEqual( + <<16#fffd/utf8, 32>>, + clean_string(<<16#f0, 16#8f, 16#bf, 16#bf, 32>>, #config{replaced_bad_utf8=true}) + )} + ]. + + maybe_escape(Bin, Config) -> [{string, String}, end_json] = decode(Bin, Config), String. From 82b7044d42c3369a3adaec85be6f2086e328906e Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sun, 24 Feb 2013 02:28:33 -0800 Subject: [PATCH 58/61] unescape tests for decoder --- src/jsx_decoder.erl | 53 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 53b7b1a..124b205 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -1566,6 +1566,59 @@ bad_utf8_test_() -> ]. +unescape(Bin, Config) -> + [{string, String}, end_json] = decode(<<34, Bin/binary, 34>>, Config), + String. + +unescape_test_() -> + [ + {"unescape backspace", ?_assertEqual( + <<"\b">>, + unescape(<<"\\b"/utf8>>, []) + )}, + {"unescape tab", ?_assertEqual( + <<"\t">>, + unescape(<<"\\t"/utf8>>, []) + )}, + {"unescape newline", ?_assertEqual( + <<"\n">>, + unescape(<<"\\n"/utf8>>, []) + )}, + {"unescape formfeed", ?_assertEqual( + <<"\f">>, + unescape(<<"\\f"/utf8>>, []) + )}, + {"unescape carriage return", ?_assertEqual( + <<"\r">>, + unescape(<<"\\r"/utf8>>, []) + )}, + {"unescape quote", ?_assertEqual( + <<"\"">>, + unescape(<<"\\\""/utf8>>, []) + )}, + {"unescape single quote", ?_assertEqual( + <<"'">>, + unescape(<<"\\'"/utf8>>, [single_quoted_strings]) + )}, + {"unescape solidus", ?_assertEqual( + <<"/">>, + unescape(<<"\\/"/utf8>>, []) + )}, + {"unescape reverse solidus", ?_assertEqual( + <<"\\">>, + unescape(<<"\\\\"/utf8>>, []) + )}, + {"unescape control", ?_assertEqual( + <<0>>, + unescape(<<"\\u0000"/utf8>>, []) + )}, + {"unescape surrogate pair", ?_assertEqual( + <<16#10000/utf8>>, + unescape(<<"\\ud800\\udc00"/utf8>>, []) + )} + ]. + + maybe_escape(Bin, Config) -> [{string, String}, end_json] = decode(Bin, Config), String. From ef566d8cd3e230379008384a31d1ef5bf13b17c3 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sun, 24 Feb 2013 02:42:29 -0800 Subject: [PATCH 59/61] remove cases from clean_string --- src/jsx_encoder.erl | 7 +------ src/jsx_parser.erl | 21 +++------------------ 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 914b35a..4784b96 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -53,12 +53,7 @@ start(Term, {Handler, State}, Config) -> value(String, {Handler, State}, Config) when is_binary(String) -> - case clean_string(String, Config) of - {error, badarg} -> - ?error([String, {Handler, State}, Config]); - CleanString -> - Handler:handle_event({string, CleanString}, State) - end; + Handler:handle_event({string, clean_string(String, Config)}, State); value(Float, {Handler, State}, _Config) when is_float(Float) -> Handler:handle_event({float, Float}, State); value(Int, {Handler, State}, _Config) when is_integer(Int) -> diff --git a/src/jsx_parser.erl b/src/jsx_parser.erl index 0cb146b..e8a3bd9 100644 --- a/src/jsx_parser.erl +++ b/src/jsx_parser.erl @@ -101,19 +101,9 @@ value([Number|Tokens], Handler, Stack, Config) when is_integer(Number) -> value([Number|Tokens], Handler, Stack, Config) when is_float(Number) -> value([{float, Number}] ++ Tokens, Handler, Stack, Config); value([{string, String}|Tokens], Handler, [], Config) when is_binary(String) -> - case clean_string(String, Config) of - {error, badarg} -> - ?error([[{string, String}|Tokens], Handler, [], Config]); - CleanString -> - done(Tokens, handle_event({string, CleanString}, Handler, Config), [], Config) - end; + done(Tokens, handle_event({string, clean_string(String, Config)}, Handler, Config), [], Config); value([{string, String}|Tokens], Handler, Stack, Config) when is_binary(String) -> - case clean_string(String, Config) of - {error, badarg} -> - ?error([[{string, String}|Tokens], Handler, Stack, Config]); - CleanString -> - maybe_done(Tokens, handle_event({string, CleanString}, Handler, Config), Stack, Config) - end; + maybe_done(Tokens, handle_event({string, clean_string(String, Config)}, Handler, Config), Stack, Config); value([String|Tokens], Handler, Stack, Config) when is_binary(String) -> value([{string, String}] ++ Tokens, Handler, Stack, Config); value([], Handler, Stack, Config) -> @@ -126,12 +116,7 @@ value(Token, Handler, Stack, Config) -> object([end_object|Tokens], Handler, [object|Stack], Config) -> maybe_done(Tokens, handle_event(end_object, Handler, Config), Stack, Config); object([{key, Key}|Tokens], Handler, Stack, Config) when is_atom(Key); is_binary(Key) -> - case clean_string(fix_key(Key), Config) of - {error, badarg} -> - ?error([[{key, Key}|Tokens], Handler, Stack, Config]); - CleanString -> - value(Tokens, handle_event({key, CleanString}, Handler, Config), Stack, Config) - end; + value(Tokens, handle_event({key, clean_string(fix_key(Key), Config)}, Handler, Config), Stack, Config); object([Key|Tokens], Handler, Stack, Config) when is_atom(Key); is_binary(Key) -> object([{key, Key}] ++ Tokens, Handler, Stack, Config); object([], Handler, Stack, Config) -> From 3a383b6cb086662957d8c1797ec920e7c77c4f1b Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sun, 24 Feb 2013 02:46:11 -0800 Subject: [PATCH 60/61] revert parser to former version --- src/jsx_parser.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jsx_parser.erl b/src/jsx_parser.erl index e8a3bd9..556d8ff 100644 --- a/src/jsx_parser.erl +++ b/src/jsx_parser.erl @@ -118,7 +118,7 @@ object([end_object|Tokens], Handler, [object|Stack], Config) -> object([{key, Key}|Tokens], Handler, Stack, Config) when is_atom(Key); is_binary(Key) -> value(Tokens, handle_event({key, clean_string(fix_key(Key), Config)}, Handler, Config), Stack, Config); object([Key|Tokens], Handler, Stack, Config) when is_atom(Key); is_binary(Key) -> - object([{key, Key}] ++ Tokens, Handler, Stack, Config); + value(Tokens, handle_event({key, clean_string(fix_key(Key), Config)}, Handler, Config), Stack, Config); object([], Handler, Stack, Config) -> ?incomplete(object, Handler, Stack, Config); object(BadTokens, Handler, Stack, Config) when is_list(BadTokens) -> From 35020643e76a3680b17a6dbcbac6698e99ab466c Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sun, 24 Feb 2013 02:57:35 -0800 Subject: [PATCH 61/61] revert encoder to prior version --- src/jsx_encoder.erl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 4784b96..96508df 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -23,7 +23,7 @@ -module(jsx_encoder). --export([encoder/3, pre_encode/2]). +-export([encoder/3]). -spec encoder(Handler::module(), State::any(), Config::jsx:config()) -> jsx:encoder(). @@ -115,12 +115,11 @@ object(Term, Handler, Config) -> ?error([Term, Handler, Config]). list([Value, Next|Rest], {Handler, State}, Config) -> list([pre_encode(Next, Config)|Rest], {Handler, value(Value, {Handler, State}, Config)}, Config); list([Value], {Handler, State}, Config) -> - list([], {Handler, value(Value, {Handler, State}, Config)}, Config); + list([], {Handler, value(Value, {Handler, State}, Config)}, Config); list([], {Handler, State}, _Config) -> Handler:handle_event(end_array, State); list(Term, Handler, Config) -> ?error([Term, Handler, Config]). - -pre_encode(Value, #config{pre_encode=false}) -> io:format("~p~n", [Value]), Value; +pre_encode(Value, #config{pre_encode=false}) -> Value; pre_encode(Value, Config) -> (Config#config.pre_encode)(Value).