From 50b00bac0f2f77db869bc5169385319ecb072fa9 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Wed, 14 Mar 2012 23:01:59 -0700 Subject: [PATCH 01/27] the option single_quotes in functions dealing with json inputs now allows json that uses single quotes to deliminate keys and strings to be processed, note that this changes the escaping rules slightly --- README.markdown | 11 +++++++++-- src/jsx.erl | 46 +++++++++++++++++++++++++++++++++++++++++++++ src/jsx_decoder.erl | 39 ++++++++++++++++++++++++++++++-------- src/jsx_opts.hrl | 2 +- src/jsx_utils.erl | 6 ++++-- 5 files changed, 91 insertions(+), 13 deletions(-) diff --git a/README.markdown b/README.markdown index 473882f..0464f00 100644 --- a/README.markdown +++ b/README.markdown @@ -32,6 +32,7 @@ types: * `Opts` = `[]` | `[Opt]` * `Opt` = - `loose_unicode` + - `single_quotes` - `labels` - `{labels, Label}` - `Label` = @@ -40,7 +41,9 @@ types: * `existing_atom` - `explicit_end` -`JSON` SHOULD be a utf8 encoded binary. if the option `loose_unicode` is present attempts are made to replace invalid codepoints with `u+FFFD` but badly encoded binaries may, in either case, result in `badarg` errors +`JSON` SHOULD be a utf8 encoded binary. if the option `loose_unicode` is present attempts are made to replace invalid codepoints with `u+FFFD` but badly encoded binaries may, in either case, result in `badarg` errors + +valid json strings are deliminated by double quotes, but some implementations allow single quotes in their place. the `single_quotes` option recognizes json texts with single quotes in the place of double quotes as valid. please be aware that if you enable this option, you MUST escape single quotes in keys and strings the option `labels` controls how keys are converted from json to erlang terms. `binary` does no conversion beyond normal escaping. `atom` converts keys to erlang atoms, and results in a badarg error if keys fall outside the range of erlang atoms. `existing_atom` is identical to `atom`, except it will not add new atoms to the atom table @@ -97,10 +100,13 @@ types: - `indent` - `{indent, N}` - `loose_unicode` + - `single_quotes` - `escape_forward_slash` - `explicit_end` -`JSON` SHOULD be a utf8 encoded binary. if the option `loose_unicode` is present attempts are made to replace invalid codepoints with `u+FFFD` but badly encoded binaries may, in either case, result in `badarg` errors +`JSON` SHOULD be a utf8 encoded binary. if the option `loose_unicode` is present attempts are made to replace invalid codepoints with `u+FFFD` but badly encoded binaries may, in either case, result in `badarg` errors + +valid json strings are deliminated by double quotes, but some implementations allow single quotes in their place. the `single_quotes` option recognizes json texts with single quotes in the place of double quotes as valid. please be aware that if you enable this option, you MUST escape single quotes in keys and strings the option `{space, N}` inserts `N` spaces after every comma and colon in your json output. `space` is an alias for `{space, 1}`. the default is `{space, 0}` @@ -125,6 +131,7 @@ types: * `Opts` = `[]` | `[Opt]` * `Opt` = - `loose_unicode` + - `single_quotes` - `explicit_end` see `json_to_term` for details of options diff --git a/src/jsx.erl b/src/jsx.erl index 6c0dbe5..722f674 100644 --- a/src/jsx.erl +++ b/src/jsx.erl @@ -120,6 +120,52 @@ encoder_decoder_equiv_test_() -> ]. +single_quotes_test_() -> + [ + {"single quoted keys", + ?_assertEqual( + to_term(<<"{'key':true}">>, [single_quotes]), + [{<<"key">>, true}] + ) + }, + {"multiple single quoted keys", + ?_assertEqual( + to_term(<<"{'key':true, 'another key':true}">>, [single_quotes]), + [{<<"key">>, true}, {<<"another key">>, true}] + ) + }, + {"nested single quoted keys", + ?_assertEqual( + to_term(<<"{'key': {'key':true, 'another key':true}}">>, [single_quotes]), + [{<<"key">>, [{<<"key">>, true}, {<<"another key">>, true}]}] + ) + }, + {"single quoted string", + ?_assertEqual( + to_term(<<"['string']">>, [single_quotes]), + [<<"string">>] + ) + }, + {"single quote in double quoted string", + ?_assertEqual( + to_term(<<"[\"a single quote: '\"]">>), + [<<"a single quote: '">>] + ) + }, + {"escaped single quote in single quoted string", + ?_assertEqual( + to_term(<<"['a single quote: \\'']">>, [single_quotes]), + [<<"a single quote: '">>] + ) + }, + {"escaped single quote when single quotes are disallowed", + ?_assertError( + badarg, + to_term(<<"[\"a single quote: \\'\"]">>) + ) + } + ]. + %% test handler init([]) -> []. diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 0a82ea2..700ba0a 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -59,7 +59,8 @@ decoder(Handler, State, Opts) -> %% kv seperator -define(comma, 16#2C). --define(quote, 16#22). +-define(doublequote, 16#22). +-define(singlequote, 16#27). -define(colon, 16#3A). %% string escape sequences @@ -130,7 +131,9 @@ decoder(Handler, State, Opts) -> -define(end_seq(Seq), unicode:characters_to_binary(lists:reverse(Seq))). -value(<>, Handler, Stack, Opts) -> +value(<>, Handler, Stack, Opts) -> + string(Rest, Handler, [?new_seq()|Stack], Opts); +value(<>, Handler, Stack, Opts = #opts{single_quotes=true}) -> string(Rest, Handler, [?new_seq()|Stack], Opts); value(<<$t, Rest/binary>>, Handler, Stack, Opts) -> tr(Rest, Handler, Stack, Opts); @@ -156,7 +159,9 @@ value(Bin, Handler, Stack, Opts) -> ?error([Bin, Handler, Stack, Opts]). -object(<>, Handler, Stack, Opts) -> +object(<>, Handler, Stack, Opts) -> + string(Rest, Handler, [?new_seq()|Stack], Opts); +object(<>, Handler, Stack, Opts = #opts{single_quotes=true}) -> string(Rest, Handler, [?new_seq()|Stack], Opts); object(<>, {Handler, State}, [key|Stack], Opts) -> maybe_done(Rest, {Handler, Handler:handle_event(end_object, State)}, Stack, Opts); @@ -168,7 +173,9 @@ object(Bin, Handler, Stack, Opts) -> ?error([Bin, Handler, Stack, Opts]). -array(<>, Handler, Stack, Opts) -> +array(<>, Handler, Stack, Opts) -> + string(Rest, Handler, [?new_seq()|Stack], Opts); +array(<>, Handler, Stack, Opts = #opts{single_quotes=true}) -> string(Rest, Handler, [?new_seq()|Stack], Opts); array(<<$t, Rest/binary>>, Handler, Stack, Opts) -> tr(Rest, Handler, Stack, Opts); @@ -206,7 +213,9 @@ colon(Bin, Handler, Stack, Opts) -> ?error([Bin, Handler, Stack, Opts]). -key(<>, Handler, Stack, Opts) -> +key(<>, Handler, Stack, Opts) -> + string(Rest, Handler, [?new_seq()|Stack], Opts); +key(<>, Handler, Stack, Opts = #opts{single_quotes=true}) -> string(Rest, Handler, [?new_seq()|Stack], Opts); key(<>, Handler, Stack, Opts) when ?is_whitespace(S) -> key(Rest, Handler, Stack, Opts); @@ -233,13 +242,25 @@ partial_utf(<>) partial_utf(_) -> false. -string(<>, {Handler, State}, [Acc, key|Stack], Opts) -> +string(<>, {Handler, State}, [Acc, key|Stack], Opts) -> colon(Rest, {Handler, Handler:handle_event({key, ?end_seq(Acc)}, State)}, [key|Stack], Opts ); -string(<>, {Handler, State}, [Acc|Stack], Opts) -> +string(<>, {Handler, State}, [Acc, key|Stack], Opts = #opts{single_quotes=true}) -> + colon(Rest, + {Handler, Handler:handle_event({key, ?end_seq(Acc)}, State)}, + [key|Stack], + Opts + ); +string(<>, {Handler, State}, [Acc|Stack], Opts) -> + maybe_done(Rest, + {Handler, Handler:handle_event({string, ?end_seq(Acc)}, State)}, + Stack, + Opts + ); +string(<>, {Handler, State}, [Acc|Stack], Opts = #opts{single_quotes=true}) -> maybe_done(Rest, {Handler, Handler:handle_event({string, ?end_seq(Acc)}, State)}, Stack, @@ -318,8 +339,10 @@ escape(<<$t, Rest/binary>>, Handler, [Acc|Stack], Opts) -> escape(<<$u, Rest/binary>>, Handler, Stack, Opts) -> escaped_unicode(Rest, Handler, [?new_seq()|Stack], Opts); escape(<>, Handler, [Acc|Stack], Opts) - when S =:= ?quote; S =:= ?solidus; S =:= ?rsolidus -> + when S =:= ?doublequote; S =:= ?solidus; S =:= ?rsolidus -> string(Rest, Handler, [?acc_seq(Acc, S)|Stack], Opts); +escape(<>, Handler, [Acc|Stack], Opts = #opts{single_quotes=true}) -> + string(Rest, Handler, [?acc_seq(Acc, ?singlequote)|Stack], Opts); escape(<<>>, Handler, Stack, Opts) -> ?incomplete(escape, <<>>, Handler, Stack, Opts); escape(Bin, Handler, Stack, Opts) -> diff --git a/src/jsx_opts.hrl b/src/jsx_opts.hrl index d49254b..f60627f 100644 --- a/src/jsx_opts.hrl +++ b/src/jsx_opts.hrl @@ -2,5 +2,5 @@ loose_unicode = false, escape_forward_slash = false, explicit_end = false, - parser = auto + single_quotes = false }). \ No newline at end of file diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index 814092c..2c8b160 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -43,6 +43,8 @@ parse_opts([escape_forward_slash|Rest], Opts) -> parse_opts(Rest, Opts#opts{escape_forward_slash=true}); parse_opts([explicit_end|Rest], Opts) -> parse_opts(Rest, Opts#opts{explicit_end=true}); +parse_opts([single_quotes|Rest], Opts) -> + parse_opts(Rest, Opts#opts{single_quotes=true}); parse_opts(_, _) -> {error, badarg}. @@ -52,12 +54,12 @@ extract_opts(Opts) -> extract_parser_opts([], Acc) -> Acc; extract_parser_opts([{K,V}|Rest], Acc) -> - case lists:member(K, [loose_unicode, escape_forward_slash, explicit_end]) of + case lists:member(K, [loose_unicode, escape_forward_slash, explicit_end, single_quotes]) of true -> extract_parser_opts(Rest, [{K,V}] ++ Acc) ; false -> extract_parser_opts(Rest, Acc) end; extract_parser_opts([K|Rest], Acc) -> - case lists:member(K, [loose_unicode, escape_forward_slash, explicit_end]) of + case lists:member(K, [loose_unicode, escape_forward_slash, explicit_end, single_quotes]) of true -> extract_parser_opts(Rest, [K] ++ Acc) ; false -> extract_parser_opts(Rest, Acc) end. From c6827d06de2b69591576336fc9ad84a2f841d74d Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Wed, 14 Mar 2012 23:01:59 -0700 Subject: [PATCH 02/27] the option single_quotes in functions dealing with json inputs now allows json that uses single quotes to deliminate keys and strings to be processed, note that this changes the escaping rules slightly --- README.markdown | 11 ++++++++-- src/jsx.erl | 52 ++++++++++++++++++++++++++++++++++++++++++++ src/jsx_decoder.erl | 53 ++++++++++++++++++++++++++++++--------------- src/jsx_opts.hrl | 2 +- src/jsx_utils.erl | 6 +++-- 5 files changed, 101 insertions(+), 23 deletions(-) diff --git a/README.markdown b/README.markdown index 473882f..0464f00 100644 --- a/README.markdown +++ b/README.markdown @@ -32,6 +32,7 @@ types: * `Opts` = `[]` | `[Opt]` * `Opt` = - `loose_unicode` + - `single_quotes` - `labels` - `{labels, Label}` - `Label` = @@ -40,7 +41,9 @@ types: * `existing_atom` - `explicit_end` -`JSON` SHOULD be a utf8 encoded binary. if the option `loose_unicode` is present attempts are made to replace invalid codepoints with `u+FFFD` but badly encoded binaries may, in either case, result in `badarg` errors +`JSON` SHOULD be a utf8 encoded binary. if the option `loose_unicode` is present attempts are made to replace invalid codepoints with `u+FFFD` but badly encoded binaries may, in either case, result in `badarg` errors + +valid json strings are deliminated by double quotes, but some implementations allow single quotes in their place. the `single_quotes` option recognizes json texts with single quotes in the place of double quotes as valid. please be aware that if you enable this option, you MUST escape single quotes in keys and strings the option `labels` controls how keys are converted from json to erlang terms. `binary` does no conversion beyond normal escaping. `atom` converts keys to erlang atoms, and results in a badarg error if keys fall outside the range of erlang atoms. `existing_atom` is identical to `atom`, except it will not add new atoms to the atom table @@ -97,10 +100,13 @@ types: - `indent` - `{indent, N}` - `loose_unicode` + - `single_quotes` - `escape_forward_slash` - `explicit_end` -`JSON` SHOULD be a utf8 encoded binary. if the option `loose_unicode` is present attempts are made to replace invalid codepoints with `u+FFFD` but badly encoded binaries may, in either case, result in `badarg` errors +`JSON` SHOULD be a utf8 encoded binary. if the option `loose_unicode` is present attempts are made to replace invalid codepoints with `u+FFFD` but badly encoded binaries may, in either case, result in `badarg` errors + +valid json strings are deliminated by double quotes, but some implementations allow single quotes in their place. the `single_quotes` option recognizes json texts with single quotes in the place of double quotes as valid. please be aware that if you enable this option, you MUST escape single quotes in keys and strings the option `{space, N}` inserts `N` spaces after every comma and colon in your json output. `space` is an alias for `{space, 1}`. the default is `{space, 0}` @@ -125,6 +131,7 @@ types: * `Opts` = `[]` | `[Opt]` * `Opt` = - `loose_unicode` + - `single_quotes` - `explicit_end` see `json_to_term` for details of options diff --git a/src/jsx.erl b/src/jsx.erl index 6c0dbe5..ff1bc09 100644 --- a/src/jsx.erl +++ b/src/jsx.erl @@ -120,6 +120,58 @@ encoder_decoder_equiv_test_() -> ]. +single_quotes_test_() -> + [ + {"single quoted keys", + ?_assertEqual( + to_term(<<"{'key':true}">>, [single_quotes]), + [{<<"key">>, true}] + ) + }, + {"multiple single quoted keys", + ?_assertEqual( + to_term(<<"{'key':true, 'another key':true}">>, [single_quotes]), + [{<<"key">>, true}, {<<"another key">>, true}] + ) + }, + {"nested single quoted keys", + ?_assertEqual( + to_term(<<"{'key': {'key':true, 'another key':true}}">>, [single_quotes]), + [{<<"key">>, [{<<"key">>, true}, {<<"another key">>, true}]}] + ) + }, + {"single quoted string", + ?_assertEqual( + to_term(<<"['string']">>, [single_quotes]), + [<<"string">>] + ) + }, + {"single quote in double quoted string", + ?_assertEqual( + to_term(<<"[\"a single quote: '\"]">>, [single_quotes]), + [<<"a single quote: '">>] + ) + }, + {"escaped single quote in single quoted string", + ?_assertEqual( + to_term(<<"['a single quote: \\'']">>, [single_quotes]), + [<<"a single quote: '">>] + ) + }, + {"escaped single quote when single quotes are disallowed", + ?_assertError( + badarg, + to_term(<<"[\"a single quote: \\'\"]">>) + ) + }, + {"mismatched quotes", + ?_assertError( + badarg, + to_term(<<"['mismatched\"]">>, [single_quotes]) + ) + } + ]. + %% test handler init([]) -> []. diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 0a82ea2..2cca939 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -59,7 +59,8 @@ decoder(Handler, State, Opts) -> %% kv seperator -define(comma, 16#2C). --define(quote, 16#22). +-define(doublequote, 16#22). +-define(singlequote, 16#27). -define(colon, 16#3A). %% string escape sequences @@ -130,8 +131,10 @@ decoder(Handler, State, Opts) -> -define(end_seq(Seq), unicode:characters_to_binary(lists:reverse(Seq))). -value(<>, Handler, Stack, Opts) -> +value(<>, Handler, Stack, Opts) -> string(Rest, Handler, [?new_seq()|Stack], Opts); +value(<>, Handler, Stack, Opts = #opts{single_quotes=true}) -> + string(Rest, Handler, [?new_seq(), single_quote|Stack], Opts); value(<<$t, Rest/binary>>, Handler, Stack, Opts) -> tr(Rest, Handler, Stack, Opts); value(<<$f, Rest/binary>>, Handler, Stack, Opts) -> @@ -156,8 +159,10 @@ value(Bin, Handler, Stack, Opts) -> ?error([Bin, Handler, Stack, Opts]). -object(<>, Handler, Stack, Opts) -> +object(<>, Handler, Stack, Opts) -> string(Rest, Handler, [?new_seq()|Stack], Opts); +object(<>, Handler, Stack, Opts = #opts{single_quotes=true}) -> + string(Rest, Handler, [?new_seq(), single_quote|Stack], Opts); object(<>, {Handler, State}, [key|Stack], Opts) -> maybe_done(Rest, {Handler, Handler:handle_event(end_object, State)}, Stack, Opts); object(<>, Handler, Stack, Opts) when ?is_whitespace(S) -> @@ -168,8 +173,10 @@ object(Bin, Handler, Stack, Opts) -> ?error([Bin, Handler, Stack, Opts]). -array(<>, Handler, Stack, Opts) -> +array(<>, Handler, Stack, Opts) -> string(Rest, Handler, [?new_seq()|Stack], Opts); +array(<>, Handler, Stack, Opts = #opts{single_quotes=true}) -> + string(Rest, Handler, [?new_seq(), single_quote|Stack], Opts); array(<<$t, Rest/binary>>, Handler, Stack, Opts) -> tr(Rest, Handler, Stack, Opts); array(<<$f, Rest/binary>>, Handler, Stack, Opts) -> @@ -206,8 +213,10 @@ colon(Bin, Handler, Stack, Opts) -> ?error([Bin, Handler, Stack, Opts]). -key(<>, Handler, Stack, Opts) -> +key(<>, Handler, Stack, Opts) -> string(Rest, Handler, [?new_seq()|Stack], Opts); +key(<>, Handler, Stack, Opts = #opts{single_quotes=true}) -> + string(Rest, Handler, [?new_seq(), single_quote|Stack], Opts); key(<>, Handler, Stack, Opts) when ?is_whitespace(S) -> key(Rest, Handler, Stack, Opts); key(<<>>, Handler, Stack, Opts) -> @@ -233,18 +242,24 @@ partial_utf(<>) partial_utf(_) -> false. -string(<>, {Handler, State}, [Acc, key|Stack], Opts) -> - colon(Rest, - {Handler, Handler:handle_event({key, ?end_seq(Acc)}, State)}, - [key|Stack], - Opts - ); -string(<>, {Handler, State}, [Acc|Stack], Opts) -> - maybe_done(Rest, - {Handler, Handler:handle_event({string, ?end_seq(Acc)}, State)}, - Stack, - Opts - ); +string(<>, {Handler, State}, S, Opts) -> + case S of + [Acc, key|Stack] -> + colon(Rest, {Handler, Handler:handle_event({key, ?end_seq(Acc)}, State)}, [key|Stack], Opts); + [Acc|Stack] -> + maybe_done(Rest, {Handler, Handler:handle_event({string, ?end_seq(Acc)}, State)}, Stack, Opts); + [Acc, single_quote|Stack] -> + ?error([<>, {Handler, State}, S, Opts]) + end; +string(<>, {Handler, State}, S, Opts = #opts{single_quotes=true}) -> + case S of + [Acc, single_quote, key|Stack] -> + colon(Rest, {Handler, Handler:handle_event({key, ?end_seq(Acc)}, State)}, [key|Stack], Opts); + [Acc, single_quote|Stack] -> + maybe_done(Rest, {Handler, Handler:handle_event({string, ?end_seq(Acc)}, State)}, Stack, Opts); + [Acc|Stack] -> + string(Rest, {Handler, State}, [?acc_seq(Acc, ?singlequote)|Stack], Opts) + end; string(<>, Handler, Stack, Opts) -> escape(Rest, Handler, Stack, Opts); %% things get dumb here. erlang doesn't properly restrict unicode non-characters @@ -318,8 +333,10 @@ escape(<<$t, Rest/binary>>, Handler, [Acc|Stack], Opts) -> escape(<<$u, Rest/binary>>, Handler, Stack, Opts) -> escaped_unicode(Rest, Handler, [?new_seq()|Stack], Opts); escape(<>, Handler, [Acc|Stack], Opts) - when S =:= ?quote; S =:= ?solidus; S =:= ?rsolidus -> + when S =:= ?doublequote; S =:= ?solidus; S =:= ?rsolidus -> string(Rest, Handler, [?acc_seq(Acc, S)|Stack], Opts); +escape(<>, Handler, [Acc|Stack], Opts = #opts{single_quotes=true}) -> + string(Rest, Handler, [?acc_seq(Acc, ?singlequote)|Stack], Opts); escape(<<>>, Handler, Stack, Opts) -> ?incomplete(escape, <<>>, Handler, Stack, Opts); escape(Bin, Handler, Stack, Opts) -> diff --git a/src/jsx_opts.hrl b/src/jsx_opts.hrl index d49254b..f60627f 100644 --- a/src/jsx_opts.hrl +++ b/src/jsx_opts.hrl @@ -2,5 +2,5 @@ loose_unicode = false, escape_forward_slash = false, explicit_end = false, - parser = auto + single_quotes = false }). \ No newline at end of file diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index 814092c..2c8b160 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -43,6 +43,8 @@ parse_opts([escape_forward_slash|Rest], Opts) -> parse_opts(Rest, Opts#opts{escape_forward_slash=true}); parse_opts([explicit_end|Rest], Opts) -> parse_opts(Rest, Opts#opts{explicit_end=true}); +parse_opts([single_quotes|Rest], Opts) -> + parse_opts(Rest, Opts#opts{single_quotes=true}); parse_opts(_, _) -> {error, badarg}. @@ -52,12 +54,12 @@ extract_opts(Opts) -> extract_parser_opts([], Acc) -> Acc; extract_parser_opts([{K,V}|Rest], Acc) -> - case lists:member(K, [loose_unicode, escape_forward_slash, explicit_end]) of + case lists:member(K, [loose_unicode, escape_forward_slash, explicit_end, single_quotes]) of true -> extract_parser_opts(Rest, [{K,V}] ++ Acc) ; false -> extract_parser_opts(Rest, Acc) end; extract_parser_opts([K|Rest], Acc) -> - case lists:member(K, [loose_unicode, escape_forward_slash, explicit_end]) of + case lists:member(K, [loose_unicode, escape_forward_slash, explicit_end, single_quotes]) of true -> extract_parser_opts(Rest, [K] ++ Acc) ; false -> extract_parser_opts(Rest, Acc) end. From 59689769de368e70bb730132b5de3fb3481510c0 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Thu, 15 Mar 2012 20:54:52 -0700 Subject: [PATCH 03/27] supress unused var errors --- src/jsx_decoder.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 2cca939..006cc22 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -246,10 +246,10 @@ string(<>, {Handler, State}, S, Opts) -> case S of [Acc, key|Stack] -> colon(Rest, {Handler, Handler:handle_event({key, ?end_seq(Acc)}, State)}, [key|Stack], Opts); + [_Acc, single_quote|_Stack] -> + ?error([<>, {Handler, State}, S, Opts]); [Acc|Stack] -> - maybe_done(Rest, {Handler, Handler:handle_event({string, ?end_seq(Acc)}, State)}, Stack, Opts); - [Acc, single_quote|Stack] -> - ?error([<>, {Handler, State}, S, Opts]) + maybe_done(Rest, {Handler, Handler:handle_event({string, ?end_seq(Acc)}, State)}, Stack, Opts) end; string(<>, {Handler, State}, S, Opts = #opts{single_quotes=true}) -> case S of From a0657303802948577826539c627f5b63490e7b97 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Wed, 14 Mar 2012 23:01:59 -0700 Subject: [PATCH 04/27] the option single_quotes in functions dealing with json inputs now allows json that uses single quotes to deliminate keys and strings to be processed, note that this changes the escaping rules slightly --- README.markdown | 11 ++++++++-- src/jsx.erl | 52 ++++++++++++++++++++++++++++++++++++++++++++ src/jsx_decoder.erl | 53 ++++++++++++++++++++++++++++++--------------- src/jsx_opts.hrl | 2 +- src/jsx_utils.erl | 6 +++-- 5 files changed, 101 insertions(+), 23 deletions(-) diff --git a/README.markdown b/README.markdown index 1b3743f..28715b1 100644 --- a/README.markdown +++ b/README.markdown @@ -32,6 +32,7 @@ types: * `Opts` = `[]` | `[Opt]` * `Opt` = - `loose_unicode` + - `single_quotes` - `labels` - `{labels, Label}` - `Label` = @@ -40,7 +41,9 @@ types: * `existing_atom` - `explicit_end` -`JSON` SHOULD be a utf8 encoded binary. if the option `loose_unicode` is present attempts are made to replace invalid codepoints with `u+FFFD` but badly encoded binaries may, in either case, result in `badarg` errors +`JSON` SHOULD be a utf8 encoded binary. if the option `loose_unicode` is present attempts are made to replace invalid codepoints with `u+FFFD` but badly encoded binaries may, in either case, result in `badarg` errors + +valid json strings are deliminated by double quotes, but some implementations allow single quotes in their place. the `single_quotes` option recognizes json texts with single quotes in the place of double quotes as valid. please be aware that if you enable this option, you MUST escape single quotes in keys and strings the option `labels` controls how keys are converted from json to erlang terms. `binary` does no conversion beyond normal escaping. `atom` converts keys to erlang atoms, and results in a badarg error if keys fall outside the range of erlang atoms. `existing_atom` is identical to `atom`, except it will not add new atoms to the atom table @@ -97,10 +100,13 @@ types: - `indent` - `{indent, N}` - `loose_unicode` + - `single_quotes` - `escape_forward_slash` - `explicit_end` -`JSON` SHOULD be a utf8 encoded binary. if the option `loose_unicode` is present attempts are made to replace invalid codepoints with `u+FFFD` but badly encoded binaries may, in either case, result in `badarg` errors +`JSON` SHOULD be a utf8 encoded binary. if the option `loose_unicode` is present attempts are made to replace invalid codepoints with `u+FFFD` but badly encoded binaries may, in either case, result in `badarg` errors + +valid json strings are deliminated by double quotes, but some implementations allow single quotes in their place. the `single_quotes` option recognizes json texts with single quotes in the place of double quotes as valid. please be aware that if you enable this option, you MUST escape single quotes in keys and strings the option `{space, N}` inserts `N` spaces after every comma and colon in your json output. `space` is an alias for `{space, 1}`. the default is `{space, 0}` @@ -125,6 +131,7 @@ types: * `Opts` = `[]` | `[Opt]` * `Opt` = - `loose_unicode` + - `single_quotes` - `explicit_end` see `json_to_term` for details of options diff --git a/src/jsx.erl b/src/jsx.erl index 6c0dbe5..ff1bc09 100644 --- a/src/jsx.erl +++ b/src/jsx.erl @@ -120,6 +120,58 @@ encoder_decoder_equiv_test_() -> ]. +single_quotes_test_() -> + [ + {"single quoted keys", + ?_assertEqual( + to_term(<<"{'key':true}">>, [single_quotes]), + [{<<"key">>, true}] + ) + }, + {"multiple single quoted keys", + ?_assertEqual( + to_term(<<"{'key':true, 'another key':true}">>, [single_quotes]), + [{<<"key">>, true}, {<<"another key">>, true}] + ) + }, + {"nested single quoted keys", + ?_assertEqual( + to_term(<<"{'key': {'key':true, 'another key':true}}">>, [single_quotes]), + [{<<"key">>, [{<<"key">>, true}, {<<"another key">>, true}]}] + ) + }, + {"single quoted string", + ?_assertEqual( + to_term(<<"['string']">>, [single_quotes]), + [<<"string">>] + ) + }, + {"single quote in double quoted string", + ?_assertEqual( + to_term(<<"[\"a single quote: '\"]">>, [single_quotes]), + [<<"a single quote: '">>] + ) + }, + {"escaped single quote in single quoted string", + ?_assertEqual( + to_term(<<"['a single quote: \\'']">>, [single_quotes]), + [<<"a single quote: '">>] + ) + }, + {"escaped single quote when single quotes are disallowed", + ?_assertError( + badarg, + to_term(<<"[\"a single quote: \\'\"]">>) + ) + }, + {"mismatched quotes", + ?_assertError( + badarg, + to_term(<<"['mismatched\"]">>, [single_quotes]) + ) + } + ]. + %% test handler init([]) -> []. diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 0a82ea2..2cca939 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -59,7 +59,8 @@ decoder(Handler, State, Opts) -> %% kv seperator -define(comma, 16#2C). --define(quote, 16#22). +-define(doublequote, 16#22). +-define(singlequote, 16#27). -define(colon, 16#3A). %% string escape sequences @@ -130,8 +131,10 @@ decoder(Handler, State, Opts) -> -define(end_seq(Seq), unicode:characters_to_binary(lists:reverse(Seq))). -value(<>, Handler, Stack, Opts) -> +value(<>, Handler, Stack, Opts) -> string(Rest, Handler, [?new_seq()|Stack], Opts); +value(<>, Handler, Stack, Opts = #opts{single_quotes=true}) -> + string(Rest, Handler, [?new_seq(), single_quote|Stack], Opts); value(<<$t, Rest/binary>>, Handler, Stack, Opts) -> tr(Rest, Handler, Stack, Opts); value(<<$f, Rest/binary>>, Handler, Stack, Opts) -> @@ -156,8 +159,10 @@ value(Bin, Handler, Stack, Opts) -> ?error([Bin, Handler, Stack, Opts]). -object(<>, Handler, Stack, Opts) -> +object(<>, Handler, Stack, Opts) -> string(Rest, Handler, [?new_seq()|Stack], Opts); +object(<>, Handler, Stack, Opts = #opts{single_quotes=true}) -> + string(Rest, Handler, [?new_seq(), single_quote|Stack], Opts); object(<>, {Handler, State}, [key|Stack], Opts) -> maybe_done(Rest, {Handler, Handler:handle_event(end_object, State)}, Stack, Opts); object(<>, Handler, Stack, Opts) when ?is_whitespace(S) -> @@ -168,8 +173,10 @@ object(Bin, Handler, Stack, Opts) -> ?error([Bin, Handler, Stack, Opts]). -array(<>, Handler, Stack, Opts) -> +array(<>, Handler, Stack, Opts) -> string(Rest, Handler, [?new_seq()|Stack], Opts); +array(<>, Handler, Stack, Opts = #opts{single_quotes=true}) -> + string(Rest, Handler, [?new_seq(), single_quote|Stack], Opts); array(<<$t, Rest/binary>>, Handler, Stack, Opts) -> tr(Rest, Handler, Stack, Opts); array(<<$f, Rest/binary>>, Handler, Stack, Opts) -> @@ -206,8 +213,10 @@ colon(Bin, Handler, Stack, Opts) -> ?error([Bin, Handler, Stack, Opts]). -key(<>, Handler, Stack, Opts) -> +key(<>, Handler, Stack, Opts) -> string(Rest, Handler, [?new_seq()|Stack], Opts); +key(<>, Handler, Stack, Opts = #opts{single_quotes=true}) -> + string(Rest, Handler, [?new_seq(), single_quote|Stack], Opts); key(<>, Handler, Stack, Opts) when ?is_whitespace(S) -> key(Rest, Handler, Stack, Opts); key(<<>>, Handler, Stack, Opts) -> @@ -233,18 +242,24 @@ partial_utf(<>) partial_utf(_) -> false. -string(<>, {Handler, State}, [Acc, key|Stack], Opts) -> - colon(Rest, - {Handler, Handler:handle_event({key, ?end_seq(Acc)}, State)}, - [key|Stack], - Opts - ); -string(<>, {Handler, State}, [Acc|Stack], Opts) -> - maybe_done(Rest, - {Handler, Handler:handle_event({string, ?end_seq(Acc)}, State)}, - Stack, - Opts - ); +string(<>, {Handler, State}, S, Opts) -> + case S of + [Acc, key|Stack] -> + colon(Rest, {Handler, Handler:handle_event({key, ?end_seq(Acc)}, State)}, [key|Stack], Opts); + [Acc|Stack] -> + maybe_done(Rest, {Handler, Handler:handle_event({string, ?end_seq(Acc)}, State)}, Stack, Opts); + [Acc, single_quote|Stack] -> + ?error([<>, {Handler, State}, S, Opts]) + end; +string(<>, {Handler, State}, S, Opts = #opts{single_quotes=true}) -> + case S of + [Acc, single_quote, key|Stack] -> + colon(Rest, {Handler, Handler:handle_event({key, ?end_seq(Acc)}, State)}, [key|Stack], Opts); + [Acc, single_quote|Stack] -> + maybe_done(Rest, {Handler, Handler:handle_event({string, ?end_seq(Acc)}, State)}, Stack, Opts); + [Acc|Stack] -> + string(Rest, {Handler, State}, [?acc_seq(Acc, ?singlequote)|Stack], Opts) + end; string(<>, Handler, Stack, Opts) -> escape(Rest, Handler, Stack, Opts); %% things get dumb here. erlang doesn't properly restrict unicode non-characters @@ -318,8 +333,10 @@ escape(<<$t, Rest/binary>>, Handler, [Acc|Stack], Opts) -> escape(<<$u, Rest/binary>>, Handler, Stack, Opts) -> escaped_unicode(Rest, Handler, [?new_seq()|Stack], Opts); escape(<>, Handler, [Acc|Stack], Opts) - when S =:= ?quote; S =:= ?solidus; S =:= ?rsolidus -> + when S =:= ?doublequote; S =:= ?solidus; S =:= ?rsolidus -> string(Rest, Handler, [?acc_seq(Acc, S)|Stack], Opts); +escape(<>, Handler, [Acc|Stack], Opts = #opts{single_quotes=true}) -> + string(Rest, Handler, [?acc_seq(Acc, ?singlequote)|Stack], Opts); escape(<<>>, Handler, Stack, Opts) -> ?incomplete(escape, <<>>, Handler, Stack, Opts); escape(Bin, Handler, Stack, Opts) -> diff --git a/src/jsx_opts.hrl b/src/jsx_opts.hrl index d49254b..f60627f 100644 --- a/src/jsx_opts.hrl +++ b/src/jsx_opts.hrl @@ -2,5 +2,5 @@ loose_unicode = false, escape_forward_slash = false, explicit_end = false, - parser = auto + single_quotes = false }). \ No newline at end of file diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index 814092c..2c8b160 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -43,6 +43,8 @@ parse_opts([escape_forward_slash|Rest], Opts) -> parse_opts(Rest, Opts#opts{escape_forward_slash=true}); parse_opts([explicit_end|Rest], Opts) -> parse_opts(Rest, Opts#opts{explicit_end=true}); +parse_opts([single_quotes|Rest], Opts) -> + parse_opts(Rest, Opts#opts{single_quotes=true}); parse_opts(_, _) -> {error, badarg}. @@ -52,12 +54,12 @@ extract_opts(Opts) -> extract_parser_opts([], Acc) -> Acc; extract_parser_opts([{K,V}|Rest], Acc) -> - case lists:member(K, [loose_unicode, escape_forward_slash, explicit_end]) of + case lists:member(K, [loose_unicode, escape_forward_slash, explicit_end, single_quotes]) of true -> extract_parser_opts(Rest, [{K,V}] ++ Acc) ; false -> extract_parser_opts(Rest, Acc) end; extract_parser_opts([K|Rest], Acc) -> - case lists:member(K, [loose_unicode, escape_forward_slash, explicit_end]) of + case lists:member(K, [loose_unicode, escape_forward_slash, explicit_end, single_quotes]) of true -> extract_parser_opts(Rest, [K] ++ Acc) ; false -> extract_parser_opts(Rest, Acc) end. From dd917eb471127627bc8c4f6e0fb4ec0652448e2b Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Thu, 15 Mar 2012 20:54:52 -0700 Subject: [PATCH 05/27] supress unused var errors --- src/jsx_decoder.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 2cca939..006cc22 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -246,10 +246,10 @@ string(<>, {Handler, State}, S, Opts) -> case S of [Acc, key|Stack] -> colon(Rest, {Handler, Handler:handle_event({key, ?end_seq(Acc)}, State)}, [key|Stack], Opts); + [_Acc, single_quote|_Stack] -> + ?error([<>, {Handler, State}, S, Opts]); [Acc|Stack] -> - maybe_done(Rest, {Handler, Handler:handle_event({string, ?end_seq(Acc)}, State)}, Stack, Opts); - [Acc, single_quote|Stack] -> - ?error([<>, {Handler, State}, S, Opts]) + maybe_done(Rest, {Handler, Handler:handle_event({string, ?end_seq(Acc)}, State)}, Stack, Opts) end; string(<>, {Handler, State}, S, Opts = #opts{single_quotes=true}) -> case S of From 30b5cea06d7e42e8877f561f4828e88ebf66d7f7 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Thu, 15 Mar 2012 22:56:21 -0700 Subject: [PATCH 06/27] escape strings and keys in the encoder --- src/jsx_encoder.erl | 29 ++++++++++++++++++++++------- src/jsx_utils.erl | 6 +++--- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 68bff84..f4d83e7 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -53,8 +53,8 @@ start(Term, {Handler, State}, Opts) -> Handler:handle_event(end_json, value(Term, {Handler, State}, Opts)). -value(String, {Handler, State}, _Opts) when is_binary(String) -> - Handler:handle_event({string, String}, State); +value(String, {Handler, State}, Opts) when is_binary(String) -> + Handler:handle_event({string, escape(String, {Handler, State}, Opts)}, State); value(Float, {Handler, State}, _Opts) when is_float(Float) -> Handler:handle_event({float, Float}, State); value(Int, {Handler, State}, _Opts) when is_integer(Int) -> @@ -78,9 +78,18 @@ list_or_object(List, {Handler, State}, Opts) -> object([{Key, Value}|Rest], {Handler, State}, Opts) -> - object(Rest, {Handler, - value(Value, {Handler, Handler:handle_event({key, fix_key(Key)}, State)}, Opts) - }, Opts); + object( + Rest, + { + Handler, + value( + Value, + {Handler, Handler:handle_event({key, escape(fix_key(Key), {Handler, State}, Opts)}, State)}, + Opts + ) + }, + Opts + ); object([], {Handler, State}, _Opts) -> Handler:handle_event(end_object, State); object(Term, Handler, Opts) -> ?error([Term, Handler, Opts]). @@ -91,8 +100,14 @@ list([], {Handler, State}, _Opts) -> Handler:handle_event(end_array, State); list(Term, Handler, Opts) -> ?error([Term, Handler, Opts]). -fix_key(Key) when is_binary(Key) -> Key; -fix_key(Key) when is_atom(Key) -> atom_to_binary(Key, utf8). +fix_key(Key) when is_atom(Key) -> fix_key(atom_to_binary(Key, utf8)); +fix_key(Key) when is_binary(Key) -> Key. + + +escape(String, Handler, Opts) -> + try jsx_utils:json_escape(String, Opts) + catch error:badarg -> erlang:error(badarg, [String, Handler, Opts]) + end. -ifdef(TEST). diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index 2c8b160..fa6db97 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -97,7 +97,7 @@ json_escape(<<$\t, Rest/binary>>, Opts, Acc) -> json_escape(<>, Opts, Acc) when C >= 0, C < $\s -> json_escape(Rest, Opts, - <> + <> ); %% escape forward slashes -- optionally -- to faciliate microsoft's retarded %% date format @@ -108,7 +108,7 @@ json_escape(<>, Opts, Acc) when C == 16#2028; C == 16#2029 -> json_escape(Rest, Opts, - <> + <> ); %% any other legal codepoint json_escape(<>, Opts, Acc) -> @@ -122,7 +122,7 @@ json_escape(Rest, Opts, Acc) -> %% convert a codepoint to it's \uXXXX equiv. json_escape_sequence(X) -> <> = <>, - [$\\, $u, (to_hex(A)), (to_hex(B)), (to_hex(C)), (to_hex(D))]. + unicode:characters_to_binary([$\\, $u, (to_hex(A)), (to_hex(B)), (to_hex(C)), (to_hex(D))]). to_hex(10) -> $a; From 6392808d318ae3501869e43ce4d313ff5c5ae2d7 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Thu, 15 Mar 2012 23:06:19 -0700 Subject: [PATCH 07/27] add 'no_jsonp_escapes' flag/option to not escape u+2028 and u+2029 --- src/jsx_opts.hrl | 3 ++- src/jsx_utils.erl | 50 ++++++++++++++++++++++++++++++++++------------- 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/src/jsx_opts.hrl b/src/jsx_opts.hrl index f60627f..938b8fb 100644 --- a/src/jsx_opts.hrl +++ b/src/jsx_opts.hrl @@ -2,5 +2,6 @@ loose_unicode = false, escape_forward_slash = false, explicit_end = false, - single_quotes = false + single_quotes = false, + no_jsonp_escapes = false }). \ No newline at end of file diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index fa6db97..2ab87e9 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -45,21 +45,33 @@ parse_opts([explicit_end|Rest], Opts) -> parse_opts(Rest, Opts#opts{explicit_end=true}); parse_opts([single_quotes|Rest], Opts) -> parse_opts(Rest, Opts#opts{single_quotes=true}); +parse_opts([no_jsonp_escapes|Rest], Opts) -> + parse_opts(Rest, Opts#opts{no_jsonp_escapes=true}); parse_opts(_, _) -> {error, badarg}. +valid_flags() -> + [ + loose_unicode, + escape_forward_slash, + explicit_end, + single_quotes, + no_jsonp_escapes + ]. + + extract_opts(Opts) -> extract_parser_opts(Opts, []). extract_parser_opts([], Acc) -> Acc; extract_parser_opts([{K,V}|Rest], Acc) -> - case lists:member(K, [loose_unicode, escape_forward_slash, explicit_end, single_quotes]) of + case lists:member(K, valid_flags()) of true -> extract_parser_opts(Rest, [{K,V}] ++ Acc) ; false -> extract_parser_opts(Rest, Acc) end; extract_parser_opts([K|Rest], Acc) -> - case lists:member(K, [loose_unicode, escape_forward_slash, explicit_end, single_quotes]) of + case lists:member(K, valid_flags()) of true -> extract_parser_opts(Rest, [K] ++ Acc) ; false -> extract_parser_opts(Rest, Acc) end. @@ -103,6 +115,10 @@ json_escape(<>, Opts, Acc) when C >= 0, C < $\s -> %% date format json_escape(<<$/, Rest/binary>>, Opts=#opts{escape_forward_slash=true}, Acc) -> json_escape(Rest, Opts, <>); +%% skip escaping u+2028 and u+2029 +json_escape(<>, Opts=#opts{no_jsonp_escapes=true}, Acc) + when C == 16#2028; C == 16#2029 -> + json_escape(Rest, Opts, <>); %% escape u+2028 and u+2029 to avoid problems with jsonp json_escape(<>, Opts, Acc) when C == 16#2028; C == 16#2029 -> @@ -143,27 +159,33 @@ to_hex(X) -> X + 48. %% ascii "1" is [49], "2" is [50], etc... binary_escape_test_() -> [ {"json string escaping", - ?_assert(json_escape( - <<"\"\\\b\f\n\r\t">>, #opts{} - ) =:= <<"\\\"\\\\\\b\\f\\n\\r\\t">> + ?_assertEqual( + json_escape(<<"\"\\\b\f\n\r\t">>, #opts{}), + <<"\\\"\\\\\\b\\f\\n\\r\\t">> ) }, {"json string hex escape", - ?_assert(json_escape( - <<1, 2, 3, 11, 26, 30, 31>>, #opts{} - ) =:= <<"\\u0001\\u0002\\u0003\\u000b\\u001a\\u001e\\u001f">> + ?_assertEqual( + json_escape(<<1, 2, 3, 11, 26, 30, 31>>, #opts{}), + <<"\\u0001\\u0002\\u0003\\u000b\\u001a\\u001e\\u001f">> ) }, {"jsonp protection", - ?_assert(json_escape( - <<226, 128, 168, 226, 128, 169>>, #opts{} - ) =:= <<"\\u2028\\u2029">> + ?_assertEqual( + json_escape(<<226, 128, 168, 226, 128, 169>>, #opts{}), + <<"\\u2028\\u2029">> + ) + }, + {"no jsonp escapes", + ?_assertEqual( + json_escape(<<226, 128, 168, 226, 128, 169>>, #opts{no_jsonp_escapes=true}), + <<226, 128, 168, 226, 128, 169>> ) }, {"microsoft i hate your date format", - ?_assert(json_escape(<<"/Date(1303502009425)/">>, - #opts{escape_forward_slash=true} - ) =:= <<"\\/Date(1303502009425)\\/">> + ?_assertEqual( + json_escape(<<"/Date(1303502009425)/">>, #opts{escape_forward_slash=true}), + <<"\\/Date(1303502009425)\\/">> ) } ]. From 732dd407472b3c6457955dc8e174ccf6a6762575 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Fri, 16 Mar 2012 15:34:57 -0700 Subject: [PATCH 08/27] remove all ?_assert and replace with ?_assertFoo's --- src/jsx_encoder.erl | 73 +++++++------ src/jsx_to_json.erl | 258 ++++++++++++++++++++------------------------ src/jsx_to_term.erl | 54 +++++----- src/jsx_verify.erl | 6 +- 4 files changed, 176 insertions(+), 215 deletions(-) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index f4d83e7..0af674c 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -118,49 +118,48 @@ encode(Term) -> (encoder(jsx, [], []))(Term). encode_test_() -> [ - {"naked string", ?_assert(encode(<<"a string">>) - =:= [{string, <<"a string">>}, end_json]) - }, - {"naked integer", ?_assert(encode(123) - =:= [{integer, 123}, end_json]) - }, - {"naked float", ?_assert(encode(1.23) - =:= [{float, 1.23}, end_json]) - }, - {"naked literal", ?_assert(encode(null) - =:= [{literal, null}, end_json]) - }, - {"empty object", ?_assert(encode([{}]) - =:= [start_object, end_object, end_json]) - }, - {"empty list", ?_assert(encode([]) - =:= [start_array, end_array, end_json]) - }, - {"simple list", ?_assert(encode([1,2,3,true,false]) - =:= [start_array, + {"naked string", ?_assertEqual(encode(<<"a string">>), [{string, <<"a string">>}, end_json])}, + {"naked integer", ?_assertEqual(encode(123), [{integer, 123}, end_json])}, + {"naked float", ?_assertEqual(encode(1.23), [{float, 1.23}, end_json])}, + {"naked literal", ?_assertEqual(encode(null), [{literal, null}, end_json])}, + {"empty object", ?_assertEqual(encode([{}]), [start_object, end_object, end_json])}, + {"empty list", ?_assertEqual(encode([]), [start_array, end_array, end_json])}, + {"simple list", ?_assertEqual( + encode([1,2,3,true,false]), + [ + start_array, {integer, 1}, {integer, 2}, {integer, 3}, {literal, true}, {literal, false}, end_array, - end_json]) + end_json + ] + ) }, - {"simple object", ?_assert(encode([{<<"a">>, true}, {<<"b">>, false}]) - =:= [start_object, + {"simple object", ?_assertEqual( + encode([{<<"a">>, true}, {<<"b">>, false}]), + [ + start_object, {key, <<"a">>}, {literal, true}, {key, <<"b">>}, {literal, false}, end_object, - end_json]) + end_json + ] + ) }, - {"complex term", ?_assert(encode([ - {<<"a">>, true}, - {<<"b">>, false}, - {<<"c">>, [1,2,3]}, - {<<"d">>, [{<<"key">>, <<"value">>}]} - ]) =:= [start_object, + {"complex term", ?_assertEqual( + encode([ + {<<"a">>, true}, + {<<"b">>, false}, + {<<"c">>, [1,2,3]}, + {<<"d">>, [{<<"key">>, <<"value">>}]} + ]), + [ + start_object, {key, <<"a">>}, {literal, true}, {key, <<"b">>}, @@ -177,14 +176,14 @@ encode_test_() -> {string, <<"value">>}, end_object, end_object, - end_json]) + end_json + ] + ) }, - {"atom keys", ?_assert(encode([{key, <<"value">>}]) - =:= [start_object, - {key, <<"key">>}, - {string, <<"value">>}, - end_object, - end_json]) + {"atom keys", ?_assertEqual( + encode([{key, <<"value">>}]), + [start_object, {key, <<"key">>}, {string, <<"value">>}, end_object, end_json] + ) } ]. diff --git a/src/jsx_to_json.erl b/src/jsx_to_json.erl index 6d012f5..0e4edf4 100644 --- a/src/jsx_to_json.erl +++ b/src/jsx_to_json.erl @@ -186,176 +186,148 @@ teardown_nicedecimal_meck(_) -> basic_format_test_() -> [ - {"empty object", ?_assert(format(<<"{}">>, []) =:= <<"{}">>)}, - {"empty array", ?_assert(format(<<"[]">>, []) =:= <<"[]">>)}, - {"naked integer", ?_assert(format(<<"123">>, []) =:= <<"123">>)}, + {"empty object", ?_assertEqual(format(<<"{}">>, []), <<"{}">>)}, + {"empty array", ?_assertEqual(format(<<"[]">>, []), <<"[]">>)}, + {"naked integer", ?_assertEqual(format(<<"123">>, []), <<"123">>)}, {foreach, fun() -> setup_nicedecimal_meck(<<"1.23">>) end, fun(R) -> teardown_nicedecimal_meck(R) end, - [{"naked float", ?_assert(format(<<"1.23">>, []) =:= <<"1.23">>)}] - }, - {"naked string", ?_assert(format(<<"\"hi\"">>, []) =:= <<"\"hi\"">>)}, - {"naked literal", ?_assert(format(<<"true">>, []) =:= <<"true">>)}, - {"simple object", - ?_assert(format(<<" { \"key\" :\n\t \"value\"\r\r\r\n } ">>, - [] - ) =:= <<"{\"key\":\"value\"}">> - ) - }, - {"really simple object", - ?_assert(format(<<"{\"k\":\"v\"}">>, []) =:= <<"{\"k\":\"v\"}">>) - }, - {"nested object", - ?_assert(format(<<"{\"k\":{\"k\":\"v\"}, \"j\":{}}">>, [] - ) =:= <<"{\"k\":{\"k\":\"v\"},\"j\":{}}">> - ) - }, - {"simple array", - ?_assert(format(<<" [\n\ttrue,\n\tfalse , \n \tnull\n] ">>, - [] - ) =:= <<"[true,false,null]">> - ) - }, - {"really simple array", ?_assert(format(<<"[1]">>, []) =:= <<"[1]">>)}, - {"nested array", ?_assert(format(<<"[[[]]]">>, []) =:= <<"[[[]]]">>)}, - {"nested structures", - ?_assert(format( - <<"[{\"key\":\"value\", - \"another key\": \"another value\", - \"a list\": [true, false] - }, - [[{}]] - ]">>, [] - ) =:= <<"[{\"key\":\"value\",\"another key\":\"another value\",\"a list\":[true,false]},[[{}]]]">> - ) + [{"naked float", ?_assertEqual(format(<<"1.23">>, []), <<"1.23">>)}] }, + {"naked string", ?_assertEqual(format(<<"\"hi\"">>, []), <<"\"hi\"">>)}, + {"naked literal", ?_assertEqual(format(<<"true">>, []), <<"true">>)}, + {"simple object", ?_assertEqual( + format(<<" { \"key\" :\n\t \"value\"\r\r\r\n } ">>, []), + <<"{\"key\":\"value\"}">> + )}, + {"really simple object", ?_assertEqual(format(<<"{\"k\":\"v\"}">>, []) , <<"{\"k\":\"v\"}">>)}, + {"nested object", ?_assertEqual( + format(<<"{\"k\":{\"k\":\"v\"}, \"j\":{}}">>, []), + <<"{\"k\":{\"k\":\"v\"},\"j\":{}}">> + )}, + {"simple array", ?_assertEqual( + format(<<" [\n\ttrue,\n\tfalse , \n \tnull\n] ">>, []), + <<"[true,false,null]">> + )}, + {"really simple array", ?_assertEqual(format(<<"[1]">>, []), <<"[1]">>)}, + {"nested array", ?_assertEqual(format(<<"[[[]]]">>, []), <<"[[[]]]">>)}, + {"nested structures", ?_assertEqual( + format(<<"[ + { + \"key\":\"value\", + \"another key\": \"another value\", + \"a list\": [true, false] + }, + [[{}]] + ]">>, []), + <<"[{\"key\":\"value\",\"another key\":\"another value\",\"a list\":[true,false]},[[{}]]]">> + )}, {"simple nested structure", - ?_assert(format(<<"[[],{\"k\":[[],{}],\"j\":{}},[]]">>, [] - ) =:= <<"[[],{\"k\":[[],{}],\"j\":{}},[]]">> + ?_assertEqual( + format(<<"[[],{\"k\":[[],{}],\"j\":{}},[]]">>, []), + <<"[[],{\"k\":[[],{}],\"j\":{}},[]]">> ) } ]. basic_to_json_test_() -> [ - {"empty object", ?_assert(to_json([{}], []) =:= <<"{}">>)}, - {"empty array", ?_assert(to_json([], []) =:= <<"[]">>)}, - {"naked integer", ?_assert(to_json(123, []) =:= <<"123">>)}, + {"empty object", ?_assertEqual(to_json([{}], []), <<"{}">>)}, + {"empty array", ?_assertEqual(to_json([], []), <<"[]">>)}, + {"naked integer", ?_assertEqual(to_json(123, []), <<"123">>)}, {foreach, fun() -> setup_nicedecimal_meck(<<"1.23">>) end, fun(R) -> teardown_nicedecimal_meck(R) end, - [{"naked float", ?_assert(to_json(1.23, []) =:= <<"1.23">>)}] + [{"naked float", ?_assertEqual(to_json(1.23, []) , <<"1.23">>)}] }, - {"naked string", ?_assert(to_json(<<"hi">>, []) =:= <<"\"hi\"">>)}, - {"naked literal", ?_assert(to_json(true, []) =:= <<"true">>)}, - {"simple object", - ?_assert(to_json( - [{<<"key">>, <<"value">>}], - [] - ) =:= <<"{\"key\":\"value\"}">> - ) - }, - {"nested object", - ?_assert(to_json( - [{<<"k">>,[{<<"k">>,<<"v">>}]},{<<"j">>,[{}]}], - [] - ) =:= <<"{\"k\":{\"k\":\"v\"},\"j\":{}}">> - ) - }, - {"simple array", - ?_assert(to_json( - [true, false, null], - [] - ) =:= <<"[true,false,null]">> - ) - }, - {"really simple array", ?_assert(to_json([1], []) =:= <<"[1]">>)}, - {"nested array", ?_assert(to_json([[[]]], []) =:= <<"[[[]]]">>)}, - {"nested structures", - ?_assert(to_json( + {"naked string", ?_assertEqual(to_json(<<"hi">>, []), <<"\"hi\"">>)}, + {"naked literal", ?_assertEqual(to_json(true, []), <<"true">>)}, + {"simple object", ?_assertEqual( + to_json( + [{<<"key">>, <<"value">>}], + [] + ), + <<"{\"key\":\"value\"}">> + )}, + {"nested object", ?_assertEqual( + to_json( + [{<<"k">>,[{<<"k">>,<<"v">>}]},{<<"j">>,[{}]}], + [] + ), + <<"{\"k\":{\"k\":\"v\"},\"j\":{}}">> + )}, + {"simple array", ?_assertEqual(to_json([true, false, null], []), <<"[true,false,null]">>)}, + {"really simple array", ?_assertEqual(to_json([1], []), <<"[1]">>)}, + {"nested array", ?_assertEqual(to_json([[[]]], []), <<"[[[]]]">>)}, + {"nested structures", ?_assertEqual( + to_json( + [ [ - [ - {<<"key">>, <<"value">>}, - {<<"another key">>, <<"another value">>}, - {<<"a list">>, [true, false]} - ], - [[[{}]]] + {<<"key">>, <<"value">>}, + {<<"another key">>, <<"another value">>}, + {<<"a list">>, [true, false]} ], - [] - ) =:= <<"[{\"key\":\"value\",\"another key\":\"another value\",\"a list\":[true,false]},[[{}]]]">> - ) - }, - {"simple nested structure", - ?_assert(to_json( - [[], [{<<"k">>, [[], [{}]]}, {<<"j">>, [{}]}], []], - [] - ) =:= <<"[[],{\"k\":[[],{}],\"j\":{}},[]]">> - ) - } + [[[{}]]] + ], + [] + ), + <<"[{\"key\":\"value\",\"another key\":\"another value\",\"a list\":[true,false]},[[{}]]]">> + )}, + {"simple nested structure", ?_assertEqual( + to_json( + [[], [{<<"k">>, [[], [{}]]}, {<<"j">>, [{}]}], []], + [] + ), + <<"[[],{\"k\":[[],{}],\"j\":{}},[]]">> + )} ]. opts_test_() -> [ - {"unspecified indent/space", - ?_assert(format(<<" [\n\ttrue,\n\tfalse,\n\tnull\n] ">>, - [space, indent] - ) =:= <<"[\n true,\n false,\n null\n]">> - ) - }, - {"specific indent/space", - ?_assert(format( - <<"\n{\n\"key\" : [],\n\"another key\" : true\n}\n">>, - [{space, 2}, {indent, 3}] - ) =:= <<"{\n \"key\": [],\n \"another key\": true\n}">> - ) - }, - {"nested structures", - ?_assert(format( - <<"[{\"key\":\"value\", - \"another key\": \"another value\" - }, - [[true, false, null]] - ]">>, - [{space, 2}, {indent, 2}] - ) =:= <<"[\n {\n \"key\": \"value\",\n \"another key\": \"another value\"\n },\n [\n [\n true,\n false,\n null\n ]\n ]\n]">> - ) - }, - {"array spaces", - ?_assert(format(<<"[1,2,3]">>, - [{space, 2}] - ) =:= <<"[1, 2, 3]">> - ) - }, - {"object spaces", - ?_assert(format(<<"{\"a\":true,\"b\":true,\"c\":true}">>, - [{space, 2}] - ) =:= <<"{\"a\": true, \"b\": true, \"c\": true}">> - ) - }, + {"unspecified indent/space", ?_assertEqual( + format(<<" [\n\ttrue,\n\tfalse,\n\tnull\n] ">>, [space, indent]), + <<"[\n true,\n false,\n null\n]">> + )}, + {"specific indent/space", ?_assertEqual( + format( + <<"\n{\n\"key\" : [],\n\"another key\" : true\n}\n">>, + [{space, 2}, {indent, 3}] + ), + <<"{\n \"key\": [],\n \"another key\": true\n}">> + )}, + {"nested structures", ?_assertEqual( + format( + <<"[{\"key\":\"value\", \"another key\": \"another value\"}, [[true, false, null]]]">>, + [{space, 2}, {indent, 2}] + ), + <<"[\n {\n \"key\": \"value\",\n \"another key\": \"another value\"\n },\n [\n [\n true,\n false,\n null\n ]\n ]\n]">> + )}, + {"array spaces", ?_assertEqual( + format(<<"[1,2,3]">>, [{space, 2}]), + <<"[1, 2, 3]">> + )}, + {"object spaces", ?_assertEqual( + format(<<"{\"a\":true,\"b\":true,\"c\":true}">>, [{space, 2}]), + <<"{\"a\": true, \"b\": true, \"c\": true}">> + )}, {foreach, fun() -> setup_nicedecimal_meck(<<"1.23">>) end, fun(R) -> teardown_nicedecimal_meck(R) end, - [{ - "array indent", - ?_assert(format(<<"[1.23, 1.23, 1.23]">>, - [{indent, 2}] - ) =:= <<"[\n 1.23,\n 1.23,\n 1.23\n]">> - ) - }] + [{"array indent", ?_assertEqual( + format(<<"[1.23, 1.23, 1.23]">>, [{indent, 2}]), + <<"[\n 1.23,\n 1.23,\n 1.23\n]">> + )}] }, - {"object indent", - ?_assert(format(<<"{\"a\":true,\"b\":true,\"c\":true}">>, - [{indent, 2}] - ) =:= <<"{\n \"a\":true,\n \"b\":true,\n \"c\":true\n}">> - ) - } + {"object indent", ?_assertEqual( + format(<<"{\"a\":true,\"b\":true,\"c\":true}">>, [{indent, 2}]), + <<"{\n \"a\":true,\n \"b\":true,\n \"c\":true\n}">> + )} ]. ext_opts_test_() -> - [{"extopts", ?_assert(format(<<"[]">>, - [loose_unicode, {escape_forward_slash, true}] - ) =:= <<"[]">> - )} - ]. + [{"extopts", ?_assertEqual( + format(<<"[]">>, [loose_unicode, {escape_forward_slash, true}]), + <<"[]">> + )}]. -endif. \ No newline at end of file diff --git a/src/jsx_to_term.erl b/src/jsx_to_term.erl index 3099829..b5b73bc 100644 --- a/src/jsx_to_term.erl +++ b/src/jsx_to_term.erl @@ -102,35 +102,29 @@ format_key(Key, Opts) -> basic_test_() -> [ - {"empty object", ?_assert(to_term(<<"{}">>, []) =:= [{}])}, - {"simple object", ?_assert(to_term(<<"{\"key\": true}">>, []) =:= [{<<"key">>, true}])}, - {"less simple object", - ?_assert(to_term(<<"{\"a\": 1, \"b\": 2}">>, []) =:= [{<<"a">>, 1}, {<<"b">>, 2}]) - }, - {"nested object", - ?_assert(to_term(<<"{\"key\": {\"key\": true}}">>, []) =:= [{<<"key">>, [{<<"key">>, true}]}]) - }, + {"empty object", ?_assertEqual(to_term(<<"{}">>, []), [{}])}, + {"simple object", ?_assertEqual(to_term(<<"{\"key\": true}">>, []), [{<<"key">>, true}])}, + {"less simple object", ?_assertEqual( + to_term(<<"{\"a\": 1, \"b\": 2}">>, []), + [{<<"a">>, 1}, {<<"b">>, 2}] + )}, + {"nested object", ?_assertEqual( + to_term(<<"{\"key\": {\"key\": true}}">>, []), + [{<<"key">>, [{<<"key">>, true}]}] + )}, {"empty array", ?_assert(to_term(<<"[]">>, []) =:= [])}, - {"list of lists", - ?_assert(to_term(<<"[[],[],[]]">>, []) =:= [[], [], []]) - }, - {"list of strings", - ?_assert(to_term(<<"[\"hi\", \"there\"]">>, []) =:= [<<"hi">>, <<"there">>]) - }, - {"list of numbers", - ?_assert(to_term(<<"[1, 2.0, 3e4, -5]">>, []) =:= [1, 2.0, 3.0e4, -5]) - }, - {"list of literals", - ?_assert(to_term(<<"[true,false,null]">>, []) =:= [true,false,null]) - }, - {"list of objects", - ?_assert(to_term(<<"[{}, {\"a\":1, \"b\":2}, {\"key\":[true,false]}]">>, []) - =:= [[{}], [{<<"a">>,1},{<<"b">>,2}], [{<<"key">>,[true,false]}]]) - } + {"list of lists", ?_assertEqual(to_term(<<"[[],[],[]]">>, []), [[], [], []])}, + {"list of strings", ?_assertEqual(to_term(<<"[\"hi\", \"there\"]">>, []), [<<"hi">>, <<"there">>])}, + {"list of numbers", ?_assertEqual(to_term(<<"[1, 2.0, 3e4, -5]">>, []), [1, 2.0, 3.0e4, -5])}, + {"list of literals", ?_assertEqual(to_term(<<"[true,false,null]">>, []), [true,false,null])}, + {"list of objects", ?_assertEqual( + to_term(<<"[{}, {\"a\":1, \"b\":2}, {\"key\":[true,false]}]">>, []), + [[{}], [{<<"a">>,1},{<<"b">>,2}], [{<<"key">>,[true,false]}]] + )} ]. comprehensive_test_() -> - {"comprehensive test", ?_assert(to_term(comp_json(), []) =:= comp_term())}. + {"comprehensive test", ?_assertEqual(to_term(comp_json(), []), comp_term())}. comp_json() -> <<"[ @@ -157,7 +151,7 @@ comp_term() -> ]. atom_labels_test_() -> - {"atom labels test", ?_assert(to_term(comp_json(), [{labels, atom}]) =:= atom_term())}. + {"atom labels test", ?_assertEqual(to_term(comp_json(), [{labels, atom}]), atom_term())}. atom_term() -> [ @@ -173,10 +167,10 @@ atom_term() -> naked_test_() -> [ - {"naked integer", ?_assert(to_term(<<"123">>, []) =:= 123)}, - {"naked float", ?_assert(to_term(<<"-4.32e-17">>, []) =:= -4.32e-17)}, - {"naked literal", ?_assert(to_term(<<"true">>, []) =:= true)}, - {"naked string", ?_assert(to_term(<<"\"string\"">>, []) =:= <<"string">>)} + {"naked integer", ?_assertEqual(to_term(<<"123">>, []), 123)}, + {"naked float", ?_assertEqual(to_term(<<"-4.32e-17">>, []), -4.32e-17)}, + {"naked literal", ?_assertEqual(to_term(<<"true">>, []), true)}, + {"naked string", ?_assertEqual(to_term(<<"\"string\"">>, []), <<"string">>)} ]. -endif. \ No newline at end of file diff --git a/src/jsx_verify.erl b/src/jsx_verify.erl index d49c90f..1a91b37 100644 --- a/src/jsx_verify.erl +++ b/src/jsx_verify.erl @@ -169,11 +169,7 @@ term_true_test_() -> {"empty array", ?_assert(is_term([], []))}, {"whitespace", ?_assert(is_term([ true ], []))}, {"nested terms", - ?_assert(is_term( - [[{x, [[{}], [{}], [{}]]}, {y, [{}]}], [{}], [[[]]]], - [] - ) - ) + ?_assert(is_term([[{x, [[{}], [{}], [{}]]}, {y, [{}]}], [{}], [[[]]]], [])) }, {"numbers", ?_assert(is_term([-1.0, -1, -0, 0, 1.0e-1, 1, 1.0, 1.0e1], [])) From 8487bcc6509a4198e4711bd248d28fec19c0a19e Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Mon, 19 Mar 2012 14:34:07 -0700 Subject: [PATCH 09/27] allow c and c++ style comments anywhere whitespace is legal --- src/jsx_decoder.erl | 267 +++++++++++++++++++++++++++++++++++++++++++- src/jsx_opts.hrl | 3 +- src/jsx_utils.erl | 5 +- 3 files changed, 271 insertions(+), 4 deletions(-) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 006cc22..eb85efa 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -77,6 +77,9 @@ decoder(Handler, State, Opts) -> -define(negative, 16#2D). -define(positive, 16#2B). +%% comments +-define(star, 16#2A). + %% some useful guards -define(is_hex(Symbol), @@ -153,6 +156,9 @@ value(<>, {Handler, State}, Stack, Opts) -> array(Rest, {Handler, Handler:handle_event(start_array, State)}, [array|Stack], Opts); value(<>, Handler, Stack, Opts) when ?is_whitespace(S) -> value(Rest, Handler, Stack, Opts); +value(<>, Handler, Stack, Opts=#opts{comments=true}) -> + Resume = fun(R, H, S, O) -> value(R, H, S, O) end, + comment(Rest, Handler, [Resume|Stack], Opts); value(<<>>, Handler, Stack, Opts) -> ?incomplete(value, <<>>, Handler, Stack, Opts); value(Bin, Handler, Stack, Opts) -> @@ -167,6 +173,9 @@ object(<>, {Handler, State}, [key|Stack], Opts) -> maybe_done(Rest, {Handler, Handler:handle_event(end_object, State)}, Stack, Opts); object(<>, Handler, Stack, Opts) when ?is_whitespace(S) -> object(Rest, Handler, Stack, Opts); +object(<>, Handler, Stack, Opts=#opts{comments=true}) -> + Resume = fun(R, H, S, O) -> object(R, H, S, O) end, + comment(Rest, Handler, [Resume|Stack], Opts); object(<<>>, Handler, Stack, Opts) -> ?incomplete(object, <<>>, Handler, Stack, Opts); object(Bin, Handler, Stack, Opts) -> @@ -196,7 +205,10 @@ array(<>, {Handler, State}, Stack, Opts) -> array(<>, {Handler, State}, [array|Stack], Opts) -> maybe_done(Rest, {Handler, Handler:handle_event(end_array, State)}, Stack, Opts); array(<>, Handler, Stack, Opts) when ?is_whitespace(S) -> - array(Rest, Handler, Stack, Opts); + array(Rest, Handler, Stack, Opts); +array(<>, Handler, Stack, Opts=#opts{comments=true}) -> + Resume = fun(R, H, S, O) -> array(R, H, S, O) end, + comment(Rest, Handler, [Resume|Stack], Opts); array(<<>>, Handler, Stack, Opts) -> ?incomplete(array, <<>>, Handler, Stack, Opts); array(Bin, Handler, Stack, Opts) -> @@ -207,6 +219,9 @@ colon(<>, Handler, [key|Stack], Opts) -> value(Rest, Handler, [object|Stack], Opts); colon(<>, Handler, Stack, Opts) when ?is_whitespace(S) -> colon(Rest, Handler, Stack, Opts); +colon(<>, Handler, Stack, Opts=#opts{comments=true}) -> + Resume = fun(R, H, S, O) -> colon(R, H, S, O) end, + comment(Rest, Handler, [Resume|Stack], Opts); colon(<<>>, Handler, Stack, Opts) -> ?incomplete(colon, <<>>, Handler, Stack, Opts); colon(Bin, Handler, Stack, Opts) -> @@ -218,7 +233,10 @@ key(<>, Handler, Stack, Opts) -> key(<>, Handler, Stack, Opts = #opts{single_quotes=true}) -> string(Rest, Handler, [?new_seq(), single_quote|Stack], Opts); key(<>, Handler, Stack, Opts) when ?is_whitespace(S) -> - key(Rest, Handler, Stack, Opts); + key(Rest, Handler, Stack, Opts); +key(<>, Handler, Stack, Opts=#opts{comments=true}) -> + Resume = fun(R, H, S, O) -> key(R, H, S, O) end, + comment(Rest, Handler, [Resume|Stack], Opts); key(<<>>, Handler, Stack, Opts) -> ?incomplete(key, <<>>, Handler, Stack, Opts); key(Bin, Handler, Stack, Opts) -> @@ -478,6 +496,9 @@ zero(<>, Handler, [Acc|Stack], Opts) -> initial_decimal(Rest, Handler, [{Acc, []}|Stack], Opts); zero(<>, {Handler, State}, [Acc|Stack], Opts) when ?is_whitespace(S) -> maybe_done(Rest, {Handler, Handler:handle_event(format_number(Acc), State)}, Stack, Opts); +zero(<>, {Handler, State}, [Acc|Stack], Opts=#opts{comments=true}) -> + Resume = fun(R, H, S, O) -> maybe_done(R, H, S, O) end, + comment(Rest, {Handler, Handler:handle_event(format_number(Acc), State)}, [Resume|Stack], Opts); zero(<<>>, {Handler, State}, [Acc|Stack], Opts = #opts{explicit_end=false}) -> maybe_done(<<>>, {Handler, Handler:handle_event(format_number(Acc), State)}, Stack, Opts); zero(<<>>, Handler, Stack, Opts) -> @@ -516,6 +537,9 @@ integer(<>, Handler, [Acc|Stack], Opts) when S =:= $e; S =:= $E e(Rest, Handler, [{Acc, [], []}|Stack], Opts); integer(<>, {Handler, State}, [Acc|Stack], Opts) when ?is_whitespace(S) -> maybe_done(Rest, {Handler, Handler:handle_event(format_number(Acc), State)}, Stack, Opts); +integer(<>, {Handler, State}, [Acc|Stack], Opts=#opts{comments=true}) -> + Resume = fun(R, H, S, O) -> maybe_done(R, H, S, O) end, + comment(Rest, {Handler, Handler:handle_event(format_number(Acc), State)}, [Resume|Stack], Opts); integer(<<>>, {Handler, State}, [Acc|Stack], Opts = #opts{explicit_end=false}) -> maybe_done(<<>>, {Handler, Handler:handle_event(format_number(Acc), State)}, Stack, Opts); integer(<<>>, Handler, Stack, Opts) -> @@ -561,6 +585,9 @@ decimal(<>, Handler, [{Int, Frac}|Stack], Opts) e(Rest, Handler, [{Int, Frac, []}|Stack], Opts); decimal(<>, {Handler, State}, [Acc|Stack], Opts) when ?is_whitespace(S) -> maybe_done(Rest, {Handler, Handler:handle_event(format_number(Acc), State)}, Stack, Opts); +decimal(<>, {Handler, State}, [Acc|Stack], Opts=#opts{comments=true}) -> + Resume = fun(R, H, S, O) -> maybe_done(R, H, S, O) end, + comment(Rest, {Handler, Handler:handle_event(format_number(Acc), State)}, [Resume|Stack], Opts); decimal(<<>>, {Handler, State}, [Acc|Stack], Opts = #opts{explicit_end=false}) -> maybe_done(<<>>, {Handler, Handler:handle_event(format_number(Acc), State)}, Stack, Opts); decimal(<<>>, Handler, Stack, Opts) -> @@ -615,6 +642,9 @@ exp(<>, {Handler, State}, [Acc, array|Stack], Opts) -> value(Rest, {Handler, Handler:handle_event(format_number(Acc), State)}, [array|Stack], Opts); exp(<>, {Handler, State}, [Acc|Stack], Opts) when ?is_whitespace(S) -> maybe_done(Rest, {Handler, Handler:handle_event(format_number(Acc), State)}, Stack, Opts); +exp(<>, {Handler, State}, [Acc|Stack], Opts=#opts{comments=true}) -> + Resume = fun(R, H, S, O) -> maybe_done(R, H, S, O) end, + comment(Rest, {Handler, Handler:handle_event(format_number(Acc), State)}, [Resume|Stack], Opts); exp(<<>>, {Handler, State}, [Acc|Stack], Opts = #opts{explicit_end=false}) -> maybe_done(<<>>, {Handler, Handler:handle_event(format_number(Acc), State)}, Stack, Opts); exp(<<>>, Handler, Stack, Opts) -> @@ -713,6 +743,48 @@ null(Bin, Handler, Stack, Opts) -> ?error([Bin, Handler, Stack, Opts]). +comment(<>, Handler, Stack, Opts) -> + single_comment(Rest, Handler, Stack, Opts); +comment(<>, Handler, Stack, Opts) -> + multi_comment(Rest, Handler, Stack, Opts); +comment(<<>>, Handler, Stack, Opts) -> + ?incomplete(comment, <<>>, Handler, Stack, Opts); +comment(Bin, Handler, Stack, Opts) -> + ?error([Bin, Handler, Stack, Opts]). + + +single_comment(<>, Handler, [Resume|Stack], Opts) -> + Resume(Rest, Handler, Stack, Opts); +single_comment(<<>>, Handler, [Resume|Stack], Opts) -> + Resume(<<>>, Handler, Stack, Opts); +single_comment(<<_S/utf8, Rest/binary>>, Handler, Stack, Opts) -> + single_comment(Rest, Handler, Stack, Opts); +single_comment(<<>>, Handler, Stack, Opts) -> + ?incomplete(single_comment, <<>>, Handler, Stack, Opts); +single_comment(Bin, Handler, Stack, Opts) -> + ?error([Bin, Handler, Stack, Opts]). + + +multi_comment(<>, Handler, Stack, Opts) -> + end_multi_comment(Rest, Handler, Stack, Opts); +multi_comment(<<_S/utf8, Rest/binary>>, Handler, Stack, Opts) -> + multi_comment(Rest, Handler, Stack, Opts); +multi_comment(<<>>, Handler, Stack, Opts) -> + ?incomplete(multi_comment, <<>>, Handler, Stack, Opts); +multi_comment(Bin, Handler, Stack, Opts) -> + ?error([Bin, Handler, Stack, Opts]). + + +end_multi_comment(<>, Handler, [Resume|Stack], Opts) -> + Resume(Rest, Handler, Stack, Opts); +end_multi_comment(<<_S/utf8, Rest/binary>>, Handler, Stack, Opts) -> + multi_comment(Rest, Handler, Stack, Opts); +end_multi_comment(<<>>, Handler, Stack, Opts) -> + ?incomplete(end_multi_comment, <<>>, Handler, Stack, Opts); +end_multi_comment(Bin, Handler, Stack, Opts) -> + ?error([Bin, Handler, Stack, Opts]). + + maybe_done(<>, {Handler, State}, [object|Stack], Opts) -> maybe_done(Rest, {Handler, Handler:handle_event(end_object, State)}, Stack, Opts); maybe_done(<>, {Handler, State}, [array|Stack], Opts) -> @@ -723,6 +795,9 @@ maybe_done(<>, Handler, [array|_] = Stack, Opts) -> value(Rest, Handler, Stack, Opts); maybe_done(<>, Handler, Stack, Opts) when ?is_whitespace(S) -> maybe_done(Rest, Handler, Stack, Opts); +maybe_done(<>, Handler, Stack, Opts=#opts{comments=true}) -> + Resume = fun(R, H, S, O) -> maybe_done(R, H, S, O) end, + comment(Rest, Handler, [Resume|Stack], Opts); maybe_done(<<>>, Handler, Stack, Opts) when length(Stack) > 0 -> ?incomplete(maybe_done, <<>>, Handler, Stack, Opts); maybe_done(Rest, {Handler, State}, [], Opts) -> @@ -733,6 +808,9 @@ maybe_done(Bin, Handler, Stack, Opts) -> done(<>, Handler, [], Opts) when ?is_whitespace(S) -> done(Rest, Handler, [], Opts); +done(<>, Handler, [], Opts=#opts{comments=true}) -> + Resume = fun(R, H, S, O) -> done(R, H, S, O) end, + comment(Rest, Handler, [Resume], Opts); done(<<>>, {Handler, State}, [], Opts = #opts{explicit_end=true}) -> {incomplete, fun(Stream) when is_binary(Stream) -> done(<>, {Handler, State}, [], Opts) @@ -748,6 +826,191 @@ done(Bin, Handler, Stack, Opts) -> ?error([Bin, Handler, Stack, Opts]). -include_lib("eunit/include/eunit.hrl"). +comments_test_() -> + [ + {"preceeding // comment", ?_assertEqual( + decode(<<"// comment ", ?newline, "[]">>, [comments]), + [start_array, end_array, end_json] + )}, + {"preceeding /**/ comment", ?_assertEqual( + decode(<<"/* comment */[]">>, [comments]), + [start_array, end_array, end_json] + )}, + {"trailing // comment", ?_assertEqual( + decode(<<"[]// comment", ?newline>>, [comments]), + [start_array, end_array, end_json] + )}, + {"trailing // comment (no newline)", ?_assertEqual( + decode(<<"[]// comment">>, [comments]), + [start_array, end_array, end_json] + )}, + {"trailing /**/ comment", ?_assertEqual( + decode(<<"[] /* comment */">>, [comments]), + [start_array, end_array, end_json] + )}, + {"// comment inside array", ?_assertEqual( + decode(<<"[ // comment", ?newline, "]">>, [comments]), + [start_array, end_array, end_json] + )}, + {"/**/ comment inside array", ?_assertEqual( + decode(<<"[ /* comment */ ]">>, [comments]), + [start_array, end_array, end_json] + )}, + {"// comment at beginning of array", ?_assertEqual( + decode(<<"[ // comment", ?newline, "true", ?newline, "]">>, [comments]), + [start_array, {literal, true}, end_array, end_json] + )}, + {"/**/ comment at beginning of array", ?_assertEqual( + decode(<<"[ /* comment */ true ]">>, [comments]), + [start_array, {literal, true}, end_array, end_json] + )}, + {"// comment at end of array", ?_assertEqual( + decode(<<"[ true // comment", ?newline, "]">>, [comments]), + [start_array, {literal, true}, end_array, end_json] + )}, + {"/**/ comment at end of array", ?_assertEqual( + decode(<<"[ true /* comment */ ]">>, [comments]), + [start_array, {literal, true}, end_array, end_json] + )}, + {"// comment midarray (post comma)", ?_assertEqual( + decode(<<"[ true, // comment", ?newline, "false ]">>, [comments]), + [start_array, {literal, true}, {literal, false}, end_array, end_json] + )}, + {"/**/ comment midarray (post comma)", ?_assertEqual( + decode(<<"[ true, /* comment */ false ]">>, [comments]), + [start_array, {literal, true}, {literal, false}, end_array, end_json] + )}, + {"// comment midarray (pre comma)", ?_assertEqual( + decode(<<"[ true// comment", ?newline, ", false ]">>, [comments]), + [start_array, {literal, true}, {literal, false}, end_array, end_json] + )}, + {"/**/ comment midarray (pre comma)", ?_assertEqual( + decode(<<"[ true/* comment */, false ]">>, [comments]), + [start_array, {literal, true}, {literal, false}, end_array, end_json] + )}, + {"// comment inside object", ?_assertEqual( + decode(<<"{ // comment", ?newline, "}">>, [comments]), + [start_object, end_object, end_json] + )}, + {"/**/ comment inside object", ?_assertEqual( + decode(<<"{ /* comment */ }">>, [comments]), + [start_object, end_object, end_json] + )}, + {"// comment at beginning of object", ?_assertEqual( + decode(<<"{ // comment", ?newline, " \"key\": true", ?newline, "}">>, [comments]), + [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json] + )}, + {"/**/ comment at beginning of object", ?_assertEqual( + decode(<<"{ /* comment */ \"key\": true }">>, [comments]), + [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json] + )}, + {"// comment at end of object", ?_assertEqual( + decode(<<"{ \"key\": true // comment", ?newline, "}">>, [comments]), + [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json] + )}, + {"/**/ comment at end of object", ?_assertEqual( + decode(<<"{ \"key\": true /* comment */ }">>, [comments]), + [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json] + )}, + {"// comment midobject (post comma)", ?_assertEqual( + decode(<<"{ \"x\": true, // comment", ?newline, "\"y\": false }">>, [comments]), + [ + start_object, + {key, <<"x">>}, + {literal, true}, + {key, <<"y">>}, + {literal, false}, + end_object, + end_json + ] + )}, + {"/**/ comment midobject (post comma)", ?_assertEqual( + decode(<<"{ \"x\": true, /* comment */", ?newline, "\"y\": false }">>, [comments]), + [ + start_object, + {key, <<"x">>}, + {literal, true}, + {key, <<"y">>}, + {literal, false}, + end_object, + end_json + ] + )}, + {"// comment midobject (pre comma)", ?_assertEqual( + decode(<<"{ \"x\": true// comment", ?newline, ", \"y\": false }">>, [comments]), + [ + start_object, + {key, <<"x">>}, + {literal, true}, + {key, <<"y">>}, + {literal, false}, + end_object, + end_json + ] + )}, + {"/**/ comment midobject (pre comma)", ?_assertEqual( + decode(<<"{ \"x\": true/* comment */", ?newline, ", \"y\": false }">>, [comments]), + [ + start_object, + {key, <<"x">>}, + {literal, true}, + {key, <<"y">>}, + {literal, false}, + end_object, + end_json + ] + )}, + {"// comment precolon", ?_assertEqual( + decode(<<"{ \"key\" // comment", ?newline, ": true }">>, [comments]), + [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json] + )}, + {"/**/ comment precolon", ?_assertEqual( + decode(<<"{ \"key\"/* comment */: true }">>, [comments]), + [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json] + )}, + {"// comment postcolon", ?_assertEqual( + decode(<<"{ \"key\": // comment", ?newline, " true }">>, [comments]), + [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json] + )}, + {"/**/ comment postcolon", ?_assertEqual( + decode(<<"{ \"key\":/* comment */ true }">>, [comments]), + [start_object, {key, <<"key">>}, {literal, true}, end_object, end_json] + )}, + {"// comment terminating zero", ?_assertEqual( + decode(<<"[ 0// comment", ?newline, "]">>, [comments]), + [start_array, {integer, 0}, end_array, end_json] + )}, + {"// comment terminating integer", ?_assertEqual( + decode(<<"[ 1// comment", ?newline, "]">>, [comments]), + [start_array, {integer, 1}, end_array, end_json] + )}, + {"// comment terminating float", ?_assertEqual( + decode(<<"[ 1.0// comment", ?newline, "]">>, [comments]), + [start_array, {float, 1.0}, end_array, end_json] + )}, + {"// comment terminating exp", ?_assertEqual( + decode(<<"[ 1e1// comment", ?newline, "]">>, [comments]), + [start_array, {float, 1.0e1}, end_array, end_json] + )}, + {"/**/ comment terminating zero", ?_assertEqual( + decode(<<"[ 0/* comment */ ]">>, [comments]), + [start_array, {integer, 0}, end_array, end_json] + )}, + {"/**/ comment terminating integer", ?_assertEqual( + decode(<<"[ 1/* comment */ ]">>, [comments]), + [start_array, {integer, 1}, end_array, end_json] + )}, + {"/**/ comment terminating float", ?_assertEqual( + decode(<<"[ 1.0/* comment */ ]">>, [comments]), + [start_array, {float, 1.0}, end_array, end_json] + )}, + {"/**/ comment terminating exp", ?_assertEqual( + decode(<<"[ 1e1/* comment */ ]">>, [comments]), + [start_array, {float, 1.0e1}, end_array, end_json] + )} + ]. + + noncharacters_test_() -> [ {"noncharacters - badjson", diff --git a/src/jsx_opts.hrl b/src/jsx_opts.hrl index 938b8fb..7741585 100644 --- a/src/jsx_opts.hrl +++ b/src/jsx_opts.hrl @@ -3,5 +3,6 @@ escape_forward_slash = false, explicit_end = false, single_quotes = false, - no_jsonp_escapes = false + no_jsonp_escapes = false, + comments = false }). \ No newline at end of file diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index 2ab87e9..21f74d2 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -47,6 +47,8 @@ parse_opts([single_quotes|Rest], Opts) -> parse_opts(Rest, Opts#opts{single_quotes=true}); parse_opts([no_jsonp_escapes|Rest], Opts) -> parse_opts(Rest, Opts#opts{no_jsonp_escapes=true}); +parse_opts([comments|Rest], Opts) -> + parse_opts(Rest, Opts#opts{comments=true}); parse_opts(_, _) -> {error, badarg}. @@ -57,7 +59,8 @@ valid_flags() -> escape_forward_slash, explicit_end, single_quotes, - no_jsonp_escapes + no_jsonp_escapes, + comments ]. From e852286e9b21d8bc19380cf3f778d3338ee96f99 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Mon, 19 Mar 2012 15:57:00 -0700 Subject: [PATCH 10/27] apply escape_forward_slash option to decoding as well as encoding --- priv/test_cases/string_escapes.json | 2 +- priv/test_cases/string_escapes.test | 1 - src/jsx_decoder.erl | 22 +++++++++++++++++----- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/priv/test_cases/string_escapes.json b/priv/test_cases/string_escapes.json index 461bc67..3c9af78 100644 --- a/priv/test_cases/string_escapes.json +++ b/priv/test_cases/string_escapes.json @@ -1 +1 @@ -["\"", "\\", "\/", "\b", "\f", "\n", "\r", "\t"] \ No newline at end of file +["\"", "\\", "\b", "\f", "\n", "\r", "\t"] \ No newline at end of file diff --git a/priv/test_cases/string_escapes.test b/priv/test_cases/string_escapes.test index 7cd460c..8f6eeed 100644 --- a/priv/test_cases/string_escapes.test +++ b/priv/test_cases/string_escapes.test @@ -2,7 +2,6 @@ {jsx, [start_array, {string,<<"\"">>}, {string,<<"\\">>}, - {string,<<"/">>}, {string,<<"\b">>}, {string,<<"\f">>}, {string,<<"\n">>}, diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index eb85efa..4055b2e 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -348,13 +348,16 @@ escape(<<$r, Rest/binary>>, Handler, [Acc|Stack], Opts) -> string(Rest, Handler, [?acc_seq(Acc, $\r)|Stack], Opts); escape(<<$t, Rest/binary>>, Handler, [Acc|Stack], Opts) -> string(Rest, Handler, [?acc_seq(Acc, $\t)|Stack], Opts); -escape(<<$u, Rest/binary>>, Handler, Stack, Opts) -> - escaped_unicode(Rest, Handler, [?new_seq()|Stack], Opts); -escape(<>, Handler, [Acc|Stack], Opts) - when S =:= ?doublequote; S =:= ?solidus; S =:= ?rsolidus -> - string(Rest, Handler, [?acc_seq(Acc, S)|Stack], Opts); +escape(<>, Handler, [Acc|Stack], Opts) -> + string(Rest, Handler, [?acc_seq(Acc, $\\)|Stack], Opts); +escape(<>, Handler, [Acc|Stack], Opts=#opts{escape_forward_slash=true}) -> + string(Rest, Handler, [?acc_seq(Acc, $/)|Stack], Opts); +escape(<>, Handler, [Acc|Stack], Opts) -> + string(Rest, Handler, [?acc_seq(Acc, $\")|Stack], Opts); escape(<>, Handler, [Acc|Stack], Opts = #opts{single_quotes=true}) -> string(Rest, Handler, [?acc_seq(Acc, ?singlequote)|Stack], Opts); +escape(<<$u, Rest/binary>>, Handler, Stack, Opts) -> + escaped_unicode(Rest, Handler, [?new_seq()|Stack], Opts); escape(<<>>, Handler, Stack, Opts) -> ?incomplete(escape, <<>>, Handler, Stack, Opts); escape(Bin, Handler, Stack, Opts) -> @@ -1011,6 +1014,15 @@ comments_test_() -> ]. +escape_forward_slash_test_() -> + [ + {"escape forward slash test", ?_assertEqual( + decode(<<"[ \" \/ \" ]">>, [escape_forward_slash]), + [start_array, {string, <<" / ">>}, end_array, end_json] + )} + ]. + + noncharacters_test_() -> [ {"noncharacters - badjson", From 5e2076065628d6440de0892265362fa42412bdbe Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Mon, 19 Mar 2012 16:01:58 -0700 Subject: [PATCH 11/27] apply loose_unicode option to decoder --- src/jsx_utils.erl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index 21f74d2..2c4526e 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -132,6 +132,8 @@ json_escape(<>, Opts, Acc) %% any other legal codepoint json_escape(<>, Opts, Acc) -> json_escape(Rest, Opts, <>); +json_escape(<<_, Rest/binary>>, Opts=#opts{loose_unicode=true}, Acc) -> + json_escape(Rest, Opts, <>); json_escape(<<>>, _Opts, Acc) -> Acc; json_escape(Rest, Opts, Acc) -> @@ -173,6 +175,12 @@ binary_escape_test_() -> <<"\\u0001\\u0002\\u0003\\u000b\\u001a\\u001e\\u001f">> ) }, + {"json string loose unicode escaping", + ?_assertEqual( + json_escape(<<16#ffff>>, #opts{loose_unicode=true}), + <<16#fffd/utf8>> + ) + }, {"jsonp protection", ?_assertEqual( json_escape(<<226, 128, 168, 226, 128, 169>>, #opts{}), From a8254887f5024f6a1a866f2f74e55d21923258b1 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 20 Mar 2012 19:36:27 -0700 Subject: [PATCH 12/27] fixes wrongheaded and stupid escaping of strings --- src/jsx_encoder.erl | 85 +++++++++++++++++++++++++++++++++++++++++---- src/jsx_utils.erl | 22 +++--------- 2 files changed, 82 insertions(+), 25 deletions(-) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 0af674c..f97bf02 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -25,7 +25,6 @@ -export([encoder/3]). - -spec encoder(Handler::module(), State::any(), Opts::jsx:opts()) -> jsx:encoder(). encoder(Handler, State, Opts) -> @@ -54,7 +53,7 @@ start(Term, {Handler, State}, Opts) -> value(String, {Handler, State}, Opts) when is_binary(String) -> - Handler:handle_event({string, escape(String, {Handler, State}, Opts)}, State); + Handler:handle_event({string, check_string(String, {Handler, State}, Opts)}, State); value(Float, {Handler, State}, _Opts) when is_float(Float) -> Handler:handle_event({float, Float}, State); value(Int, {Handler, State}, _Opts) when is_integer(Int) -> @@ -84,7 +83,7 @@ object([{Key, Value}|Rest], {Handler, State}, Opts) -> Handler, value( Value, - {Handler, Handler:handle_event({key, escape(fix_key(Key), {Handler, State}, Opts)}, State)}, + {Handler, Handler:handle_event({key, check_string(fix_key(Key), {Handler, State}, Opts)}, State)}, Opts ) }, @@ -104,10 +103,70 @@ fix_key(Key) when is_atom(Key) -> fix_key(atom_to_binary(Key, utf8)); fix_key(Key) when is_binary(Key) -> Key. -escape(String, Handler, Opts) -> - try jsx_utils:json_escape(String, Opts) - catch error:badarg -> erlang:error(badarg, [String, Handler, Opts]) - end. +check_string(String, Handler, Opts) -> + case check_string(String) of + true -> String; + false -> + case Opts#opts.loose_unicode of + true -> clean_string(String, <<>>); + false -> erlang:error(badarg, [String, Handler, Opts]) + end + end. + +check_string(<>) when C < 16#fdd0 -> + check_string(Rest); +check_string(<>) when C > 16#fdef, C < 16#fffe -> + check_string(Rest); +check_string(<>) + when C =/= 16#fffe andalso C =/= 16#ffff andalso + C =/= 16#1fffe andalso C =/= 16#1ffff andalso + C =/= 16#2fffe andalso C =/= 16#2ffff andalso + C =/= 16#3fffe andalso C =/= 16#3ffff andalso + C =/= 16#4fffe andalso C =/= 16#4ffff andalso + C =/= 16#5fffe andalso C =/= 16#5ffff andalso + C =/= 16#6fffe andalso C =/= 16#6ffff andalso + C =/= 16#7fffe andalso C =/= 16#7ffff andalso + C =/= 16#8fffe andalso C =/= 16#8ffff andalso + C =/= 16#9fffe andalso C =/= 16#9ffff andalso + C =/= 16#afffe andalso C =/= 16#affff andalso + C =/= 16#bfffe andalso C =/= 16#bffff andalso + C =/= 16#cfffe andalso C =/= 16#cffff andalso + C =/= 16#dfffe andalso C =/= 16#dffff andalso + C =/= 16#efffe andalso C =/= 16#effff andalso + C =/= 16#ffffe andalso C =/= 16#fffff andalso + C =/= 16#10fffe andalso C =/= 16#10ffff -> + check_string(Rest); +check_string(<<>>) -> true; +check_string(<<_, _/binary>>) -> false. + +clean_string(<>, Acc) when C >= 16#fdd0, C =< 16#fdef -> + io:format("1: ~p~n", [C]), + clean_string(Rest, <>); +clean_string(<>, Acc) + when C == 16#fffe orelse C == 16#ffff orelse + C == 16#1fffe orelse C == 16#1ffff orelse + C == 16#2fffe orelse C == 16#2ffff orelse + C == 16#3fffe orelse C == 16#3ffff orelse + C == 16#4fffe orelse C == 16#4ffff orelse + C == 16#5fffe orelse C == 16#5ffff orelse + C == 16#6fffe orelse C == 16#6ffff orelse + C == 16#7fffe orelse C == 16#7ffff orelse + C == 16#8fffe orelse C == 16#8ffff orelse + C == 16#9fffe orelse C == 16#9ffff orelse + C == 16#afffe orelse C == 16#affff orelse + C == 16#bfffe orelse C == 16#bffff orelse + C == 16#cfffe orelse C == 16#cffff orelse + C == 16#dfffe orelse C == 16#dffff orelse + C == 16#efffe orelse C == 16#effff orelse + C == 16#ffffe orelse C == 16#fffff orelse + C == 16#10fffe orelse C == 16#10ffff -> + io:format("2: ~p~n", [C]), + clean_string(Rest, <>); +clean_string(<>, Acc) -> + io:format("3: ~p~n", [C]), + clean_string(Rest, <>); +clean_string(<<>>, Acc) -> Acc. + -ifdef(TEST). @@ -115,6 +174,8 @@ escape(String, Handler, Opts) -> encode(Term) -> (encoder(jsx, [], []))(Term). +encode(Term, Opts) -> (encoder(jsx, [], Opts))(Term). + encode_test_() -> [ @@ -184,6 +245,16 @@ encode_test_() -> encode([{key, <<"value">>}]), [start_object, {key, <<"key">>}, {string, <<"value">>}, end_object, end_json] ) + }, + {"bad string", ?_assertError( + badarg, + encode([<<"a bad string: ", 16#ffff/utf8>>]) + ) + }, + {"allow bad string", ?_assertEqual( + encode([<<"a bad string: ", 16#1ffff/utf8>>], [loose_unicode]), + [start_array, {string, <<"a bad string: ", 16#fffd/utf8>>}, end_array, end_json] + ) } ]. diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index 2c4526e..0ca6e8c 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -110,10 +110,7 @@ json_escape(<<$\t, Rest/binary>>, Opts, Acc) -> json_escape(Rest, Opts, <>); %% other control characters json_escape(<>, Opts, Acc) when C >= 0, C < $\s -> - json_escape(Rest, - Opts, - <> - ); + json_escape(Rest, Opts, <>); %% escape forward slashes -- optionally -- to faciliate microsoft's retarded %% date format json_escape(<<$/, Rest/binary>>, Opts=#opts{escape_forward_slash=true}, Acc) -> @@ -125,19 +122,14 @@ json_escape(<>, Opts=#opts{no_jsonp_escapes=true}, Acc) %% escape u+2028 and u+2029 to avoid problems with jsonp json_escape(<>, Opts, Acc) when C == 16#2028; C == 16#2029 -> - json_escape(Rest, - Opts, - <> - ); + json_escape(Rest, Opts, <>); %% any other legal codepoint json_escape(<>, Opts, Acc) -> json_escape(Rest, Opts, <>); -json_escape(<<_, Rest/binary>>, Opts=#opts{loose_unicode=true}, Acc) -> - json_escape(Rest, Opts, <>); json_escape(<<>>, _Opts, Acc) -> Acc; -json_escape(Rest, Opts, Acc) -> - erlang:error(badarg, [Rest, Opts, Acc]). +json_escape(Bin, Opts, Acc) -> + erlang:error(badarg, [Bin, Opts, Acc]). %% convert a codepoint to it's \uXXXX equiv. @@ -175,12 +167,6 @@ binary_escape_test_() -> <<"\\u0001\\u0002\\u0003\\u000b\\u001a\\u001e\\u001f">> ) }, - {"json string loose unicode escaping", - ?_assertEqual( - json_escape(<<16#ffff>>, #opts{loose_unicode=true}), - <<16#fffd/utf8>> - ) - }, {"jsonp protection", ?_assertEqual( json_escape(<<226, 128, 168, 226, 128, 169>>, #opts{}), From 6f63b1183fbb8dbd253f2c6fa2af758e65ac4b94 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 20 Mar 2012 19:37:02 -0700 Subject: [PATCH 13/27] largely rewritten readme, hopefully more useful --- README.markdown | 314 +++++++++++++++++++++++++++++++----------------- 1 file changed, 201 insertions(+), 113 deletions(-) diff --git a/README.markdown b/README.markdown index 28715b1..b8da7de 100644 --- a/README.markdown +++ b/README.markdown @@ -1,209 +1,297 @@ -## jsx (v1.0) ## +# jsx (v1.0) # -a sane json implementation for erlang, inspired by [yajl][yajl] +a sane [json][json] implementation for erlang, inspired by [yajl][yajl] copyright 2011, 2012 alisdair sullivan jsx is released under the terms of the [MIT][MIT] license -jsx uses [rebar][rebar] and [meck][meck] +jsx uses [rebar][rebar] for it's build chain and [meck][meck] for it's test suite -[![Build Status](https://secure.travis-ci.org/talentdeficit/jsx.png?branch=master)](http://travis-ci.org/talentdeficit/jsx) +[![Build Status](https://secure.travis-ci.org/talentdeficit/jsx.png?branch=develop)](http://travis-ci.org/talentdeficit/jsx) -## api ## +## index ## + +* #### [introduction](#intro) #### +* #### [quickstart](#quickstart) #### +* #### [the api](#api) #### + - [json <-> erlang mapping](#mapping) + - [options](#options) + - [incomplete input](#incompletes) + - [the encoder and decoder](#core) + - [handler callbacks](#handler) + - [converting json to erlang and vice versa](#convert) + - [formatting and minifying json text](#format) + - [verifying json and terms are valid input](#verify) + * #### [acknowledgments](#thanks) #### -**converting json to erlang terms** -parses a JSON text (a utf8 encoded binary) and produces an erlang term (see json <-> erlang mapping details below) +## quickstart ## + +to build the library: `rebar compile` + +to convert a utf8 binary containing a json string into an erlang term: `jsx:to_term(JSON)` + +to convert an erlang term into a utf8 binary containing a json string: `jsx:to_json(Term)` + +to check if a binary is valid json: `jsx:is_json(JSON)` + +to check if a term is valid json: `jsx:is_term(Term)` + +to minify a json string: `jsx:format(JSON)` + + +## api ## + + +### json <-> erlang mapping ### + +**json** | **erlang** +--------------------------------|-------------------------------- +`number` | `integer()` and `float()` +`string` | `binary()` +`true`, `false` and `null` | `true`, `false` and `null` +`array` | `[]` and `[JSON]` +`object` | `[{}]` and `[{binary() OR atom(), JSON}]` + +#### json #### + +json must be a binary encoded in `utf8`. if it's invalid `utf8` or invalid json, it probably won't parse without errors. there are a few non-standard extensions to the parser available that may change that, they are detailed in the options section below + +jsx also supports json fragments; valid json values that are not complete json. that means jsx will parse things like `<<"1">`, `<<"true">>` and `<<"\"hello world\"">>` without problems + +#### erlang #### + +only the erlang terms in the table above are supported. non supported terms result in badarg errors. jsx is never going to support erlang lists instead of binaries, mostly because you can't discriminate between lists of integers and strings without hinting, and hinting is silly + +#### numbers #### + +javascript and thus json represent all numeric values with floats. as this is woefully insufficient for many uses, **jsx**, just like erlang, supports bigints. whenever possible, this library will interpret json numbers that look like integers as integers. other numbers will be converted to erlang's floating point type, which is nearly but not quite iee754. negative zero is not representable in erlang (zero is unsigned in erlang and `0` is equivalent to `-0`) and will be interpreted as regular zero. numbers not representable are beyond the concern of this implementation, and will result in parsing errors + +when converting from erlang to json, numbers are represented with their shortest representation that will round trip without loss of precision. this means that some floats may be superficially dissimilar (although functionally equivalent). for example, `1.0000000000000001` will be represented by `1.0` + +#### strings #### + +the [json spec][rfc4627] is frustratingly vague on the exact details of json strings. json must be unicode, but no encoding is specified. javascript explicitly allows strings containing codepoints explicitly disallowed by unicode. json allows implementations to set limits on the content of strings and other implementations attempt to resolve this in various ways. this implementation, in default operation, only accepts strings that meet the constraints set out in the json spec (properly escaped control characters and quotes) and that are encoded in `utf8`. in the interests of pragmatism, there is an option for looser parsing, see options below + +all erlang strings are represented by *valid* `utf8` encoded binaries + +this implementation performs no normalization on strings beyond that detailed here. be careful when comparing strings as equivalent strings may have different `utf8` encodings + +#### true, false and null #### + +the json primitives `true`, `false` and `null` are represented by the erlang atoms `true`, `false` and `null`. surprise + +#### arrays #### + +json arrays are represented with erlang lists of json values as described in this section + +#### objects #### + +json objects are represented by erlang proplists. the empty object has the special representation `[{}]` to differentiate it from the empty list. ambiguities like `[true, false]` prevent using the shorthand form of property lists using atoms as properties. all properties must be tuples. all keys must be encoded as in `string`, above, or as atoms (which will be escaped and converted to binaries for presentation to handlers) + + +### options ### + +jsx functions all take a common set of options. not all flags have meaning in all contexts, but they are always valid options. flags are always atoms and have no value. functions may have additional options beyond these, see individual function documentation for details + +#### `loose_unicode` #### + +json text input and json strings SHOULD be utf8 encoded binaries, appropriately escaped as per the json spec. if this option is present attempts are made to replace invalid codepoints with `u+FFFD` as per the unicode spec + +#### `escape_forward_slash` #### + +json strings are escaped according to the json spec. this means forward slashes are never escaped. unfortunately, a microsoft implementation of json uses escaped forward slashes in json formatted date strings. without this option it is impossible to get date strings that some microsoft tools understand + +#### `explicit_end` #### + +this option treats all exhausted inputs as incomplete, as explained below. the parser will not attempt to return a final state until the function is called with the value `end_stream` + +#### `single_quotes` #### + +some parsers allow double quotes (`u+0022`) to be replaced by single quotes (`u+0027`) to deliminate keys and strings. this option allows json containing single quotes as structural (deliminator) characters to be parsed without errors. note that the parser expects strings to be terminated by the same quote type that opened it and that single quotes must, obviously, be escaped within strings deliminated by single quotes. the parser will never emit json with keys or strings deliminated by single quotes + +#### `no_jsonp_escapes` #### + +javascript interpreters treat the codepoints `u+2028` and `u+2029` as significant whitespace. json strings that contain either of these codepoints will be parsed incorrectly by some javascript interpreters. by default, these codepoints are escaped (to `"\u2028"` and `\u2029`, respectively) to retain compatibility. this option simply removes that escaping if, for some reason, you object to this + +#### `comments` #### + +json has no official comments but some parsers allow c style comments. this flag allows comments (both `// ...` and `/* ... */` style) anywhere whitespace is allowed + + +### incomplete input ### + +jsx handles incomplete json texts. if a partial json text is parsed, rather than returning a term from your callback handler, jsx returns `{incomplete, F}` where `F` is a function with an identical API to the anonymous fun returned from `decoder/3`. it retains the internal state of the parser at the point where input was exhausted. this allows you to parse as you stream json over a socket or file descriptor or to parse large json texts without needing to keep them entirely in memory + +however, it is important to recognize that jsx is greedy by default. if input is exhausted and the json text is not unambiguously incomplete jsx will consider the parsing complete. this is mostly relevant when parsing bare numbers like `<<"1234">>`. this could be a complete json integer or just the beginning of a json integer that is being parsed incrementally. jsx will treat it as a whole integer. the option `explicit_end` can be used to modify this behaviour, see above + + +### the encoder and decoder ### + +jsx is built on top of two finite state automata, one that handles json texts and one that handles erlang terms. both take a callback module as an argument that acts similar to a fold over a list of json 'events'. these events and the handler module's callbacks are detailed in the next section + +`jsx:decoder/3` and `jsx:encoder/3` are the entry points for the decoder and encoder, respectively + +`decoder(Handler, InitialState, Opts)` -> `Fun((JSON) -> Any)` +`encoder(Handler, InitialState, Opts)` -> `Fun((Term) -> Any)` + +types: + +- `Handler` = `atom()`, should be the name of a callback module, see below +- `InitialState` = `term()`, passed as is to `Handler:init/1` +- `Opts` = see above +- `JSON` = `utf8` encoded json text +- `Term` = an erlang term as specified above in the mapping section +- `Any` = `term()` + +decoder returns an anonymous function that handles binary json input and encoder returns an anonymous function that handles erlang term input. these are safe to reuse for multiple inputs + + +### handler callbacks ### + +`Handler` should export the following pair of functions + +`Handler:init(InitialState)` -> `State` +`Handler:handle_event(Event, State)` -> `NewState` + +types: + +- `InitialState`, `State`, `NewState` = any erlang term +- `Event` = + * `start_object` + * `end_object` + * `start_array` + * `end_array` + * `end_json` + * `{key, binary()}` + * `{string, binary()}` + * `{integer, integer()}` + * `{float, float()}` + * `{literal, true}` + * `{literal, false}` + * `{literal, null}` + +`init/1` is called with the `initialState` argument from `decoder/3` or `encoder/3` and should take care of any initialization your handler requires and return a new state + +`handle_event/2` is called for each `Event` emitted by the decoder/encoder with the output of the previous `handle_event/2` call (or `init/1` call, if `handle_event/2` has not yet been called) + +the event `end_json` will always be the last event emitted, you should take care of any cleanup in `handle_event/2` when encountering `end_json`. the state returned from this call will be returned as the final result of the anonymous function + +both `key` and `string` are `utf8` encoded binaries with all escaped values converted into the appropriate codepoints + + +### converting json to erlang and vice versa ### + +#### converting json to erlang terms #### + +`to_term` parses a JSON text (a utf8 encoded binary) and produces an erlang term (see json <-> erlang mapping details above) `to_term(JSON)` -> `Term` - `to_term(JSON, Opts)` -> `Term` types: -* `JSON` = `binary()` -* `Term` = `[]` | `[{}]` | `[Value]` | `[{Label, Value}]` | `{incomplete, Fun}` -* `Value` = `binary()` | `integer()` | `float()` | `true` | `false` | `null` -* `Label` = `binary()` | `atom()` -* `Fun` = `fun(JSON)` -> `Term` -* `Opts` = `[]` | `[Opt]` +* `JSON` = as above in the mapping section +* `Term` = as above in the mapping section +* `Opts` = as above in the opts section, but see also additional opts below * `Opt` = - - `loose_unicode` - - `single_quotes` - `labels` - `{labels, Label}` - `Label` = * `binary` * `atom` * `existing_atom` - - `explicit_end` - -`JSON` SHOULD be a utf8 encoded binary. if the option `loose_unicode` is present attempts are made to replace invalid codepoints with `u+FFFD` but badly encoded binaries may, in either case, result in `badarg` errors - -valid json strings are deliminated by double quotes, but some implementations allow single quotes in their place. the `single_quotes` option recognizes json texts with single quotes in the place of double quotes as valid. please be aware that if you enable this option, you MUST escape single quotes in keys and strings the option `labels` controls how keys are converted from json to erlang terms. `binary` does no conversion beyond normal escaping. `atom` converts keys to erlang atoms, and results in a badarg error if keys fall outside the range of erlang atoms. `existing_atom` is identical to `atom`, except it will not add new atoms to the atom table -see the note below about streaming mode for details of `explicit_end` - -**converting erlang terms to json** +#### converting erlang terms to json #### -produces a JSON text from an erlang term (see json <-> erlang mapping details below) +`to_json` parses an erlang term and produces a JSON text (see json <-> erlang mapping details below) `to_json(Term)` -> `JSON` - `to_json(Term, Opts)` -> `JSON` types: -* `JSON` = `binary()` -* `Term` = `[]` | `[{}]` | `[Value]` | `[{Label, Value}]` | `{incomplete, Fun}` -* `Value` = `binary()` | `integer()` | `float()` | `true` | `false` | `null` -* `Label` = `binary()` | `atom()` -* `Opts` = `[]` | `[Opt]` +* `JSON` = as above in the mapping section +* `Term` = as above in the mapping section +* `Opts` = as above in the opts section, but see also additional opts below * `Opt` = - `space` - `{space, N}` - `indent` - `{indent, N}` - - `escape_forward_slash` the option `{space, N}` inserts `N` spaces after every comma and colon in your json output. `space` is an alias for `{space, 1}`. the default is `{space, 0}` the option `{indent, N}` inserts a newline and `N` spaces for each level of indentation in your json output. note that this overrides spaces inserted after a comma. `indent` is an alias for `{indent, 1}`. the default is `{indent, 0}` - -if the option `escape_forward_slash` is enabled, `$/` is escaped. this is not normally required but is necessary for compatibility with microsoft's json date format -**formatting json texts** +### formatting and minifying json text ### + +#### formatting json texts #### produces a JSON text from JSON text, reformatted `format(JSON)` -> `JSON` - `format(JSON, Opts)` -> `JSON` types: -* `JSON` = `binary()` -* `Term` = `[]` | `[{}]` | `[Value]` | `[{Label, Value}]` | `{incomplete, Fun}` -* `Value` = `binary()` | `integer()` | `float()` | `true` | `false` | `null` -* `Label` = `binary()` | `atom()` -* `Fun` = `fun(JSON)` -> `Term` -* `Opts` = `[]` | `[Opt]` +* `JSON` = as above in the mapping section +* `Opts` = as above in the opts section, but see also additional opts below * `Opt` = - `space` - `{space, N}` - `indent` - `{indent, N}` - - `loose_unicode` - - `single_quotes` - - `escape_forward_slash` - - `explicit_end` - -`JSON` SHOULD be a utf8 encoded binary. if the option `loose_unicode` is present attempts are made to replace invalid codepoints with `u+FFFD` but badly encoded binaries may, in either case, result in `badarg` errors - -valid json strings are deliminated by double quotes, but some implementations allow single quotes in their place. the `single_quotes` option recognizes json texts with single quotes in the place of double quotes as valid. please be aware that if you enable this option, you MUST escape single quotes in keys and strings the option `{space, N}` inserts `N` spaces after every comma and colon in your json output. `space` is an alias for `{space, 1}`. the default is `{space, 0}` the option `{indent, N}` inserts a newline and `N` spaces for each level of indentation in your json output. note that this overrides spaces inserted after a comma. `indent` is an alias for `{indent, 1}`. the default is `{indent, 0}` -if the option `escape_forward_slash` is enabled, `$/` is escaped. this is not normally required but is necessary for compatibility with microsoft's json date format - -see the note below about streaming mode for details of `explicit_end` +calling `format` with no options results in minified json text -**verifying json texts** +### verifying json and terms are valid input ### + +#### verifying json texts #### returns true if input is a valid JSON text, false if not `is_json(MaybeJSON)` -> `true` | `false` | `{incomplete, Fun}` - `is_json(MaybeJSON, Opts)` -> `true` | `false` | `{incomplete, Fun}` types: * `MaybeJSON` = `any()` -* `Opts` = `[]` | `[Opt]` -* `Opt` = - - `loose_unicode` - - `single_quotes` - - `explicit_end` - -see `json_to_term` for details of options +* `Opts` = as above -**verifying json texts** +#### verifying terms #### returns true if input is a valid erlang term that represents a JSON text, false if not `is_term(MaybeJSON)` -> `true` | `false` +`is_term(MaybeJSON, Opts)` -> `true` | `false` types: * `MaybeJSON` = `any()` +* `Opts` = as above -**streaming mode** - -this implementation is interruptable and reentrant and may be used to incrementally parse json texts. it's greedy and will exhaust input, returning when the stream buffer is empty. if the json text is so far valid, but incomplete (or if the option `explicit_end` has been selected), `{incomplete, Fun}` will be returned. `Fun/1` may be called with additional input (or the atom `end_stream` to force the end of parsing) - -`explicit_end` is of use when parsing bare numbers (like `123` or `-0.987` for example) as they may have no unambiguous end when encountered in a stream. it is also of use when reading from a socket or file and there may be unprocessed white space (or errors) left in the stream - - -## json <-> erlang ## - -**json** | **erlang** ---------------------------------|-------------------------------- -`number` | `integer()` OR `float()` -`string` | `binary()` -`true`, `false` and `null` | `true`, `false` and `null` -`array` | `[]` OR `[JSON]` -`object` | `[{}]` OR `[{binary(), JSON}]` - -**json** - -json must be encoded in `utf8`. if it's invalid `utf8`, it probably won't parse without errors. one optional exception is made for json strings that are otherwise `utf8`, see under `strings` below. - -**numbers** - -javascript and thus json represent all numeric values with floats. as this is woefully insufficient for many uses, **jsx**, just like erlang, supports bigints. whenever possible, this library will interpret json numbers that look like integers as integers. other numbers will be converted to erlang's floating point type, which is nearly but not quite iee754. negative zero is not representable in erlang (zero is unsigned in erlang and `0` is equivalent to `-0`) and will be interpreted as regular zero. numbers not representable are beyond the concern of this implementation, and will result in parsing errors - -when converting from erlang to json, numbers are represented with their shortest representation that will round trip without loss of precision. this means that some floats may be superficially dissimilar (although functionally equivalent). for example, `1.0000000000000001` will be represented by `1.0` - -**strings** - -the [json spec][rfc4627] is frustratingly vague on the exact details of json strings. json must be unicode, but no encoding is specified. javascript explicitly allows strings containing codepoints explicitly disallowed by unicode. json allows implementations to set limits on the content of strings and other implementations attempt to resolve this in various ways. this implementation, in default operation, only accepts strings that meet the constraints set out in the json spec (properly escaped control characters and quotes) and that are encoded in `utf8`. in the interests of pragmatism, however, the parser option `loose_unicode` attempts to replace invalid `utf8` sequences with the replacement codepoint `u+fffd` when possible - -all erlang strings are represented by *valid* `utf8` encoded binaries - -this implementation performs no normalization on strings beyond that detailed here. be careful when comparing strings as equivalent strings may have different `utf8` encodings - -**true, false and null** - -the json primitives `true`, `false` and `null` are represented by the erlang atoms `true`, `false` and `null`. surprise - -**arrays** - -json arrays are represented with erlang lists of json values as described in this document - -**objects** - -json objects are represented by erlang proplists. the empty object has the special representation `[{}]` to differentiate it from the empty list. ambiguities like `[true, false]` prevent using the shorthand form of property lists using atoms as properties. all properties must be tuples. all keys must be encoded as in `string`, above, or as atoms (which will be escaped and converted to binaries for presentation to handlers) - - -## acknowledgements ## +## acknowledgements ## paul davis, lloyd hilaiel, john engelhart, bob ippolito, fernando benavides and alex kropivny have all contributed to the development of jsx, whether they know it or not +[json]: http://json.org [yajl]: http://lloyd.github.com/yajl [MIT]: http://www.opensource.org/licenses/mit-license.html [rebar]: https://github.com/basho/rebar [meck]: https://github.com/eproxus/meck -[json]: http://json.org [rfc4627]: http://tools.ietf.org/html/rfc4627 \ No newline at end of file From 95e0c20e0d7b31e664ad38a8b9211d76e6fd1abf Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 20 Mar 2012 19:42:03 -0700 Subject: [PATCH 14/27] readme fixes for github's markdown --- README.markdown | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/README.markdown b/README.markdown index b8da7de..89aa2d8 100644 --- a/README.markdown +++ b/README.markdown @@ -13,9 +13,9 @@ jsx uses [rebar][rebar] for it's build chain and [meck][meck] for it's test suit ## index ## -* #### [introduction](#intro) #### -* #### [quickstart](#quickstart) #### -* #### [the api](#api) #### +* [introduction](#intro) +* [quickstart](#quickstart) +* [the api](#api) - [json <-> erlang mapping](#mapping) - [options](#options) - [incomplete input](#incompletes) @@ -24,7 +24,7 @@ jsx uses [rebar][rebar] for it's build chain and [meck][meck] for it's test suit - [converting json to erlang and vice versa](#convert) - [formatting and minifying json text](#format) - [verifying json and terms are valid input](#verify) - * #### [acknowledgments](#thanks) #### + * [acknowledgments](#thanks) @@ -136,6 +136,7 @@ jsx is built on top of two finite state automata, one that handles json texts an `jsx:decoder/3` and `jsx:encoder/3` are the entry points for the decoder and encoder, respectively `decoder(Handler, InitialState, Opts)` -> `Fun((JSON) -> Any)` + `encoder(Handler, InitialState, Opts)` -> `Fun((Term) -> Any)` types: @@ -155,6 +156,7 @@ decoder returns an anonymous function that handles binary json input and encoder `Handler` should export the following pair of functions `Handler:init(InitialState)` -> `State` + `Handler:handle_event(Event, State)` -> `NewState` types: @@ -190,6 +192,7 @@ both `key` and `string` are `utf8` encoded binaries with all escaped values conv `to_term` parses a JSON text (a utf8 encoded binary) and produces an erlang term (see json <-> erlang mapping details above) `to_term(JSON)` -> `Term` + `to_term(JSON, Opts)` -> `Term` types: @@ -212,6 +215,7 @@ the option `labels` controls how keys are converted from json to erlang terms. ` `to_json` parses an erlang term and produces a JSON text (see json <-> erlang mapping details below) `to_json(Term)` -> `JSON` + `to_json(Term, Opts)` -> `JSON` types: @@ -237,6 +241,7 @@ the option `{indent, N}` inserts a newline and `N` spaces for each level of inde produces a JSON text from JSON text, reformatted `format(JSON)` -> `JSON` + `format(JSON, Opts)` -> `JSON` types: @@ -263,6 +268,7 @@ calling `format` with no options results in minified json text returns true if input is a valid JSON text, false if not `is_json(MaybeJSON)` -> `true` | `false` | `{incomplete, Fun}` + `is_json(MaybeJSON, Opts)` -> `true` | `false` | `{incomplete, Fun}` types: @@ -276,6 +282,7 @@ types: returns true if input is a valid erlang term that represents a JSON text, false if not `is_term(MaybeJSON)` -> `true` | `false` + `is_term(MaybeJSON, Opts)` -> `true` | `false` types: From 1028a229c5df6bd123014a661e90cdf59e891eb7 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 20 Mar 2012 22:42:58 -0700 Subject: [PATCH 15/27] minor fixes for illegal utf8 sequences and better testing thereof --- src/jsx_encoder.erl | 130 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 116 insertions(+), 14 deletions(-) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index f97bf02..9e362ee 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -118,7 +118,8 @@ check_string(<>) when C < 16#fdd0 -> check_string(<>) when C > 16#fdef, C < 16#fffe -> check_string(Rest); check_string(<>) - when C =/= 16#fffe andalso C =/= 16#ffff andalso + when C > 16#fffd andalso + C =/= 16#fffe andalso C =/= 16#ffff andalso C =/= 16#1fffe andalso C =/= 16#1ffff andalso C =/= 16#2fffe andalso C =/= 16#2ffff andalso C =/= 16#3fffe andalso C =/= 16#3ffff andalso @@ -140,7 +141,6 @@ check_string(<<>>) -> true; check_string(<<_, _/binary>>) -> false. clean_string(<>, Acc) when C >= 16#fdd0, C =< 16#fdef -> - io:format("1: ~p~n", [C]), clean_string(Rest, <>); clean_string(<>, Acc) when C == 16#fffe orelse C == 16#ffff orelse @@ -160,11 +160,13 @@ clean_string(<>, Acc) C == 16#efffe orelse C == 16#effff orelse C == 16#ffffe orelse C == 16#fffff orelse C == 16#10fffe orelse C == 16#10ffff -> - io:format("2: ~p~n", [C]), clean_string(Rest, <>); clean_string(<>, Acc) -> - io:format("3: ~p~n", [C]), clean_string(Rest, <>); +clean_string(<<237, X, _, Rest/binary>>, Acc) when X >= 160 -> + clean_string(Rest, <>); +clean_string(<<_, Rest/binary>>, Acc) -> + clean_string(Rest, <>); clean_string(<<>>, Acc) -> Acc. @@ -174,7 +176,10 @@ clean_string(<<>>, Acc) -> Acc. encode(Term) -> (encoder(jsx, [], []))(Term). -encode(Term, Opts) -> (encoder(jsx, [], Opts))(Term). +encode(Term, Opts) -> + try (encoder(jsx, [], Opts))(Term) + catch _:_ -> {error, badjson} + end. encode_test_() -> @@ -245,17 +250,114 @@ encode_test_() -> encode([{key, <<"value">>}]), [start_object, {key, <<"key">>}, {string, <<"value">>}, end_object, end_json] ) + } + ]. + +noncharacters_test_() -> + [ + {"noncharacters - badjson", + ?_assertEqual(check_bad(noncharacters()), []) }, - {"bad string", ?_assertError( - badarg, - encode([<<"a bad string: ", 16#ffff/utf8>>]) - ) - }, - {"allow bad string", ?_assertEqual( - encode([<<"a bad string: ", 16#1ffff/utf8>>], [loose_unicode]), - [start_array, {string, <<"a bad string: ", 16#fffd/utf8>>}, end_array, end_json] - ) + {"noncharacters - replaced", + ?_assertEqual(check_replaced(noncharacters()), []) } ]. +extended_noncharacters_test_() -> + [ + {"extended noncharacters - badjson", + ?_assertEqual(check_bad(extended_noncharacters()), []) + }, + {"extended noncharacters - replaced", + ?_assertEqual(check_replaced(extended_noncharacters()), []) + } + ]. + +surrogates_test_() -> + [ + {"surrogates - badjson", + ?_assertEqual(check_bad(surrogates()), []) + }, + {"surrogates - replaced", + ?_assertEqual(check_replaced(surrogates()), []) + } + ]. + +reserved_test_() -> + [ + {"reserved noncharacters - badjson", + ?_assertEqual(check_bad(reserved_space()), []) + }, + {"reserved noncharacters - replaced", + ?_assertEqual(check_replaced(reserved_space()), []) + } + ]. + +good_characters_test_() -> + [ + {"acceptable codepoints", + ?_assertEqual(check_good(good()), []) + }, + {"acceptable extended", + ?_assertEqual(check_good(good_extended()), []) + } + ]. + + +check_bad(List) -> + lists:dropwhile(fun({_, {error, badjson}}) -> true ; (_) -> false end, + check(List, [], []) + ). + +check_replaced(List) -> + lists:dropwhile(fun({_, [{string, <<16#fffd/utf8>>}|_]}) -> true + ; (_) -> false + end, + check(List, [loose_unicode], []) + ). + +check_good(List) -> + lists:dropwhile(fun({_, [{string, _}|_]}) -> true ; (_) -> false end, + check(List, [], []) + ). + +check([], _Opts, Acc) -> Acc; +check([H|T], Opts, Acc) -> + R = encode(to_fake_utf(H, utf8), Opts), + check(T, Opts, [{H, R}] ++ Acc). + + + +noncharacters() -> lists:seq(16#fffe, 16#ffff). + +extended_noncharacters() -> + [16#1fffe, 16#1ffff, 16#2fffe, 16#2ffff] + ++ [16#3fffe, 16#3ffff, 16#4fffe, 16#4ffff] + ++ [16#5fffe, 16#5ffff, 16#6fffe, 16#6ffff] + ++ [16#7fffe, 16#7ffff, 16#8fffe, 16#8ffff] + ++ [16#9fffe, 16#9ffff, 16#afffe, 16#affff] + ++ [16#bfffe, 16#bffff, 16#cfffe, 16#cffff] + ++ [16#dfffe, 16#dffff, 16#efffe, 16#effff] + ++ [16#ffffe, 16#fffff, 16#10fffe, 16#10ffff]. + +surrogates() -> lists:seq(16#d800, 16#dfff). + +reserved_space() -> lists:seq(16#fdd0, 16#fdef). + +good() -> lists:seq(1, 16#d7ff) ++ lists:seq(16#e000, 16#fdcf) ++ lists:seq(16#fdf0, 16#fffd). + +good_extended() -> lists:seq(16#100000, 16#10fffd). + +%% erlang refuses to encode certain codepoints, so fake them all +to_fake_utf(N, utf8) when N < 16#0080 -> <>; +to_fake_utf(N, utf8) when N < 16#0800 -> + <<0:5, Y:5, X:6>> = <>, + <<2#110:3, Y:5, 2#10:2, X:6>>; +to_fake_utf(N, utf8) when N < 16#10000 -> + <> = <>, + <<2#1110:4, Z:4, 2#10:2, Y:6, 2#10:2, X:6>>; +to_fake_utf(N, utf8) -> + <<0:3, W:3, Z:6, Y:6, X:6>> = <>, + <<2#11110:5, W:3, 2#10:2, Z:6, 2#10:2, Y:6, 2#10:2, X:6>>. + -endif. \ No newline at end of file From aa919ce50048080f6f38946f95e2b127a80bb8a9 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 20 Mar 2012 22:47:16 -0700 Subject: [PATCH 16/27] minor fixes for illegal utf8 sequences and better testing thereof --- src/jsx_encoder.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 9e362ee..a2c5bbc 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -161,10 +161,10 @@ clean_string(<>, Acc) C == 16#ffffe orelse C == 16#fffff orelse C == 16#10fffe orelse C == 16#10ffff -> clean_string(Rest, <>); -clean_string(<>, Acc) -> - clean_string(Rest, <>); clean_string(<<237, X, _, Rest/binary>>, Acc) when X >= 160 -> clean_string(Rest, <>); +clean_string(<>, Acc) -> + clean_string(Rest, <>); clean_string(<<_, Rest/binary>>, Acc) -> clean_string(Rest, <>); clean_string(<<>>, Acc) -> Acc. From 9d4edd6c4d48aa972992e873241292f01a68a428 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 20 Mar 2012 22:53:18 -0700 Subject: [PATCH 17/27] fix for older erts versions where the private space reserved characters are not recognized --- src/jsx_encoder.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 9e362ee..9c10a19 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -161,10 +161,12 @@ clean_string(<>, Acc) C == 16#ffffe orelse C == 16#fffff orelse C == 16#10fffe orelse C == 16#10ffff -> clean_string(Rest, <>); -clean_string(<>, Acc) -> - clean_string(Rest, <>); clean_string(<<237, X, _, Rest/binary>>, Acc) when X >= 160 -> clean_string(Rest, <>); +clean_string(<<239, 183, X, Rest/binary>>, Acc) when X >= 144, X =< 175 -> + clean_string(Rest, <>); +clean_string(<>, Acc) -> + clean_string(Rest, <>); clean_string(<<_, Rest/binary>>, Acc) -> clean_string(Rest, <>); clean_string(<<>>, Acc) -> Acc. From 66add159b5c4113903a3565257b1f8c4ade1ed01 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 20 Mar 2012 23:00:33 -0700 Subject: [PATCH 18/27] fix for older erts that don't allow noncharacters --- src/jsx_encoder.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 9c10a19..21e5269 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -165,6 +165,8 @@ clean_string(<<237, X, _, Rest/binary>>, Acc) when X >= 160 -> clean_string(Rest, <>); clean_string(<<239, 183, X, Rest/binary>>, Acc) when X >= 144, X =< 175 -> clean_string(Rest, <>); +clean_string(<<239, 191, X, Rest/binary>>, Acc) when X == 190, X == 191 -> + clean_string(Rest, <>); clean_string(<>, Acc) -> clean_string(Rest, <>); clean_string(<<_, Rest/binary>>, Acc) -> From b406afaa779566ce685942b6e2291e230cd8cc3d Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 20 Mar 2012 23:07:13 -0700 Subject: [PATCH 19/27] remove rogue function head --- src/jsx_encoder.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 21e5269..6f993f1 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -140,8 +140,6 @@ check_string(<>) check_string(<<>>) -> true; check_string(<<_, _/binary>>) -> false. -clean_string(<>, Acc) when C >= 16#fdd0, C =< 16#fdef -> - clean_string(Rest, <>); clean_string(<>, Acc) when C == 16#fffe orelse C == 16#ffff orelse C == 16#1fffe orelse C == 16#1ffff orelse @@ -161,10 +159,13 @@ clean_string(<>, Acc) C == 16#ffffe orelse C == 16#fffff orelse C == 16#10fffe orelse C == 16#10ffff -> clean_string(Rest, <>); +%% surrogates clean_string(<<237, X, _, Rest/binary>>, Acc) when X >= 160 -> clean_string(Rest, <>); +%% private use noncharacters clean_string(<<239, 183, X, Rest/binary>>, Acc) when X >= 144, X =< 175 -> clean_string(Rest, <>); +%% u+fffe and u+ffff (required for otp < r15) clean_string(<<239, 191, X, Rest/binary>>, Acc) when X == 190, X == 191 -> clean_string(Rest, <>); clean_string(<>, Acc) -> From 07c1f5716c8071130624bc08209c9ed96a53e2f0 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 20 Mar 2012 23:13:27 -0700 Subject: [PATCH 20/27] finally found actual cause of otp r14x bug --- src/jsx_encoder.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 6f993f1..35c0af2 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -118,7 +118,7 @@ check_string(<>) when C < 16#fdd0 -> check_string(<>) when C > 16#fdef, C < 16#fffe -> check_string(Rest); check_string(<>) - when C > 16#fffd andalso + when C > 16#ffff andalso C =/= 16#fffe andalso C =/= 16#ffff andalso C =/= 16#1fffe andalso C =/= 16#1ffff andalso C =/= 16#2fffe andalso C =/= 16#2ffff andalso From f8f436e0a0527348a2bc33bfcd1101c8e89f2a22 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 20 Mar 2012 23:17:57 -0700 Subject: [PATCH 21/27] ok, now it's fixed for older releases --- src/jsx_encoder.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 35c0af2..f6c0c2f 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -24,6 +24,7 @@ -module(jsx_encoder). -export([encoder/3]). +-compile(export_all). -spec encoder(Handler::module(), State::any(), Opts::jsx:opts()) -> jsx:encoder(). @@ -118,8 +119,7 @@ check_string(<>) when C < 16#fdd0 -> check_string(<>) when C > 16#fdef, C < 16#fffe -> check_string(Rest); check_string(<>) - when C > 16#ffff andalso - C =/= 16#fffe andalso C =/= 16#ffff andalso + when C > 16#ffff andalso C =/= 16#1fffe andalso C =/= 16#1ffff andalso C =/= 16#2fffe andalso C =/= 16#2ffff andalso C =/= 16#3fffe andalso C =/= 16#3ffff andalso From 0b789147a59053e2439685875746c39b4adaaf14 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 20 Mar 2012 23:18:39 -0700 Subject: [PATCH 22/27] remove export_all flag --- src/jsx_encoder.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index f6c0c2f..1308a6e 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -24,7 +24,6 @@ -module(jsx_encoder). -export([encoder/3]). --compile(export_all). -spec encoder(Handler::module(), State::any(), Opts::jsx:opts()) -> jsx:encoder(). From c9ea2975bd10038209e80ac0d52cd4c58029ad01 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 20 Mar 2012 23:58:22 -0700 Subject: [PATCH 23/27] whitelist allowed codepoints rather than blacklist disallowed codepoints in jsx_encoder --- src/jsx_encoder.erl | 56 +++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 1308a6e..4ee0fa5 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -139,37 +139,33 @@ check_string(<>) check_string(<<>>) -> true; check_string(<<_, _/binary>>) -> false. -clean_string(<>, Acc) - when C == 16#fffe orelse C == 16#ffff orelse - C == 16#1fffe orelse C == 16#1ffff orelse - C == 16#2fffe orelse C == 16#2ffff orelse - C == 16#3fffe orelse C == 16#3ffff orelse - C == 16#4fffe orelse C == 16#4ffff orelse - C == 16#5fffe orelse C == 16#5ffff orelse - C == 16#6fffe orelse C == 16#6ffff orelse - C == 16#7fffe orelse C == 16#7ffff orelse - C == 16#8fffe orelse C == 16#8ffff orelse - C == 16#9fffe orelse C == 16#9ffff orelse - C == 16#afffe orelse C == 16#affff orelse - C == 16#bfffe orelse C == 16#bffff orelse - C == 16#cfffe orelse C == 16#cffff orelse - C == 16#dfffe orelse C == 16#dffff orelse - C == 16#efffe orelse C == 16#effff orelse - C == 16#ffffe orelse C == 16#fffff orelse - C == 16#10fffe orelse C == 16#10ffff -> - clean_string(Rest, <>); -%% surrogates -clean_string(<<237, X, _, Rest/binary>>, Acc) when X >= 160 -> - clean_string(Rest, <>); -%% private use noncharacters -clean_string(<<239, 183, X, Rest/binary>>, Acc) when X >= 144, X =< 175 -> - clean_string(Rest, <>); -%% u+fffe and u+ffff (required for otp < r15) -clean_string(<<239, 191, X, Rest/binary>>, Acc) when X == 190, X == 191 -> - clean_string(Rest, <>); -clean_string(<>, Acc) -> + +clean_string(<>, Acc) when C < 16#fdd0 -> clean_string(Rest, <>); -clean_string(<<_, Rest/binary>>, Acc) -> +clean_string(<>, Acc) when C > 16#fdef, C < 16#fffe -> + clean_string(Rest, <>); +clean_string(<>, Acc) + when C > 16#ffff andalso + C =/= 16#1fffe andalso C =/= 16#1ffff andalso + C =/= 16#2fffe andalso C =/= 16#2ffff andalso + C =/= 16#3fffe andalso C =/= 16#3ffff andalso + C =/= 16#4fffe andalso C =/= 16#4ffff andalso + C =/= 16#5fffe andalso C =/= 16#5ffff andalso + C =/= 16#6fffe andalso C =/= 16#6ffff andalso + C =/= 16#7fffe andalso C =/= 16#7ffff andalso + C =/= 16#8fffe andalso C =/= 16#8ffff andalso + C =/= 16#9fffe andalso C =/= 16#9ffff andalso + C =/= 16#afffe andalso C =/= 16#affff andalso + C =/= 16#bfffe andalso C =/= 16#bffff andalso + C =/= 16#cfffe andalso C =/= 16#cffff andalso + C =/= 16#dfffe andalso C =/= 16#dffff andalso + C =/= 16#efffe andalso C =/= 16#effff andalso + C =/= 16#ffffe andalso C =/= 16#fffff andalso + C =/= 16#10fffe andalso C =/= 16#10ffff -> + clean_string(Rest, <>); +clean_string(<>, Acc) when X == 237; X == 239 -> + clean_string(Rest, <>); +clean_string(<<_, _, _, _, Rest/binary>>, Acc) -> clean_string(Rest, <>); clean_string(<<>>, Acc) -> Acc. From e103ddbed26c53bcfad36ab0be93875834a0cba3 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Wed, 21 Mar 2012 00:00:34 -0700 Subject: [PATCH 24/27] typo in readme --- README.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index 89aa2d8..5256175 100644 --- a/README.markdown +++ b/README.markdown @@ -24,7 +24,7 @@ jsx uses [rebar][rebar] for it's build chain and [meck][meck] for it's test suit - [converting json to erlang and vice versa](#convert) - [formatting and minifying json text](#format) - [verifying json and terms are valid input](#verify) - * [acknowledgments](#thanks) +* [acknowledgments](#thanks) From 145368708008bd84973ed13839ca0d86a270c6e9 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Wed, 21 Mar 2012 04:34:33 -0700 Subject: [PATCH 25/27] readme updates and clarifications --- README.markdown | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/README.markdown b/README.markdown index 5256175..81bb529 100644 --- a/README.markdown +++ b/README.markdown @@ -74,9 +74,17 @@ when converting from erlang to json, numbers are represented with their shortest #### strings #### -the [json spec][rfc4627] is frustratingly vague on the exact details of json strings. json must be unicode, but no encoding is specified. javascript explicitly allows strings containing codepoints explicitly disallowed by unicode. json allows implementations to set limits on the content of strings and other implementations attempt to resolve this in various ways. this implementation, in default operation, only accepts strings that meet the constraints set out in the json spec (properly escaped control characters and quotes) and that are encoded in `utf8`. in the interests of pragmatism, there is an option for looser parsing, see options below +the [json spec][rfc4627] is frustratingly vague on the exact details of json strings. json must be unicode, but no encoding is specified. javascript explicitly allows strings containing codepoints explicitly disallowed by unicode. json allows implementations to set limits on the content of strings and other implementations attempt to resolve this in various ways. this implementation, in default operation, only accepts strings that meet the constraints set out in the json spec (properly escaped control characters, `"` and the escape character, `\`) and that are encoded in `utf8` -all erlang strings are represented by *valid* `utf8` encoded binaries +this means some codepoints that are allowed in javascript strings are not accepted by the parser. the noncharacters are specifically disallowed. the range `u+fdd0` to `u+fdef` is reserved for internal implementation use by the unicode standard and codepoints of the form `u+Xfffe` and `u+Xffff` are reserved for error detection. strings containing these codepoints are generally assumed to be invalid or improper + +also disallowed are improperly paired surrogates. `u+d800` to `u+dfff` are allowed, but only when they form valid surrogate pairs. surrogates that appear otherwise are an error + +json string escapes of the form `\uXXXX` will be converted to their equivalent codepoint during parsing. this means control characters and other codepoints disallowed by the json spec may be encountered in resulting strings, but codepoints disallowed by the unicode spec (like the two cases above) will not be + +in the interests of pragmatism, there is an option for looser parsing, see options below + +all erlang strings are represented by *valid* `utf8` encoded binaries. the encoder will check strings for conformance. the same restrictions apply as for strings encountered within json texts. that means no unpaired surrogates and no non-characters this implementation performs no normalization on strings beyond that detailed here. be careful when comparing strings as equivalent strings may have different `utf8` encodings @@ -90,7 +98,7 @@ json arrays are represented with erlang lists of json values as described in thi #### objects #### -json objects are represented by erlang proplists. the empty object has the special representation `[{}]` to differentiate it from the empty list. ambiguities like `[true, false]` prevent using the shorthand form of property lists using atoms as properties. all properties must be tuples. all keys must be encoded as in `string`, above, or as atoms (which will be escaped and converted to binaries for presentation to handlers) +json objects are represented by erlang proplists. the empty object has the special representation `[{}]` to differentiate it from the empty list. ambiguities like `[true, false]` prevent using the shorthand form of property lists using atoms as properties so all properties must be tuples. all keys must be encoded as in `string`, above, or as atoms (which will be escaped and converted to binaries for presentation to handlers). values should be valid json values ### options ### @@ -99,7 +107,7 @@ jsx functions all take a common set of options. not all flags have meaning in al #### `loose_unicode` #### -json text input and json strings SHOULD be utf8 encoded binaries, appropriately escaped as per the json spec. if this option is present attempts are made to replace invalid codepoints with `u+FFFD` as per the unicode spec +json text input and json strings SHOULD be utf8 encoded binaries, appropriately escaped as per the json spec. if this option is present attempts are made to replace invalid codepoints with `u+FFFD` as per the unicode spec. this applies both to malformed unicode and disallowed codepoints #### `escape_forward_slash` #### From be89f5f395c0d8e65920b91556b1b9988ace98f4 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Wed, 21 Mar 2012 05:19:47 -0700 Subject: [PATCH 26/27] corrected handling of malformed utf8 sequences --- src/jsx_decoder.erl | 61 ++++++++++++++++++++++++++++++++++++++++++++ src/jsx_encoder.erl | 62 ++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 120 insertions(+), 3 deletions(-) diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 4055b2e..f70e0f8 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -334,6 +334,20 @@ noncharacter(<<239, 191, X, Rest/binary>>, Handler, [Acc|Stack], Opts) %% surrogates noncharacter(<<237, X, _, Rest/binary>>, Handler, [Acc|Stack], Opts) when X >= 160 -> string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Opts); +noncharacter(<>, Handler, [Acc|Stack], Opts) + when ( + (X == 240 andalso Y == 159) orelse + (X == 240 andalso Y == 175) orelse + (X == 240 andalso Y == 191) orelse + ( + (X == 241 orelse X == 242 orelse X == 243) andalso + (Y == 143 orelse Y == 159 orelse Y == 175 orelse Y == 191) + ) orelse + (X == 244 andalso Y == 143) + ) andalso (Z == 190 orelse Z == 191) -> + string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Opts); +noncharacter(<<_, Rest/binary>>, Handler, [Acc|Stack], Opts) -> + string(Rest, Handler, [?acc_seq(Acc, 16#fffd)|Stack], Opts); noncharacter(Bin, Handler, Stack, Opts) -> ?error([Bin, Handler, Stack, Opts]). @@ -1079,6 +1093,51 @@ good_characters_test_() -> ?_assertEqual(check_good(good_extended()), []) } ]. + +malformed_test_() -> + [ + {"malformed codepoint with 1 byte", + ?_assertEqual({error, badjson}, decode(<<128>>)) + }, + {"malformed codepoint with 2 bytes", + ?_assertEqual({error, badjson}, decode(<<128, 192>>)) + }, + {"malformed codepoint with 3 bytes", + ?_assertEqual({error, badjson}, decode(<<128, 192, 192>>)) + }, + {"malformed codepoint with 4 bytes", + ?_assertEqual({error, badjson}, decode(<<128, 192, 192, 192>>)) + } + ]. + +malformed_replaced_test_() -> + F = <<16#fffd/utf8>>, + [ + {"malformed codepoint with 1 byte", + ?_assertEqual( + [{string, <>}, end_json], + decode(<<34, 128, 34>>, [loose_unicode]) + ) + }, + {"malformed codepoint with 2 bytes", + ?_assertEqual( + [{string, <>}, end_json], + decode(<<34, 128, 192, 34>>, [loose_unicode]) + ) + }, + {"malformed codepoint with 3 bytes", + ?_assertEqual( + [{string, <>}, end_json], + decode(<<34, 128, 192, 192, 34>>, [loose_unicode]) + ) + }, + {"malformed codepoint with 4 bytes", + ?_assertEqual( + [{string, <>}, end_json], + decode(<<34, 128, 192, 192, 192, 34>>, [loose_unicode]) + ) + } + ]. check_bad(List) -> @@ -1104,6 +1163,8 @@ check([H|T], Opts, Acc) -> check(T, Opts, [{H, R}] ++ Acc). +decode(JSON) -> decode(JSON, []). + decode(JSON, Opts) -> try (decoder(jsx, [], Opts))(JSON) diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 4ee0fa5..5cd9934 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -163,9 +163,29 @@ clean_string(<>, Acc) C =/= 16#ffffe andalso C =/= 16#fffff andalso C =/= 16#10fffe andalso C =/= 16#10ffff -> clean_string(Rest, <>); -clean_string(<>, Acc) when X == 237; X == 239 -> +%% surrogates +clean_string(<<237, X, _, Rest/binary>>, Acc) when X >= 160 -> clean_string(Rest, <>); -clean_string(<<_, _, _, _, Rest/binary>>, Acc) -> +%% private use noncharacters +clean_string(<<239, 183, X, Rest/binary>>, Acc) when X >= 143, X =< 175 -> + clean_string(Rest, <>); +%% u+fffe and u+ffff +clean_string(<<239, 191, X, Rest/binary>>, Acc) when X == 190; X == 191 -> + clean_string(Rest, <>); +%% the u+Xfffe and u+Xffff noncharacters +clean_string(<>, Acc) + when ( + (X == 240 andalso Y == 159) orelse + (X == 240 andalso Y == 175) orelse + (X == 240 andalso Y == 191) orelse + ( + (X == 241 orelse X == 242 orelse X == 243) andalso + (Y == 143 orelse Y == 159 orelse Y == 175 orelse Y == 191) + ) orelse + (X == 244 andalso Y == 143) + ) andalso (Z == 190 orelse Z == 191) -> + clean_string(Rest, <>); +clean_string(<<_, Rest/binary>>, Acc) -> clean_string(Rest, <>); clean_string(<<>>, Acc) -> Acc. @@ -302,7 +322,43 @@ good_characters_test_() -> ?_assertEqual(check_good(good_extended()), []) } ]. - + +malformed_test_() -> + [ + {"malformed codepoint with 1 byte", ?_assertError(badarg, encode(<<128>>))}, + {"malformed codepoint with 2 bytes", ?_assertError(badarg, encode(<<128, 192>>))}, + {"malformed codepoint with 3 bytes", ?_assertError(badarg, encode(<<128, 192, 192>>))}, + {"malformed codepoint with 4 bytes", ?_assertError(badarg, encode(<<128, 192, 192, 192>>))} + ]. + +malformed_replaced_test_() -> + F = <<16#fffd/utf8>>, + [ + {"malformed codepoint with 1 byte", + ?_assertEqual( + [{string, <>}, end_json], + encode(<<128>>, [loose_unicode]) + ) + }, + {"malformed codepoint with 2 bytes", + ?_assertEqual( + [{string, <>}, end_json], + encode(<<128, 192>>, [loose_unicode]) + ) + }, + {"malformed codepoint with 3 bytes", + ?_assertEqual( + [{string, <>}, end_json], + encode(<<128, 192, 192>>, [loose_unicode]) + ) + }, + {"malformed codepoint with 4 bytes", + ?_assertEqual( + [{string, <>}, end_json], + encode(<<128, 192, 192, 192>>, [loose_unicode]) + ) + } + ]. check_bad(List) -> lists:dropwhile(fun({_, {error, badjson}}) -> true ; (_) -> false end, From 46767102e1c8c1982441a57f8361e7e2f0d771d4 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Wed, 21 Mar 2012 05:20:41 -0700 Subject: [PATCH 27/27] freezing for 1.1rc --- README.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index 81bb529..60cf62d 100644 --- a/README.markdown +++ b/README.markdown @@ -1,4 +1,4 @@ -# jsx (v1.0) # +# jsx (v1.1) # a sane [json][json] implementation for erlang, inspired by [yajl][yajl]