diff --git a/CHANGES.md b/CHANGES.md index a49bb21..438b7dd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,14 @@ +v1.4.5 + +* various fixes to typespecs uncovered by dialyzer +* allow integer keys during encoding +* convert atoms (other than `true`, `false` and `null`) to strings during encoding + +v1.4.4 + +* typespec for `json_term/0` fixed +* incorrect boolean shortcircuiting fixed in multibyte escape processing + v1.4.3 * add empty rebar.config for mix build tool @@ -72,4 +83,4 @@ v1.0.2 v1.0.1 -* rebar fix \ No newline at end of file +* rebar fix diff --git a/README.md b/README.md index 5537d00..b0316d0 100644 --- a/README.md +++ b/README.md @@ -144,8 +144,8 @@ ignores bad escape sequences **json** | **erlang** --------------------------------|-------------------------------- -`number` | `integer()` if possible, `float()` otherwise -`string` | `binary()` +`number` | `integer()` and `float()` +`string` | `binary()` and `atom()` `true`, `false` and `null` | `true`, `false` and `null` `array` | `[]` and `[JSON]` `object` | `[{}]` and `[{binary() OR atom(), JSON}]` @@ -171,9 +171,10 @@ ignores bad escape sequences * strings - json strings must be unicode. in practice, because **jsx** only accepts - `utf8` all strings must be `utf8`. in addition to being unicode json strings - restrict a number of codepoints and define a number of escape sequences + json strings must be unicode encoded binaries or erlang atoms. in practice, + because **jsx** only accepts `utf8` binaries all binary strings must be `utf8`. + in addition to being unicode json strings restrict a number of codepoints and + define a number of escape sequences json string escapes of the form `\uXXXX` will be converted to their equivalent codepoints during parsing. this means control characters and @@ -208,9 +209,9 @@ ignores bad escape sequences special representation `[{}]` to differentiate it from the empty list. ambiguities like `[true, false]` prevent the use of the shorthand form of property lists using atoms as properties so all properties must be tuples. - all keys must be encoded as in `string` or as atoms (which will be escaped - and converted to binaries for presentation to handlers). values should be - valid json values + all keys must be encoded as in `string` or as atoms or integers (which will + be escaped and converted to binaries for presentation to handlers). values + should be valid json values ### incomplete input ### @@ -254,6 +255,7 @@ json_term() = [json_term()] | integer() | float() | binary() + | atom() ``` the erlang representation of json. binaries should be `utf8` encoded, or close @@ -631,7 +633,7 @@ following events must be handled: ## acknowledgements ## -**jsx** wouldn't be what it is without the contributions of [paul davis](https://github.com/davisp), [lloyd hilaiel](https://github.com/lloyd), [john engelhart](https://github.com/johnezang), [bob ippolito](https://github.com/etrepum), [fernando benavides](https://github.com/elbrujohalcon), [alex kropivny](https://github.com/amtal), [steve strong](https://github.com/srstrong), [michael truog](https://github.com/okeuday), [dmitry kolesnikov](https://github.com/fogfish) and [emptytea](https://github.com/emptytea) +jsx wouldn't be what it is without the contributions of [paul davis](https://github.com/davisp), [lloyd hilaiel](https://github.com/lloyd), [john engelhart](https://github.com/johnezang), [bob ippolito](https://github.com/etrepum), [fernando benavides](https://github.com/elbrujohalcon), [alex kropivny](https://github.com/amtal), [steve strong](https://github.com/srstrong), [michael truog](https://github.com/okeuday), [devin torres](https://github.com/devinus), [dmitry kolesnikov](https://github.com/fogfish), [emptytea](https://github.com/emptytea), [john daily](https://github.com/macintux), [ola bäckström](https://github.com/olabackstrom), [joseph crowe](https://github.com/JosephCrowe), [patrick gombert](https://github.com/patrickgombert), [eskuat](https://github.com/eskuat) and [max lapshin](https://github.com/maxlapshin) [json]: http://json.org [yajl]: http://lloyd.github.com/yajl @@ -639,4 +641,4 @@ following events must be handled: [rebar]: https://github.com/rebar/rebar [meck]: https://github.com/eproxus/meck [rfc4627]: http://tools.ietf.org/html/rfc4627 -[travis]: https://travis-ci.org/ \ No newline at end of file +[travis]: https://travis-ci.org/ diff --git a/src/jsx.app.src b/src/jsx.app.src index 3837287..b71593d 100644 --- a/src/jsx.app.src +++ b/src/jsx.app.src @@ -1,7 +1,7 @@ {application, jsx, [ {description, "a streaming, evented json parsing toolkit"}, - {vsn, "1.4.3"}, + {vsn, "1.4.4"}, {modules, [ jsx, jsx_encoder, diff --git a/src/jsx.erl b/src/jsx.erl index b791699..57a1d45 100644 --- a/src/jsx.erl +++ b/src/jsx.erl @@ -48,11 +48,11 @@ | null | integer() | float() - | binary(). + | binary() + | atom(). -type json_text() :: binary(). - -spec encode(Source::json_term()) -> json_text() | {incomplete, encoder()}. -spec encode(Source::json_term(), Config::jsx_to_json:config()) -> json_text() | {incomplete, encoder()}. diff --git a/src/jsx_config.erl b/src/jsx_config.erl index 9039d3f..8633d8d 100644 --- a/src/jsx_config.erl +++ b/src/jsx_config.erl @@ -33,8 +33,22 @@ -include("jsx_config.hrl"). +-type handler_type(Handler) :: + fun((jsx:json_text() | end_stream | + jsx:json_term(), + {decoder, any(), module(), null | list(), list()} | + {parser, any(), module(), list()} | + {encoder, any(), module()}, + list({pre_encode, fun((any()) -> any())} | + {error_handler, Handler} | + {incomplete_handler, Handler} | + atom())) -> any()). +-type handler() :: handler_type(handler()). +-export_type([handler/0]). %% parsing of jsx config +-spec parse_config(Config::proplists:proplist()) -> jsx:config(). + parse_config(Config) -> parse_config(Config, #config{}). parse_config([], Config) -> Config; @@ -83,6 +97,8 @@ parse_strict(_Strict, _Rest, _Config) -> +-spec config_to_list(Config::jsx:config()) -> proplists:proplist(). + config_to_list(Config) -> reduce_config(lists:map( fun ({error_handler, F}) -> {error_handler, F}; @@ -116,6 +132,8 @@ reduce_config([Else|Input], Output, Strict) -> reduce_config(Input, [Else] ++ Output, Strict). +-spec valid_flags() -> [atom()]. + valid_flags() -> [ escaped_forward_slashes, @@ -129,6 +147,8 @@ valid_flags() -> ]. +-spec extract_config(Config::proplists:proplist()) -> proplists:proplist(). + extract_config(Config) -> extract_parser_config(Config, []). @@ -281,4 +301,4 @@ config_to_list_test_() -> fake_error_handler(_, _, _) -> ok. --endif. \ No newline at end of file +-endif. diff --git a/src/jsx_config.hrl b/src/jsx_config.hrl index ae333fd..89f7824 100644 --- a/src/jsx_config.hrl +++ b/src/jsx_config.hrl @@ -1,13 +1,13 @@ -record(config, { - escaped_forward_slashes = false, - escaped_strings = false, - unescaped_jsonp = false, - dirty_strings = false, - strict_comments = false, - strict_utf8 = false, - strict_single_quotes = false, - strict_escapes = false, - stream = false, - error_handler = false, - incomplete_handler = false -}). \ No newline at end of file + escaped_forward_slashes = false :: boolean(), + escaped_strings = false :: boolean(), + unescaped_jsonp = false :: boolean(), + dirty_strings = false :: boolean(), + strict_comments = false :: boolean(), + strict_utf8 = false :: boolean(), + strict_single_quotes = false :: boolean(), + strict_escapes = false :: boolean(), + stream = false :: boolean(), + error_handler = false :: false | jsx_config:handler(), + incomplete_handler = false :: false | jsx_config:handler() +}). diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index 71ac4b7..a9d5c6e 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -33,7 +33,7 @@ -export([decoder/3, resume/6]). --spec decoder(Handler::module(), State::any(), Config::jsx:config()) -> jsx:decoder(). +-spec decoder(Handler::module(), State::any(), Config::list()) -> jsx:decoder(). decoder(Handler, State, Config) -> fun(JSON) -> start(JSON, {Handler, Handler:init(State)}, [], jsx_config:parse_config(Config)) end. @@ -48,7 +48,7 @@ decoder(Handler, State, Config) -> Acc::any(), Stack::list(atom()), Config::jsx:config() - ) -> jsx:decoder(). + ) -> jsx:decoder() | {incomplete, jsx:decoder()}. resume(Rest, State, Handler, Acc, Stack, Config) -> case State of @@ -671,12 +671,12 @@ unescape(<<$u, $d, A, B, C, ?rsolidus, $u, W, X, Y, Z, Rest/binary>>, Handler, A false -> string(Rest, Handler, acc_seq(Acc, [16#fffd, 16#fffd]), Stack, Config) end; unescape(<<$u, $d, A, B, C, ?rsolidus, Rest/binary>>, Handler, Acc, Stack, Config) - when (A == $8 orelse A == $9 orelse A == $a orelse A == $b) andalso + when (A == $8 orelse A == $9 orelse A == $a orelse A == $b), ?is_hex(B), ?is_hex(C) -> incomplete(string, <>, Handler, Acc, Stack, Config); unescape(<<$u, $d, A, B, C>>, Handler, Acc, Stack, Config) - when (A == $8 orelse A == $9 orelse A == $a orelse A == $b) andalso + when (A == $8 orelse A == $9 orelse A == $a orelse A == $b), ?is_hex(B), ?is_hex(C) -> incomplete(string, <>, Handler, Acc, Stack, Config); @@ -1687,4 +1687,4 @@ custom_incomplete_handler_test_() -> ]. --endif. \ No newline at end of file +-endif. diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 35acffe..0eaf2f2 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -25,7 +25,7 @@ -export([encoder/3, encode/1, encode/2, unzip/1]). --spec encoder(Handler::module(), State::any(), Config::jsx:config()) -> jsx:encoder(). +-spec encoder(Handler::module(), State::any(), Config::list()) -> jsx:encoder(). encoder(Handler, State, Config) -> Parser = jsx:parser(Handler, State, Config), @@ -57,8 +57,7 @@ encode(Else, _EntryPoint) -> [Else]. unzip(List) -> unzip(List, []). unzip([], Acc) -> lists:reverse(Acc); -unzip([{K, V}|Rest], Acc) when is_binary(K); is_atom(K) -> unzip(Rest, [V, K] ++ Acc). - +unzip([{K, V}|Rest], Acc) when is_binary(K); is_atom(K); is_integer(K) -> unzip(Rest, [V, K] ++ Acc). -ifdef(TEST). @@ -67,6 +66,7 @@ unzip([{K, V}|Rest], Acc) when is_binary(K); is_atom(K) -> unzip(Rest, [V, K] ++ parser(Term, Opts) -> (jsx:parser(jsx, [], Opts))(Term). + error_test_() -> [ {"value error", ?_assertError(badarg, parser(self(), []))}, diff --git a/src/jsx_parser.erl b/src/jsx_parser.erl index dc2d436..769ffd7 100644 --- a/src/jsx_parser.erl +++ b/src/jsx_parser.erl @@ -27,7 +27,7 @@ -export([init/1, handle_event/2]). --spec parser(Handler::module(), State::any(), Config::jsx:config()) -> jsx:parser(). +-spec parser(Handler::module(), State::any(), Config::list()) -> jsx:parser(). parser(Handler, State, Config) -> fun(Tokens) -> value(Tokens, {Handler, Handler:init(State)}, [], jsx_config:parse_config(Config)) end. @@ -36,12 +36,12 @@ parser(Handler, State, Config) -> %% resume allows continuation from interrupted decoding without having to explicitly export %% all states -spec resume( - Rest::binary(), + Rest::jsx:token(), State::atom(), Handler::{atom(), any()}, Stack::list(atom()), Config::jsx:config() - ) -> jsx:parser(). + ) -> jsx:parser() | {incomplete, jsx:parser()}. resume(Rest, State, Handler, Stack, Config) -> case State of @@ -84,8 +84,6 @@ incomplete(State, Handler, Stack, Config=#config{incomplete_handler=F}) -> F([], {parser, State, Handler, Stack}, jsx_config:config_to_list(Config)). -handle_event([], Handler, _Config) -> Handler; -handle_event([Event|Rest], Handler, Config) -> handle_event(Rest, handle_event(Event, Handler, Config), Config); handle_event(Event, {Handler, State}, _Config) -> {Handler, Handler:handle_event(Event, State)}. @@ -128,13 +126,13 @@ value(Token, Handler, Stack, Config) -> object([end_object|Tokens], Handler, [object|Stack], Config) -> maybe_done(Tokens, handle_event(end_object, Handler, Config), Stack, Config); -object([{key, Key}|Tokens], Handler, Stack, Config) when is_atom(Key); is_binary(Key) -> +object([{key, Key}|Tokens], Handler, Stack, Config) when is_atom(Key); is_binary(Key); is_integer(Key) -> case clean_string(fix_key(Key), Tokens, Handler, Stack, Config) of Clean when is_binary(Clean) -> value(Tokens, handle_event({key, Clean}, Handler, Config), Stack, Config); Error -> Error end; -object([Key|Tokens], Handler, Stack, Config) when is_atom(Key); is_binary(Key) -> +object([Key|Tokens], Handler, Stack, Config) when is_atom(Key); is_binary(Key); is_integer(Key) -> case clean_string(fix_key(Key), Tokens, Handler, Stack, Config) of Clean when is_binary(Clean) -> value(Tokens, handle_event({key, Clean}, Handler, Config), Stack, Config); @@ -178,7 +176,8 @@ done(Token, Handler, Stack, Config) -> done([Token], Handler, Stack, Config). -fix_key(Key) when is_atom(Key) -> fix_key(atom_to_binary(Key, utf8)); +fix_key(Key) when is_atom(Key) -> atom_to_binary(Key, utf8); +fix_key(Key) when is_integer(Key) -> list_to_binary(integer_to_list(Key)); fix_key(Key) when is_binary(Key) -> Key. @@ -440,8 +439,12 @@ to_hex(X) -> X + 48. %% ascii "1" is [49], "2" is [50], etc... %% for raw input +-spec init(proplists:proplist()) -> list(). + init([]) -> []. +-spec handle_event(Event::any(), Acc::list()) -> list(). + handle_event(end_json, State) -> lists:reverse(State); handle_event(Event, State) -> [Event] ++ State. @@ -997,4 +1000,11 @@ json_escape_sequence_test_() -> ]. --endif. \ No newline at end of file +fix_key_test_() -> + [ + {"binary key", ?_assertEqual(fix_key(<<"foo">>), <<"foo">>)}, + {"atom key", ?_assertEqual(fix_key(foo), <<"foo">>)}, + {"integer key", ?_assertEqual(fix_key(123), <<"123">>)} + ]. + +-endif. diff --git a/src/jsx_to_json.erl b/src/jsx_to_json.erl index 21fea69..6f4114e 100644 --- a/src/jsx_to_json.erl +++ b/src/jsx_to_json.erl @@ -35,6 +35,7 @@ }). -type config() :: list(). +-export_type([config/0]). -spec to_json(Source::any(), Config::config()) -> binary(). @@ -85,9 +86,14 @@ parse_config([], Config) -> -define(newline, <<"\n">>). +-type state() :: {unicode:charlist(), #config{}}. +-spec init(Config::proplists:proplist()) -> state(). + init(Config) -> {[], parse_config(Config)}. +-spec handle_event(Event::any(), State::state()) -> state(). + handle_event(end_json, {Term, _Config}) -> Term; handle_event(start_object, State) -> start_object(State); diff --git a/src/jsx_to_term.erl b/src/jsx_to_term.erl index 173d190..fd90d10 100644 --- a/src/jsx_to_term.erl +++ b/src/jsx_to_term.erl @@ -33,6 +33,8 @@ }). -type config() :: list(). +-export_type([config/0]). + -type json_value() :: list({binary(), json_value()}) | list(json_value()) @@ -70,9 +72,12 @@ parse_config([K|Rest] = Options, Config) -> parse_config([], Config) -> Config. +-type state() :: {[any()], #config{}}. +-spec init(Config::proplists:proplist()) -> state(). init(Config) -> {[], parse_config(Config)}. +-spec handle_event(Event::any(), State::state()) -> state(). handle_event(end_json, {Term, _Config}) -> Term; diff --git a/src/jsx_verify.erl b/src/jsx_verify.erl index 90251b3..34c50e5 100644 --- a/src/jsx_verify.erl +++ b/src/jsx_verify.erl @@ -32,6 +32,7 @@ }). -type config() :: []. +-export_type([config/0]). -spec is_json(Source::binary(), Config::config()) -> true | false. @@ -72,10 +73,14 @@ parse_config([K|Rest] = Options, Config) -> parse_config([], Config) -> Config. +-type state() :: {#config{}, any()}. +-spec init(Config::proplists:proplist()) -> state(). init(Config) -> {parse_config(Config), []}. +-spec handle_event(Event::any(), State::state()) -> state(). + handle_event(end_json, _) -> true; handle_event(_, {Config, _} = State) when Config#config.repeated_keys == true -> State; @@ -165,4 +170,4 @@ handle_event_test_() -> ]. --endif. \ No newline at end of file +-endif.