From 200fb231a94b1e3a0984888a2213a8e074c48b0c Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Fri, 21 Oct 2011 18:16:16 -0700 Subject: [PATCH] fresh api. virtually whole new thing, entirely new interface and heavily modified encoder/decoder --- include/jsx_opts.hrl | 3 +- include/jsx_tokenizer.hrl | 107 -------- include/jsx_types.hrl | 37 --- src/jsx.app.src | 2 +- src/jsx.erl | 140 ++++++++--- .../jsx_scanner.hrl => src/jsx_decoder.erl | 214 +++++++++++++++- src/jsx_encoder.erl | 229 ++++++++++++++++++ src/jsx_tokenizer.erl | 110 --------- src/jsx_utils.erl | 2 + test/cases/bad_naked_number.json | 1 + test/cases/bad_naked_number.test | 3 + 11 files changed, 559 insertions(+), 289 deletions(-) delete mode 100644 include/jsx_tokenizer.hrl delete mode 100644 include/jsx_types.hrl rename include/jsx_scanner.hrl => src/jsx_decoder.erl (80%) create mode 100644 src/jsx_encoder.erl delete mode 100644 src/jsx_tokenizer.erl create mode 100644 test/cases/bad_naked_number.json create mode 100644 test/cases/bad_naked_number.test diff --git a/include/jsx_opts.hrl b/include/jsx_opts.hrl index 380de54..a572170 100644 --- a/include/jsx_opts.hrl +++ b/include/jsx_opts.hrl @@ -1,4 +1,5 @@ -record(opts, { loose_unicode = false, - escape_forward_slash = false + escape_forward_slash = false, + explicit_end = false }). \ No newline at end of file diff --git a/include/jsx_tokenizer.hrl b/include/jsx_tokenizer.hrl deleted file mode 100644 index 1070dac..0000000 --- a/include/jsx_tokenizer.hrl +++ /dev/null @@ -1,107 +0,0 @@ --ifndef(error). --define(error(Args), - erlang:error(badarg, Args) -). --endif. - - --ifndef(incomplete). --define(incomplete(State, T, Stack, Opts), - {ok, lists:reverse(T), fun(Stream) when is_list(Stream) -> - State(Stream, [], Stack, Opts) - end - } -). --endif. - - --ifndef(event). --define(event(Event, State, Rest, T, Stack, Opts), - State(Rest, Event ++ T, Stack, Opts) -). --endif. - - - - -start({string, String}, [], [], Opts) when is_binary(String); is_list(String) -> - {ok, - [{string, unicode:characters_to_list(jsx_utils:json_escape(String, Opts))}, end_json], - fun(X) when is_list(X) -> ?error([X, [], [], Opts]) end - }; -start({float, Float}, [], [], Opts) when is_float(Float) -> - {ok, - [{float, Float}, end_json], - fun(X) when is_list(X) -> ?error([X, [], [], Opts]) end - }; -start({integer, Int}, [], [], Opts) when is_integer(Int) -> - {ok, - [{integer, Int}, end_json], - fun(X) when is_list(X) -> ?error([X, [], [], Opts]) end - }; -start({literal, Atom}, [], [], Opts) when Atom == true; Atom == false; Atom == null -> - {ok, - [{literal, Atom}, end_json], - fun(X) when is_list(X) -> ?error([X, [], [], Opts]) end - }; -%% third parameter is a stack to match end_foos to start_foos -start(Forms, [], [], Opts) -> list_or_object(Forms, [], [], Opts). - - -list_or_object([start_object|Forms], T, Stack, Opts) -> - ?event([start_object], key, Forms, T, [object] ++ Stack, Opts); -list_or_object([start_array|Forms], T, Stack, Opts) -> - ?event([start_array], value, Forms, T, [array] ++ Stack, Opts); -list_or_object([], T, Stack, Opts) -> ?incomplete(list_or_object, T, Stack, Opts); -list_or_object(Forms, T, Stack, Opts) -> ?error([Forms, T, Stack, Opts]). - - -key([{key, Key}|Forms], T, Stack, Opts) when is_binary(Key); is_list(Key) -> - ?event([{key, unicode:characters_to_list(jsx_utils:json_escape(Key, Opts))}], - value, Forms, T, Stack, Opts - ); -key([end_object|Forms], T, [object|Stack], Opts) -> - ?event([end_object], maybe_done, Forms, T, Stack, Opts); -key([], T, Stack, Opts) -> ?incomplete(key, T, Stack, Opts); -key(Forms, T, Stack, Opts) -> ?error([Forms, T, Stack, Opts]). - - -value([{string, S}|Forms], T, Stack, Opts) when is_binary(S); is_list(S) -> - ?event([{string, unicode:characters_to_list(jsx_utils:json_escape(S, Opts))}], - maybe_done, Forms, T, Stack, Opts - ); -value([{float, F}|Forms], T, Stack, Opts) when is_float(F) -> - ?event([{float, F}], maybe_done, Forms, T, Stack, Opts); -value([{integer, I}|Forms], T, Stack, Opts) when is_integer(I) -> - ?event([{integer, I}], maybe_done, Forms, T, Stack, Opts); -value([{literal, L}|Forms], T, Stack, Opts) - when L == true; L == false; L == null -> - ?event([{literal, L}], maybe_done, Forms, T, Stack, Opts); -value([start_object|Forms], T, Stack, Opts) -> - ?event([start_object], key, Forms, T, [object] ++ Stack, Opts); -value([start_array|Forms], T, Stack, Opts) -> - ?event([start_array], maybe_done, Forms, T, [array] ++ Stack, Opts); -value([end_array|Forms], T, [array|Stack], Opts) -> - ?event([end_array], maybe_done, Forms, T, Stack, Opts); -value([], T, Stack, Opts) -> ?incomplete(value, T, Stack, Opts); -value(Forms, T, Stack, Opts) -> ?error([Forms, T, Stack, Opts]). - - -maybe_done([end_json], T, [], Opts) -> - ?event([end_json], done, [], T, [], Opts); -maybe_done([end_object|Forms], T, [object|Stack], Opts) -> - ?event([end_object], maybe_done, Forms, T, Stack, Opts); -maybe_done([end_array|Forms], T, [array|Stack], Opts) -> - ?event([end_array], maybe_done, Forms, T, Stack, Opts); -maybe_done(Forms, T, [object|_] = Stack, Opts) -> key(Forms, T, Stack, Opts); -maybe_done(Forms, T, [array|_] = Stack, Opts) -> value(Forms, T, Stack, Opts); -maybe_done([], T, Stack, Opts) -> ?incomplete(maybe_done, T, Stack, Opts); -maybe_done(Forms, T, Stack, Opts) -> ?error([Forms, T, Stack, Opts]). - - -done([], T, [], Opts) -> - {ok, lists:reverse(T), fun(X) when is_list(X) -> - done(X, T, [], Opts) - end - }; -done(Forms, T, Stack, Opts) -> ?error([Forms, T, Stack, Opts]). \ No newline at end of file diff --git a/include/jsx_types.hrl b/include/jsx_types.hrl deleted file mode 100644 index a928ea1..0000000 --- a/include/jsx_types.hrl +++ /dev/null @@ -1,37 +0,0 @@ --type jsx_opts() :: [jsx_opt()]. --type jsx_opt() :: loose_unicode - | escape_forward_slashes. - - --type jsx_event() :: start_object - | end_object - | start_array - | end_array - | end_json - | {key, list()} - | {string, list()} - | {integer, integer()} - | {float, float()} - | {literal, true} - | {literal, false} - | {literal, null}. - - --type jsx_encodeable() :: jsx_event() | [jsx_encodeable()]. - - --type jsx_iterator() :: jsx_scanner(). - - --type jsx_scanner() :: jsx_decoder() | jsx_tokenizer(). - - --type jsx_decoder() :: fun((binary()) -> jsx_iterator_result()). --type jsx_tokenizer() :: fun((jsx_encodeable()) -> jsx_iterator_result()). - - --type jsx_iterator_result() :: - {jsx, jsx_event(), fun(() -> jsx_iterator_result())} - | {jsx, [jsx_event()], fun(() -> jsx_iterator_result())} - | {jsx, incomplete, jsx_iterator()} - | {error, {badjson, any()}}. \ No newline at end of file diff --git a/src/jsx.app.src b/src/jsx.app.src index c12e084..d007316 100644 --- a/src/jsx.app.src +++ b/src/jsx.app.src @@ -4,7 +4,7 @@ {vsn, "0.10.0"}, {modules, [ jsx, - jsx_tokenizer, + jsx_encoder, jsx_decoder, jsx_utils ]}, diff --git a/src/jsx.erl b/src/jsx.erl index 5487f80..63c7b8a 100644 --- a/src/jsx.erl +++ b/src/jsx.erl @@ -23,25 +23,99 @@ -module(jsx). - -%% the core parser api -export([scanner/0, scanner/1]). - --include("../include/jsx_types.hrl"). +-export([encoder/0, encoder/1]). +-export([decoder/0, decoder/1]). +-export([fold/3, fold/4]). --spec scanner() -> jsx_scanner(). +%% various semi-useful types with nowhere else to hang out + +-type events() :: [event()]. +-type event() :: start_object + | end_object + | start_array + | end_array + | end_json + | {key, list()} + | {string, list()} + | {integer, integer()} + | {float, float()} + | {literal, true} + | {literal, false} + | {literal, null}. + +%% definition of the opts record for the encoder and decoder +-include("../include/jsx_opts.hrl"). + +-type opts() :: [opt()]. +-type opt() :: loose_unicode | escape_forward_slashes | explicit_end. + +-type scanner() :: decoder() | encoder(). + + +-spec scanner() -> scanner(). +-spec scanner(OptsList::opts()) -> scanner(). + scanner() -> scanner([]). --spec scanner(OptsList::jsx_opts()) -> jsx_scanner(). -scanner(OptsList) -> - fun(Stream) when is_binary(Stream) -> - (jsx_decoder:decoder(OptsList))(Stream) - ; (Stream) when is_list(Stream); is_tuple(Stream) -> - (jsx_tokenizer:tokenizer(OptsList))(Stream) +scanner(OptsList) when is_list(OptsList) -> + fun(JSON) when is_binary(JSON) -> (decoder(OptsList))(JSON) + ; (Terms) when is_list(Terms); is_tuple(Terms) -> + (encoder(OptsList))(Terms) end. +-type decoder() :: fun((binary()) -> {ok, events()} | {incomplete, decoder()}). + +-spec decoder() -> decoder(). +-spec decoder(OptsList::opts()) -> decoder(). + +decoder() -> decoder([]). + +decoder(OptsList) when is_list(OptsList) -> jsx_decoder:decoder(OptsList). + + +-type encoder() :: fun((list() | tuple()) -> + {ok, events()} | {incomplete, decoder()}). + +-spec encoder() -> encoder(). +-spec encoder(OptsList::opts()) -> encoder(). + +encoder() -> encoder([]). + +encoder(OptsList) when is_list(OptsList) -> jsx_encoder:encoder(OptsList). + + +-spec fold(fun((Elem::event(), AccIn::any()) -> AccOut::any()), + AccInitial::any(), + Source::binary()) -> + {ok, AccFinal::any()} | {incomplete, decoder()} + ; (fun((Elem::event(), AccIn::any()) -> AccOut::any()), + AccInitial::any(), + Source::list()) -> + {ok, AccFinal::any()} | {incomplete, encoder()}. +-spec fold(fun((Elem::event(), AccIn::any()) -> AccOut::any()), + AccInitial::any(), + Source::binary(), + OptsLists::opts()) -> + {ok, AccFinal::any()} | {incomplete, decoder()} + ; (fun((Elem::event(), AccIn::any()) -> AccOut::any()), + AccInitial::any(), + Source::list(), + OptsList::opts()) -> + {ok, AccFinal::any()} | {incomplete, encoder()}. + +fold(Fun, Acc, Source) -> fold(Fun, Acc, Source, []). + +fold(Fun, Acc, Source, Opts) -> + case (scanner(Opts))(Source) of + {ok, Events} -> lists:foldl(Fun, Acc, Events) + ; {incomplete, F} -> {incomplete, F} + end. + + + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -53,9 +127,9 @@ jsx_decoder_test_() -> encoder_decoder_equiv_test_() -> [ {"encoder/decoder equivalency", - ?_assert(begin {ok, X, _} = (jsx:scanner())( + ?_assert(begin {ok, X} = (jsx:decoder())( <<"[\"a\", 17, 3.14, true, {\"k\":false}, []]">> - ), X end =:= begin {ok, Y, _} = (jsx:scanner())( + ), X end =:= begin {ok, Y} = (jsx:encoder())( [start_array, {string, <<"a">>}, {integer, 17}, @@ -73,6 +147,16 @@ encoder_decoder_equiv_test_() -> ) } ]. + + +fold_test_() -> + [{"fold test", + ?_assert(fold(fun(_, true) -> true end, + true, + <<"[\"a\", 17, 3.14, true, {\"k\":false}, []]">>, + [] + )) + }]. jsx_decoder_gen([]) -> []; @@ -124,13 +208,13 @@ parse_tests([], _Dir, Acc) -> decode(JSON, Flags) -> try - P = jsx:scanner(Flags), - {ok, X, More} = P(JSON), - {ok, Y, _More} = More(<<" ">>), - V = X ++ Y, - case lists:reverse(V) of - [end_json|_] -> V - ; _ -> {error, badjson} + case (jsx:scanner(Flags))(JSON) of + {ok, Events} -> Events + ; {incomplete, More} -> + case More(<<" ">>) of + {ok, Events} -> Events + ; _ -> {error, badjson} + end end catch error:badarg -> {error, badjson} @@ -138,20 +222,18 @@ decode(JSON, Flags) -> incremental_decode(<>, Flags) -> - P = jsx:scanner(Flags), - try incremental_decode_loop(P(C), Rest, []) + P = jsx:scanner(Flags ++ [explicit_end]), + try incremental_decode_loop(P(C), Rest) catch error:badarg -> io:format("~p~n", [erlang:get_stacktrace()]), {error, badjson} end. -incremental_decode_loop({ok, X, More}, <<>>, Acc) -> - {ok, Y, _} = More(<<" ">>), %% clear any naked numbers - V = Acc ++ X ++ Y, - case lists:reverse(V) of - [end_json|_] -> V +incremental_decode_loop({incomplete, More}, <<>>) -> + case More(end_stream) of + {ok, X} -> X ; _ -> {error, badjson} end; -incremental_decode_loop({ok, T, More}, <>, Acc) -> - incremental_decode_loop(More(C), Rest, Acc ++ T). +incremental_decode_loop({incomplete, More}, <>) -> + incremental_decode_loop(More(C), Rest). -endif. \ No newline at end of file diff --git a/include/jsx_scanner.hrl b/src/jsx_decoder.erl similarity index 80% rename from include/jsx_scanner.hrl rename to src/jsx_decoder.erl index 7e06bb1..aaa84db 100644 --- a/include/jsx_scanner.hrl +++ b/src/jsx_decoder.erl @@ -1,3 +1,40 @@ +%% The MIT License + +%% Copyright (c) 2010 Alisdair Sullivan + +%% Permission is hereby granted, free of charge, to any person obtaining a copy +%% of this software and associated documentation files (the "Software"), to deal +%% in the Software without restriction, including without limitation the rights +%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%% copies of the Software, and to permit persons to whom the Software is +%% furnished to do so, subject to the following conditions: + +%% The above copyright notice and this permission notice shall be included in +%% all copies or substantial portions of the Software. + +%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%% THE SOFTWARE. + + +-module(jsx_decoder). + +-export([decoder/1]). + + +-spec decoder(Opts::jsx:opts()) -> jsx:decoder(). + +decoder(Opts) -> + fun(JSON) -> start(JSON, [], [], jsx_utils:parse_opts(Opts)) end. + + +-include("../include/jsx_opts.hrl"). + + %% whitespace -define(space, 16#20). -define(tab, 16#09). @@ -61,8 +98,16 @@ -ifndef(incomplete). -define(incomplete(State, Rest, Out, Stack, Opts), - {ok, lists:reverse(Out), fun(Stream) when is_binary(Stream) -> - State(<>, [], Stack, Opts) + {incomplete, fun(Stream) when is_binary(Stream) -> + State(<>, Out, Stack, Opts) + ; (end_stream) -> + case State(<>/binary>>, + Out, + Stack, + Opts#opts{explicit_end=false}) of + {incomplete, _} -> ?error([Rest, Out, Stack, Opts]) + ; {ok, Events} -> {ok, Events} + end end } ). @@ -643,5 +688,166 @@ maybe_done(Bin, Out, Stack, Opts) -> done(<>, Out, [], Opts) when ?is_whitespace(S) -> done(Rest, Out, [], Opts); -done(<<>>, Out, [], Opts) -> ?incomplete(done, <<>>, Out, [], Opts); -done(Bin, Out, Stack, Opts) -> ?error([Bin, Out, Stack, Opts]). \ No newline at end of file +done(<<>>, Out, [], Opts = #opts{explicit_end=true}) -> + {incomplete, fun(Stream) when is_binary(Stream) -> + done(<>, Out, [], Opts) + ; (end_stream) -> + {ok, lists:reverse(Out)} + end + }; +done(<<>>, Out, [], _Opts) -> {ok, lists:reverse(Out)}; +done(Bin, Out, Stack, Opts) -> ?error([Bin, Out, Stack, Opts]). + + + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + + +noncharacters_test_() -> + [ + {"noncharacters - badjson", + ?_assertEqual(check_bad(noncharacters()), []) + }, + {"noncharacters - replaced", + ?_assertEqual(check_replaced(noncharacters()), []) + } + ]. + +extended_noncharacters_test_() -> + [ + {"extended noncharacters - badjson", + ?_assertEqual(check_bad(extended_noncharacters()), []) + }, + {"extended noncharacters - replaced", + ?_assertEqual(check_replaced(extended_noncharacters()), []) + } + ]. + +surrogates_test_() -> + [ + {"surrogates - badjson", + ?_assertEqual(check_bad(surrogates()), []) + }, + {"surrogates - replaced", + ?_assertEqual(check_replaced(surrogates()), []) + } + ]. + +control_test_() -> + [ + {"control characters - badjson", + ?_assertEqual(check_bad(control_characters()), []) + } + ]. + +reserved_test_() -> + [ + {"reserved noncharacters - badjson", + ?_assertEqual(check_bad(reserved_space()), []) + }, + {"reserved noncharacters - replaced", + ?_assertEqual(check_replaced(reserved_space()), []) + } + ]. + +zero_test_() -> + [ + {"nullbyte - badjson", + ?_assertEqual(check_bad(zero()), []) + } + ]. + +good_characters_test_() -> + [ + {"acceptable codepoints", + ?_assertEqual(check_good(good()), []) + }, + {"acceptable extended", + ?_assertEqual(check_good(good_extended()), []) + } + ]. + + +check_bad(List) -> + lists:dropwhile(fun({_, {error, badjson}}) -> true ; (_) -> false end, + check(List, [], []) + ). + +check_replaced(List) -> + lists:dropwhile(fun({_, [{string, [16#fffd]}|_]}) -> + true + ; (_) -> + false + end, + check(List, [loose_unicode], []) + ). + +check_good(List) -> + lists:dropwhile(fun({_, [{string, _}]}) -> true ; (_) -> false end, + check(List, [], []) + ). + +check([], _Opts, Acc) -> Acc; +check([H|T], Opts, Acc) -> + R = decode(to_fake_utf(H, utf8), Opts), + check(T, Opts, [{H, R}] ++ Acc). + + +decode(JSON, Opts) -> + try + {ok, Events} = (decoder(Opts))(JSON), + loop(Events, []) + catch + error:badarg -> {error, badjson} + end. + + +loop([end_json], Acc) -> lists:reverse(Acc); +loop([Event|Events], Acc) -> loop(Events, [Event] ++ Acc); +loop(_, _) -> {error, badjson}. + + + +noncharacters() -> lists:seq(16#fffe, 16#ffff). + +extended_noncharacters() -> + [16#1fffe, 16#1ffff, 16#2fffe, 16#2ffff] + ++ [16#3fffe, 16#3ffff, 16#4fffe, 16#4ffff] + ++ [16#5fffe, 16#5ffff, 16#6fffe, 16#6ffff] + ++ [16#7fffe, 16#7ffff, 16#8fffe, 16#8ffff] + ++ [16#9fffe, 16#9ffff, 16#afffe, 16#affff] + ++ [16#bfffe, 16#bffff, 16#cfffe, 16#cffff] + ++ [16#dfffe, 16#dffff, 16#efffe, 16#effff] + ++ [16#ffffe, 16#fffff, 16#10fffe, 16#10ffff]. + +surrogates() -> lists:seq(16#d800, 16#dfff). + +control_characters() -> lists:seq(1, 31). + +reserved_space() -> lists:seq(16#fdd0, 16#fdef). + +zero() -> [0]. + +good() -> [32, 33] + ++ lists:seq(16#23, 16#5b) + ++ lists:seq(16#5d, 16#d7ff) + ++ lists:seq(16#e000, 16#fdcf) + ++ lists:seq(16#fdf0, 16#fffd). + +good_extended() -> lists:seq(16#100000, 16#10fffd). + +%% erlang refuses to encode certain codepoints, so fake them all +to_fake_utf(N, utf8) when N < 16#0080 -> <<34/utf8, N:8, 34/utf8>>; +to_fake_utf(N, utf8) when N < 16#0800 -> + <<0:5, Y:5, X:6>> = <>, + <<34/utf8, 2#110:3, Y:5, 2#10:2, X:6, 34/utf8>>; +to_fake_utf(N, utf8) when N < 16#10000 -> + <> = <>, + <<34/utf8, 2#1110:4, Z:4, 2#10:2, Y:6, 2#10:2, X:6, 34/utf8>>; +to_fake_utf(N, utf8) -> + <<0:3, W:3, Z:6, Y:6, X:6>> = <>, + <<34/utf8, 2#11110:5, W:3, 2#10:2, Z:6, 2#10:2, Y:6, 2#10:2, X:6, 34/utf8>>. + + +-endif. \ No newline at end of file diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl new file mode 100644 index 0000000..e544795 --- /dev/null +++ b/src/jsx_encoder.erl @@ -0,0 +1,229 @@ +%% The MIT License + +%% Copyright (c) 2011 Alisdair Sullivan + +%% Permission is hereby granted, free of charge, to any person obtaining a copy +%% of this software and associated documentation files (the "Software"), to deal +%% in the Software without restriction, including without limitation the rights +%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%% copies of the Software, and to permit persons to whom the Software is +%% furnished to do so, subject to the following conditions: + +%% The above copyright notice and this permission notice shall be included in +%% all copies or substantial portions of the Software. + +%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%% THE SOFTWARE. + + +-module(jsx_encoder). + +-export([encoder/1]). + + +-spec encoder(OptsList::jsx:opts()) -> jsx:encoder(). + +encoder(OptsList) -> + fun(Forms) -> start(Forms, [], [], jsx_utils:parse_opts(OptsList)) end. + + +-include("../include/jsx_opts.hrl"). + + +-ifndef(error). +-define(error(Args), + erlang:error(badarg, Args) +). +-endif. + + +-ifndef(incomplete). +-define(incomplete(State, T, Stack, Opts), + {incomplete, fun(Stream) when is_list(Stream) -> + State(Stream, T, Stack, Opts) + end + } +). +-endif. + + +-ifndef(event). +-define(event(Event, State, Rest, T, Stack, Opts), + State(Rest, Event ++ T, Stack, Opts) +). +-endif. + + + + +start({string, String}, [], [], Opts) when is_binary(String); is_list(String) -> + {ok, + [{string, + unicode:characters_to_list(jsx_utils:json_escape(String, Opts))}, + end_json + ] + }; +start({float, Float}, [], [], _Opts) when is_float(Float) -> + {ok, + [{float, Float}, end_json] + }; +start({integer, Int}, [], [], _Opts) when is_integer(Int) -> + {ok, + [{integer, Int}, end_json] + }; +start({literal, Atom}, [], [], _Opts) + when Atom == true; Atom == false; Atom == null -> + {ok, + [{literal, Atom}, end_json] + }; +%% third parameter is a stack to match end_foos to start_foos +start(Forms, [], [], Opts) when is_list(Forms) -> + list_or_object(Forms, [], [], Opts); +start(Raw, [], [], Opts) -> + start(term_to_event(Raw), [], [], Opts). + + +list_or_object([start_object|Forms], T, Stack, Opts) -> + ?event([start_object], key, Forms, T, [object] ++ Stack, Opts); +list_or_object([start_array|Forms], T, Stack, Opts) -> + ?event([start_array], value, Forms, T, [array] ++ Stack, Opts); +list_or_object([], T, Stack, Opts) -> + ?incomplete(list_or_object, T, Stack, Opts); +list_or_object(Forms, T, Stack, Opts) -> ?error([Forms, T, Stack, Opts]). + + +key([{key, Key}|Forms], T, Stack, Opts) when is_binary(Key); is_list(Key) -> + ?event([{key, + unicode:characters_to_list(jsx_utils:json_escape(Key, Opts)) + }], + value, Forms, T, Stack, Opts + ); +key([end_object|Forms], T, [object|Stack], Opts) -> + ?event([end_object], maybe_done, Forms, T, Stack, Opts); +key([], T, Stack, Opts) -> ?incomplete(key, T, Stack, Opts); +key(Forms, T, Stack, Opts) -> ?error([Forms, T, Stack, Opts]). + + +value([{string, S}|Forms], T, Stack, Opts) when is_binary(S); is_list(S) -> + ?event([{string, unicode:characters_to_list(jsx_utils:json_escape(S, Opts))}], + maybe_done, Forms, T, Stack, Opts + ); +value([{float, F}|Forms], T, Stack, Opts) when is_float(F) -> + ?event([{float, F}], maybe_done, Forms, T, Stack, Opts); +value([{integer, I}|Forms], T, Stack, Opts) when is_integer(I) -> + ?event([{integer, I}], maybe_done, Forms, T, Stack, Opts); +value([{literal, L}|Forms], T, Stack, Opts) + when L == true; L == false; L == null -> + ?event([{literal, L}], maybe_done, Forms, T, Stack, Opts); +value([start_object|Forms], T, Stack, Opts) -> + ?event([start_object], key, Forms, T, [object] ++ Stack, Opts); +value([start_array|Forms], T, Stack, Opts) -> + ?event([start_array], maybe_done, Forms, T, [array] ++ Stack, Opts); +value([end_array|Forms], T, [array|Stack], Opts) -> + ?event([end_array], maybe_done, Forms, T, Stack, Opts); +value([], T, Stack, Opts) -> ?incomplete(value, T, Stack, Opts); +value(Forms, T, Stack, Opts) -> ?error([Forms, T, Stack, Opts]). + + +maybe_done([end_json], T, [], Opts) -> + ?event([end_json], done, [], T, [], Opts); +maybe_done([end_object|Forms], T, [object|Stack], Opts) -> + ?event([end_object], maybe_done, Forms, T, Stack, Opts); +maybe_done([end_array|Forms], T, [array|Stack], Opts) -> + ?event([end_array], maybe_done, Forms, T, Stack, Opts); +maybe_done(Forms, T, [object|_] = Stack, Opts) -> key(Forms, T, Stack, Opts); +maybe_done(Forms, T, [array|_] = Stack, Opts) -> value(Forms, T, Stack, Opts); +maybe_done([], T, Stack, Opts) -> ?incomplete(maybe_done, T, Stack, Opts); +maybe_done(Forms, T, Stack, Opts) -> ?error([Forms, T, Stack, Opts]). + + +done([], T, [], _Opts) -> + {ok, lists:reverse(T)}; +done(Forms, T, Stack, Opts) -> ?error([Forms, T, Stack, Opts]). + + +term_to_event(X) when is_integer(X) -> {integer, X}; +term_to_event(X) when is_float(X) -> {float, X}; +term_to_event(String) when is_binary(String) -> {string, String}; +term_to_event(true) -> {literal, true}; +term_to_event(false) -> {literal, false}; +term_to_event(null) -> {literal, null}. + + + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +encode(Terms) -> + try case (jsx:encoder([]))(Terms) of + {ok, Terms} -> + true + %% matches [foo, end_json], aka naked terms + ; {ok, [Terms, end_json]} -> + true + end + catch + error:badarg -> false + end. + + +encode_test_() -> + [ + {"empty object", ?_assert(encode([start_object, end_object, end_json]))}, + {"empty array", ?_assert(encode([start_array, end_array, end_json]))}, + {"nested empty objects", ?_assert(encode([start_object, + {key, "empty object"}, + start_object, + {key, "empty object"}, + start_object, + end_object, + end_object, + end_object, + end_json + ]))}, + {"nested empty arrays", ?_assert(encode([start_array, + start_array, + start_array, + end_array, + end_array, + end_array, + end_json + ]))}, + {"simple object", ?_assert(encode([start_object, + {key, "a"}, + {string, "hello"}, + {key, "b"}, + {integer, 1}, + {key, "c"}, + {float, 1.0}, + {key, "d"}, + {literal, true}, + end_object, + end_json + ]))}, + {"simple array", ?_assert(encode([start_array, + {string, "hello"}, + {integer, 1}, + {float, 1.0}, + {literal, true}, + end_array, + end_json + ]))}, + {"unbalanced array", ?_assertNot(encode([start_array, + end_array, + blerg, + end_array, + end_json + ]))}, + {"naked string", ?_assert(encode({string, "hello"}))}, + {"naked literal", ?_assert(encode({literal, true}))}, + {"naked integer", ?_assert(encode({integer, 1}))}, + {"naked float", ?_assert(encode({float, 1.0}))} + ]. + +-endif. \ No newline at end of file diff --git a/src/jsx_tokenizer.erl b/src/jsx_tokenizer.erl deleted file mode 100644 index 9298345..0000000 --- a/src/jsx_tokenizer.erl +++ /dev/null @@ -1,110 +0,0 @@ -%% The MIT License - -%% Copyright (c) 2011 Alisdair Sullivan - -%% Permission is hereby granted, free of charge, to any person obtaining a copy -%% of this software and associated documentation files (the "Software"), to deal -%% in the Software without restriction, including without limitation the rights -%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -%% copies of the Software, and to permit persons to whom the Software is -%% furnished to do so, subject to the following conditions: - -%% The above copyright notice and this permission notice shall be included in -%% all copies or substantial portions of the Software. - -%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -%% THE SOFTWARE. - - --module(jsx_tokenizer). - - --include("../include/jsx_types.hrl"). - - --export([tokenizer/1]). - - --spec tokenizer(OptsList::jsx_opts()) -> jsx_tokenizer(). -tokenizer(OptsList) -> - fun(Forms) -> start(Forms, [], [], jsx_utils:parse_opts(OptsList)) end. - --include("../include/jsx_opts.hrl"). - --include("../include/jsx_tokenizer.hrl"). - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). - -encode(Terms) -> - try case (jsx:scanner([]))(Terms) of - {ok, Terms, _} -> - true - %% matches [foo, end_json], aka naked terms - ; {ok, [Terms, end_json], _} -> - true - end - catch - error:badarg -> false - end. - - -encode_test_() -> - [ - {"empty object", ?_assert(encode([start_object, end_object, end_json]))}, - {"empty array", ?_assert(encode([start_array, end_array, end_json]))}, - {"nested empty objects", ?_assert(encode([start_object, - {key, "empty object"}, - start_object, - {key, "empty object"}, - start_object, - end_object, - end_object, - end_object, - end_json - ]))}, - {"nested empty arrays", ?_assert(encode([start_array, - start_array, - start_array, - end_array, - end_array, - end_array, - end_json - ]))}, - {"simple object", ?_assert(encode([start_object, - {key, "a"}, - {string, "hello"}, - {key, "b"}, - {integer, 1}, - {key, "c"}, - {float, 1.0}, - {key, "d"}, - {literal, true}, - end_object, - end_json - ]))}, - {"simple array", ?_assert(encode([start_array, - {string, "hello"}, - {integer, 1}, - {float, 1.0}, - {literal, true}, - end_array, - end_json - ]))}, - {"unbalanced array", ?_assertNot(encode([start_array, - end_array, - end_array, - end_json - ]))}, - {"naked string", ?_assert(encode({string, "hello"}))}, - {"naked literal", ?_assert(encode({literal, true}))}, - {"naked integer", ?_assert(encode({integer, 1}))}, - {"naked float", ?_assert(encode({float, 1.0}))} - ]. - --endif. \ No newline at end of file diff --git a/src/jsx_utils.erl b/src/jsx_utils.erl index 72528b2..a8160f8 100644 --- a/src/jsx_utils.erl +++ b/src/jsx_utils.erl @@ -39,6 +39,8 @@ parse_opts([loose_unicode|Rest], Opts) -> parse_opts(Rest, Opts#opts{loose_unicode=true}); parse_opts([escape_forward_slash|Rest], Opts) -> parse_opts(Rest, Opts#opts{escape_forward_slash=true}); +parse_opts([explicit_end|Rest], Opts) -> + parse_opts(Rest, Opts#opts{explicit_end=true}); parse_opts(_, _) -> {error, badarg}. diff --git a/test/cases/bad_naked_number.json b/test/cases/bad_naked_number.json new file mode 100644 index 0000000..92880af --- /dev/null +++ b/test/cases/bad_naked_number.json @@ -0,0 +1 @@ +1 1 \ No newline at end of file diff --git a/test/cases/bad_naked_number.test b/test/cases/bad_naked_number.test new file mode 100644 index 0000000..2781ebb --- /dev/null +++ b/test/cases/bad_naked_number.test @@ -0,0 +1,3 @@ +{name, "bad naked number"}. +{jsx, {error, badjson}}. +{json, "bad_naked_number.json"}.