From 37f9d2a07a8bfa6fbe8fafa9aca9b918e29ce0b7 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Mon, 21 Jun 2010 23:06:55 -0700 Subject: [PATCH] brought test suite up to date, fixed a dumb error that occured when parsing integers larger than 100 --- makefile | 9 +++ priv/jsx_compile.escript | 3 + priv/jsx_expand.escript | 2 +- priv/jsx_test.escript | 107 +++++++++++++++++++++++++++++++ src/jsx.erl | 112 +++++++++++++++++++++------------ src/jsx_decoder.erl | 4 +- test/cases/naked_number_g.json | 1 + test/cases/naked_number_g.test | 1 + 8 files changed, 196 insertions(+), 43 deletions(-) create mode 100755 priv/jsx_test.escript create mode 100644 test/cases/naked_number_g.json create mode 100644 test/cases/naked_number_g.test diff --git a/makefile b/makefile index 13cccaa..3903544 100644 --- a/makefile +++ b/makefile @@ -7,6 +7,15 @@ compile: expand src/jsx_utf32\ src/jsx_utf32le +debug: expand + ./priv/jsx_compile.escript -d\ + src/jsx\ + src/jsx_utf8\ + src/jsx_utf16\ + src/jsx_utf16le\ + src/jsx_utf32\ + src/jsx_utf32le + expand: ./priv/jsx_expand.escript utf8 utf16 utf16le utf32 utf32le diff --git a/priv/jsx_compile.escript b/priv/jsx_compile.escript index 94f6984..92669b5 100755 --- a/priv/jsx_compile.escript +++ b/priv/jsx_compile.escript @@ -22,6 +22,9 @@ %% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN %% THE SOFTWARE. +main(["-d"|Mods]) -> + compile_files(Mods, [{outdir, "ebin"}, debug_info]); + main(Mods) -> compile_files(Mods, [{outdir, "ebin"}]). diff --git a/priv/jsx_expand.escript b/priv/jsx_expand.escript index 0e38fa6..580567b 100755 --- a/priv/jsx_expand.escript +++ b/priv/jsx_expand.escript @@ -27,7 +27,7 @@ main(Backends) -> to_abf(Backend) -> case os:getenv("TMPDIR") of - false -> Out = "/tmp" + false -> Out = "." ; Out -> Out end, Name = to_modname(Backend), diff --git a/priv/jsx_test.escript b/priv/jsx_test.escript new file mode 100755 index 0000000..0d1ba45 --- /dev/null +++ b/priv/jsx_test.escript @@ -0,0 +1,107 @@ +#!/usr/bin/env escript + +%% The MIT License + +%% Copyright (c) 2010 Alisdair Sullivan + +%% Permission is hereby granted, free of charge, to any person obtaining a copy +%% of this software and associated documentation files (the "Software"), to deal +%% in the Software without restriction, including without limitation the rights +%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%% copies of the Software, and to permit persons to whom the Software is +%% furnished to do so, subject to the following conditions: + +%% The above copyright notice and this permission notice shall be included in +%% all copies or substantial portions of the Software. + +%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%% THE SOFTWARE. + +-mode(compile). + +main([Path]) -> + test(Path). + + +test(Dir) -> + code:add_path("ebin"), + + ValidJSONTests = load_tests(Dir), + + etap:plan(length(ValidJSONTests) * 5), + Before = erlang:now(), + run_tests(ValidJSONTests), + After = erlang:now(), + etap:end_tests(), + + io:format("Elapsed Time: ~p~n", [timer:now_diff(After, Before)]). + + +load_tests(Dir) -> + TestSpecs = filelib:wildcard("*.test", Dir), + load_tests(TestSpecs, Dir, []). + +load_tests([], _Dir, Acc) -> + lists:reverse(Acc); +load_tests([Test|Rest], Dir, Acc) -> + try + TestName = filename:basename(Test, ".test"), + {ok, JSON} = file:read_file(Dir ++ "/" ++ TestName ++ ".json"), + case file:consult(Dir ++ "/" ++ Test) of + {ok, [Events]} -> + load_tests(Rest, Dir, [{TestName, JSON, Events, []}] ++ Acc) + ; {ok, [Events, Flags]} -> + load_tests(Rest, Dir, [{TestName, JSON, Events, Flags}] ++ Acc) + end + catch _:_ -> load_tests(Rest, Dir, Acc) end. + +run_tests([]) -> + ok; +run_tests([{TestName, JSON, Events, Flags}|Rest]) -> + etap:is(decode(JSON, Flags), Events, TestName ++ ": utf8"), + etap:is(incremental_decode(JSON, Flags), Events, TestName ++ ": incremental utf8"), + etap:is(decode(to_utf16(JSON), Flags), Events, TestName ++ ": utf16"), + etap:is(incremental_decode(to_utf16(JSON), Flags), Events, TestName ++ ": incremental utf16"), + etap:is(decode(to_utf16le(JSON), Flags), Events, TestName ++ ": utf16le"), + etap:is(incremental_decode(to_utf16le(JSON), Flags), Events, TestName ++ ": incremental utf16le"), + etap:is(decode(to_utf32(JSON), Flags), Events, TestName ++ ": utf32"), + etap:is(incremental_decode(to_utf32(JSON), Flags), Events, TestName ++ ": incremental utf32"), + etap:is(decode(to_utf32le(JSON), Flags), Events, TestName ++ ": utf32le"), + etap:is(incremental_decode(to_utf32le(JSON), Flags), Events, TestName ++ ": incremental utf32le"), + run_tests(Rest). + +incremental_decode(<>, Flags) -> + P = jsx:parser(Flags), + incremental_decode(P(C), Rest, []). + +incremental_decode({incomplete, Next, _}, <>, Acc) -> + incremental_decode(Next(C), Rest, Acc); +incremental_decode({incomplete, _, Force}, <<>>, Acc) -> + incremental_decode(Force(), <<>>, Acc); +incremental_decode({event, end_json, Next}, <>, Acc) -> + incremental_decode(Next(C), Rest, Acc); +incremental_decode({event, end_json, _}, <<>>, Acc) -> + lists:reverse(Acc); +incremental_decode({event, Event, F}, Rest, Acc) -> + incremental_decode(F(), Rest, [Event] ++ Acc). + + +decode(JSON, Flags) -> + {ok, Result} = jsx:fold(fun(end_json, S) -> + lists:reverse(S) + ;(E, S) -> + [E] ++ S + end, [], JSON, Flags), + Result. + +to_utf16(Bin) -> unicode:characters_to_binary(Bin, utf8, utf16). +to_utf16le(Bin) -> unicode:characters_to_binary(Bin, utf8, {utf16,little}). +to_utf32(Bin) -> unicode:characters_to_binary(Bin, utf8, utf32). +to_utf32le(Bin) -> unicode:characters_to_binary(Bin, utf8, {utf32,little}). + + \ No newline at end of file diff --git a/src/jsx.erl b/src/jsx.erl index a8c28d2..317d0a5 100644 --- a/src/jsx.erl +++ b/src/jsx.erl @@ -28,12 +28,13 @@ -export([parser/0, parser/1]). %% example usage of core api --export([decode/1, decode/2]). -export([is_json/1, is_json/2]). --export([fold/3, fold/4]). +-export([decode/1, decode/2, decode/3, decode/4]). +-export([fold/1, fold/2, fold/3, fold/4]). %% types for function specifications -include("jsx_types.hrl"). + -spec parser() -> jsx_parser(). @@ -58,36 +59,52 @@ start(F, OptsList) -> fun(Stream) -> F(Stream, Opts) end. -%% decode is an example decoder using the jsx api. it converts the events into a simple -%% list and converts incomplete parses into errors. - --spec decode(JSON::json()) -> {ok, [jsx_event(),...]} | {error, badjson}. --spec decode(JSON::json(), Opts::jsx_opts()) -> {ok, [jsx_event(),...]} | {error, badjson}. - -decode(JSON) -> - decode(JSON, []). - -decode(JSON, Opts) -> - fold(fun(end_json, State) -> - lists:reverse(State) - ;(Event, State) -> [Event] ++ State end, - [], JSON, Opts). - - -spec is_json(JSON::json()) -> true | false. -spec is_json(JSON::json(), Opts::jsx_opts()) -> true | false. is_json(JSON) -> is_json(JSON, []). - + is_json(JSON, Opts) -> case fold(fun(end_json, ok) -> true ;(_, _) -> ok end, ok, JSON, Opts) of {incomplete, _} -> false ; {error, _} -> false ; {ok, true} -> true end. + +-spec decode(JSON::json()) -> {ok, [jsx_event(),...]} | {error, atom()}. +-spec decode(JSON::json(), Parse::jsx_opts() | jsx_parser()) -> {ok, [jsx_event(),...]} | {error, atom()}. +-spec decode(F::fun((jsx_event(), any()) -> any()), + Acc::any(), + JSON::json()) -> + {ok, any()} | {error, atom()}. +-spec decode(F::fun((jsx_event(), any()) -> any()), + Acc::any(), + JSON::json(), + Parse::jsx_opts() | jsx_parser()) -> + {ok, any()} | {error, atom()}. +decode(JSON) -> + decode(JSON, []). + +decode(JSON, Parse) -> + F = fun(end_json, S) -> lists:reverse(S) ;(E, S) -> [E] ++ S end, + decode(F, [], JSON, Parse). + +decode(F, Acc, JSON) -> + decode(F, Acc, JSON, []). + +decode(F, Acc, JSON, Parse) -> + case fold(F, Acc, JSON, Parse) of + {ok, Result} -> {ok, Result} + ; _ -> {error, badjson} + end. + +-spec fold(JSON::json()) -> + {ok, [jsx_event(),...]} | {incomplete, jsx_parser(), fun(() -> jsx_parser_result())} | {error, atom()}. +-spec fold(JSON::json(), Parse::jsx_opts() | jsx_parser()) -> + {ok, [jsx_event(),...]} | {incomplete, jsx_parser(), fun(() -> jsx_parser_result())} | {error, atom()}. -spec fold(F::fun((jsx_event(), any()) -> any()), Acc::any(), JSON::json()) -> @@ -103,6 +120,13 @@ is_json(JSON, Opts) -> Parser::jsx_parser()) -> {ok, any()} | {incomplete, jsx_parser(), fun(() -> jsx_parser_result())} | {error, atom()}. +fold(JSON) -> + fold(JSON, []). + +fold(JSON, Parse) -> + F = fun(end_json, S) -> lists:reverse(S) ;(E, S) -> [E] ++ S end, + fold(F, [], JSON, Parse). + fold(F, Acc, JSON) -> P = jsx:parser(), fold(F, Acc, JSON, P). @@ -200,31 +224,39 @@ detect_encoding(<> = JSON, Opts) when X =/= 0, Y =/= 0 -> %% to conclusively determine the encoding correctly. below is an attempt to solve %% the problem detect_encoding(<>, Opts) when X =/= 0 -> - try jsx_utf8:parse(<>, Opts) - catch error:function_clause -> - {incomplete, fun(Stream) -> - detect_encoding(<>, Opts) - end} - end; + {incomplete, + fun(Stream) -> detect_encoding(<>, Opts) end, + fun() -> try + {incomplete, _, Force} = jsx_utf8:parse(<>, Opts), + Force() + catch error:function_clause -> {error, badjson} + end + end + }; detect_encoding(<<0, X>>, Opts) when X =/= 0 -> - try jsx_utf16:parse(<<0, X>>, Opts) - catch error:function_clause -> - {incomplete, fun(Stream) -> - detect_encoding(<<0, X, Stream/binary>>, Opts) - end} - end; + {incomplete, + fun(Stream) -> detect_encoding(<<0, X, Stream/binary>>, Opts) end, + fun() -> try + {incomplete, _, Force} = jsx_utf16:parse(<<0, X>>, Opts), + Force() + catch error:function_clause -> {error, badjson} + end + end + }; detect_encoding(<>, Opts) when X =/= 0 -> - try jsx_utf16le:parse(<>, Opts) - catch error:function_clause -> - {incomplete, fun(Stream) -> - detect_encoding(<>, Opts) - end} - end; + {incomplete, + fun(Stream) -> detect_encoding(<>, Opts) end, + fun() -> try + {incomplete, _, Force} = jsx_utf16le:parse(<>, Opts), + Force() + catch error:function_clause -> {error, badjson} + end + end + }; %% not enough input, request more detect_encoding(Bin, Opts) -> {incomplete, - fun(Stream) -> - detect_encoding(<>, Opts) - end + fun(Stream) -> detect_encoding(<>, Opts) end, + fun() -> {error, badjson} end }. \ No newline at end of file diff --git a/src/jsx_decoder.erl b/src/jsx_decoder.erl index f086588..087f041 100644 --- a/src/jsx_decoder.erl +++ b/src/jsx_decoder.erl @@ -217,7 +217,7 @@ string(<>, Stack, Opts, Acc) when ?is_noncontrol(S) -> string(Bin, Stack, Opts, Acc) -> case partial_utf(Bin) of false -> {error, badjson} - ; _ -> {incomplete, fun(Stream) -> string(<>, Stack, Opts, Acc) end} + ; _ -> {incomplete, fun(Stream) -> string(<>, Stack, Opts, Acc) end, ?ferror} end. @@ -436,7 +436,7 @@ integer(<>, Stack, ?comments_enabled(Opts), Acc maybe_comment(Rest, fun(Resume) -> integer(Resume, Stack, Opts, Acc) end); integer(<<>>, [], Opts, Acc) -> {incomplete, - fun(Stream) -> zero(Stream, [], Opts, Acc) end, + fun(Stream) -> integer(Stream, [], Opts, Acc) end, fun() -> {event, {integer, lists:reverse(Acc)}, fun() -> maybe_done(<<>>, [], Opts) end} end }; integer(Bin, Stack, Opts, Acc) -> diff --git a/test/cases/naked_number_g.json b/test/cases/naked_number_g.json new file mode 100644 index 0000000..c793025 --- /dev/null +++ b/test/cases/naked_number_g.json @@ -0,0 +1 @@ +7 \ No newline at end of file diff --git a/test/cases/naked_number_g.test b/test/cases/naked_number_g.test new file mode 100644 index 0000000..5f98355 --- /dev/null +++ b/test/cases/naked_number_g.test @@ -0,0 +1 @@ +[{integer, "7"}]. \ No newline at end of file