brought test suite up to date, fixed a dumb error that occured when parsing integers larger than 100

makefile

@@ -7,6 +7,15 @@ compile: expand
 	 	src/jsx_utf32\
 	 	src/jsx_utf32le
 	
+debug: expand
+	./priv/jsx_compile.escript -d\
+		src/jsx\
+	 	src/jsx_utf8\
+	 	src/jsx_utf16\
+	 	src/jsx_utf16le\
+	 	src/jsx_utf32\
+	 	src/jsx_utf32le
+	
 expand:
 	./priv/jsx_expand.escript utf8 utf16 utf16le utf32 utf32le
     

priv/jsx_compile.escript

@@ -22,6 +22,9 @@
 %% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 %% THE SOFTWARE.
 
+main(["-d"|Mods]) ->
+    compile_files(Mods, [{outdir, "ebin"}, debug_info]);
+
 main(Mods) ->
     compile_files(Mods, [{outdir, "ebin"}]).
     

priv/jsx_expand.escript

@@ -27,7 +27,7 @@ main(Backends) ->
     
 to_abf(Backend) ->
     case os:getenv("TMPDIR") of
-        false -> Out = "/tmp"
+        false -> Out = "."
         ; Out -> Out
     end,
     Name = to_modname(Backend),

priv/jsx_test.escript

@@ -0,0 +1,107 @@
+#!/usr/bin/env escript
+
+%% The MIT License
+
+%% Copyright (c) 2010 Alisdair Sullivan <alisdairsullivan@yahoo.ca>
+
+%% Permission is hereby granted, free of charge, to any person obtaining a copy
+%% of this software and associated documentation files (the "Software"), to deal
+%% in the Software without restriction, including without limitation the rights
+%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+%% copies of the Software, and to permit persons to whom the Software is
+%% furnished to do so, subject to the following conditions:
+
+%% The above copyright notice and this permission notice shall be included in
+%% all copies or substantial portions of the Software.
+
+%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+%% THE SOFTWARE.
+
+-mode(compile).
+
+main([Path]) ->
+    test(Path).
+
+
+test(Dir) ->
+	code:add_path("ebin"),
+
+    ValidJSONTests = load_tests(Dir),
+    
+    etap:plan(length(ValidJSONTests) * 5),
+    Before = erlang:now(),
+    run_tests(ValidJSONTests),
+    After = erlang:now(),
+    etap:end_tests(),
+    
+    io:format("Elapsed Time:  ~p~n", [timer:now_diff(After, Before)]).
+
+
+load_tests(Dir) ->
+    TestSpecs = filelib:wildcard("*.test", Dir),
+    load_tests(TestSpecs, Dir, []).
+    
+load_tests([], _Dir, Acc) ->
+    lists:reverse(Acc);
+load_tests([Test|Rest], Dir, Acc) ->
+    try
+        TestName = filename:basename(Test, ".test"),
+        {ok, JSON} = file:read_file(Dir ++ "/" ++ TestName ++ ".json"),
+        case file:consult(Dir ++ "/" ++ Test) of
+            {ok, [Events]} ->
+                load_tests(Rest, Dir, [{TestName, JSON, Events, []}] ++ Acc)
+            ; {ok, [Events, Flags]} ->
+                load_tests(Rest, Dir, [{TestName, JSON, Events, Flags}] ++ Acc)
+        end
+    catch _:_ -> load_tests(Rest, Dir, Acc) end.
+    
+run_tests([]) ->
+    ok;
+run_tests([{TestName, JSON, Events, Flags}|Rest]) ->
+    etap:is(decode(JSON, Flags), Events, TestName ++ ": utf8"),
+    etap:is(incremental_decode(JSON, Flags), Events, TestName ++ ": incremental utf8"),
+    etap:is(decode(to_utf16(JSON), Flags), Events, TestName ++ ": utf16"),
+    etap:is(incremental_decode(to_utf16(JSON), Flags), Events, TestName ++ ": incremental utf16"),
+    etap:is(decode(to_utf16le(JSON), Flags), Events, TestName ++ ": utf16le"),
+    etap:is(incremental_decode(to_utf16le(JSON), Flags), Events, TestName ++ ": incremental utf16le"),
+    etap:is(decode(to_utf32(JSON), Flags), Events, TestName ++ ": utf32"),
+    etap:is(incremental_decode(to_utf32(JSON), Flags), Events, TestName ++ ": incremental utf32"),
+    etap:is(decode(to_utf32le(JSON), Flags), Events, TestName ++ ": utf32le"),
+    etap:is(incremental_decode(to_utf32le(JSON), Flags), Events, TestName ++ ": incremental utf32le"),
+    run_tests(Rest).
+    
+incremental_decode(<<C:1/binary, Rest/binary>>, Flags) ->
+	P = jsx:parser(Flags),
+	incremental_decode(P(C), Rest, []).
+	
+incremental_decode({incomplete, Next, _}, <<C:1/binary, Rest/binary>>, Acc) ->
+	incremental_decode(Next(C), Rest, Acc);
+incremental_decode({incomplete, _, Force}, <<>>, Acc) ->
+	incremental_decode(Force(), <<>>, Acc);
+incremental_decode({event, end_json, Next}, <<C:1/binary, Rest/binary>>, Acc) ->
+    incremental_decode(Next(C), Rest, Acc);
+incremental_decode({event, end_json, _}, <<>>, Acc) ->
+    lists:reverse(Acc);
+incremental_decode({event, Event, F}, Rest, Acc) ->
+	incremental_decode(F(), Rest, [Event] ++ Acc).
+
+
+decode(JSON, Flags) ->
+    {ok, Result} = jsx:fold(fun(end_json, S) -> 
+				lists:reverse(S) 
+			;(E, S) -> 
+				[E] ++ S 
+		end, [], JSON, Flags),
+	Result.
+    
+to_utf16(Bin) -> unicode:characters_to_binary(Bin, utf8, utf16).
+to_utf16le(Bin) -> unicode:characters_to_binary(Bin, utf8, {utf16,little}).
+to_utf32(Bin) -> unicode:characters_to_binary(Bin, utf8, utf32).
+to_utf32le(Bin) -> unicode:characters_to_binary(Bin, utf8, {utf32,little}).
+
+    

src/jsx.erl

@@ -28,12 +28,13 @@
 -export([parser/0, parser/1]).
 
 %% example usage of core api
--export([decode/1, decode/2]).
 -export([is_json/1, is_json/2]).
--export([fold/3, fold/4]).
+-export([decode/1, decode/2, decode/3, decode/4]).
+-export([fold/1, fold/2, fold/3, fold/4]).
 
 %% types for function specifications
 -include("jsx_types.hrl").
+
     
 
 -spec parser() -> jsx_parser().
@@ -58,36 +59,52 @@ start(F, OptsList) ->
     fun(Stream) -> F(Stream, Opts) end.
 
 
-%% decode is an example decoder using the jsx api. it converts the events into a simple
-%%   list and converts incomplete parses into errors. 
-    
--spec decode(JSON::json()) -> {ok, [jsx_event(),...]} | {error, badjson}.
--spec decode(JSON::json(), Opts::jsx_opts()) -> {ok, [jsx_event(),...]} | {error, badjson}.
-
-decode(JSON) ->
-    decode(JSON, []).
-
-decode(JSON, Opts) ->
-    fold(fun(end_json, State) -> 
-                lists:reverse(State) 
-            ;(Event, State) -> [Event] ++ State end, 
-        [], JSON, Opts).
-        
-
 -spec is_json(JSON::json()) -> true | false.
 -spec is_json(JSON::json(), Opts::jsx_opts()) -> true | false.
 
 is_json(JSON) ->
     is_json(JSON, []).
-    
+
 is_json(JSON, Opts) ->
     case fold(fun(end_json, ok) -> true ;(_, _) -> ok end, ok, JSON, Opts) of
         {incomplete, _} -> false
         ; {error, _} -> false
         ; {ok, true} -> true
     end.
+ 
 
+-spec decode(JSON::json()) -> {ok, [jsx_event(),...]} | {error, atom()}.
+-spec decode(JSON::json(), Parse::jsx_opts() | jsx_parser()) -> {ok, [jsx_event(),...]} | {error, atom()}. 
+-spec decode(F::fun((jsx_event(), any()) -> any()), 
+        Acc::any(), 
+        JSON::json()) -> 
+    {ok, any()} | {error, atom()}.
+-spec decode(F::fun((jsx_event(), any()) -> any()), 
+        Acc::any(), 
+        JSON::json(), 
+        Parse::jsx_opts() | jsx_parser()) -> 
+    {ok, any()} | {error, atom()}.
 
+decode(JSON) ->
+    decode(JSON, []).
+    
+decode(JSON, Parse) ->
+    F = fun(end_json, S) -> lists:reverse(S) ;(E, S) -> [E] ++ S end,
+    decode(F, [], JSON, Parse).
+
+decode(F, Acc, JSON) ->
+    decode(F, Acc, JSON, []).
+
+decode(F, Acc, JSON, Parse) ->
+    case fold(F, Acc, JSON, Parse) of
+        {ok, Result} -> {ok, Result}
+        ; _ -> {error, badjson}
+    end.
+        
+-spec fold(JSON::json()) -> 
+    {ok, [jsx_event(),...]} | {incomplete, jsx_parser(), fun(() -> jsx_parser_result())} | {error, atom()}.
+-spec fold(JSON::json(), Parse::jsx_opts() | jsx_parser()) -> 
+    {ok, [jsx_event(),...]} | {incomplete, jsx_parser(), fun(() -> jsx_parser_result())} | {error, atom()}.
 -spec fold(F::fun((jsx_event(), any()) -> any()), 
             Acc::any(), 
             JSON::json()) -> 
@@ -103,6 +120,13 @@ is_json(JSON, Opts) ->
             Parser::jsx_parser()) -> 
         {ok, any()} | {incomplete, jsx_parser(), fun(() -> jsx_parser_result())} | {error, atom()}.
     
+fold(JSON) ->
+    fold(JSON, []).
+
+fold(JSON, Parse) ->
+    F = fun(end_json, S) -> lists:reverse(S) ;(E, S) -> [E] ++ S end,
+    fold(F, [], JSON, Parse).
+        
 fold(F, Acc, JSON) ->
     P = jsx:parser(),
     fold(F, Acc, JSON, P).
@@ -200,31 +224,39 @@ detect_encoding(<<X, Y, _Rest/binary>> = JSON, Opts) when X =/= 0, Y =/= 0 ->
 %%   to conclusively determine the encoding correctly. below is an attempt to solve
 %%   the problem
 detect_encoding(<<X>>, Opts) when X =/= 0 ->
-    try jsx_utf8:parse(<<X>>, Opts)
+    {incomplete, 
-    catch error:function_clause ->
+        fun(Stream) -> detect_encoding(<<X, Stream/binary>>, Opts) end,
-        {incomplete, fun(Stream) ->
+        fun() -> try 
-            detect_encoding(<<X, Stream/binary>>, Opts)
+                {incomplete, _, Force} = jsx_utf8:parse(<<X>>, Opts),
-        end}
+                Force()
-    end;
+                catch error:function_clause -> {error, badjson} 
+            end 
+        end
+    };
 detect_encoding(<<0, X>>, Opts) when X =/= 0 ->
-    try jsx_utf16:parse(<<0, X>>, Opts)
+    {incomplete, 
-    catch error:function_clause -> 
+        fun(Stream) -> detect_encoding(<<0, X, Stream/binary>>, Opts) end,
-        {incomplete, fun(Stream) ->
+        fun() -> try 
-            detect_encoding(<<0, X, Stream/binary>>, Opts)
+                {incomplete, _, Force} = jsx_utf16:parse(<<0, X>>, Opts),
-        end}
+                Force()
-    end;
+                catch error:function_clause -> {error, badjson} 
+            end 
+        end
+    };
 detect_encoding(<<X, 0>>, Opts) when X =/= 0 ->
-    try jsx_utf16le:parse(<<X, 0>>, Opts)
+    {incomplete, 
-    catch error:function_clause ->
+        fun(Stream) -> detect_encoding(<<X, 0, Stream/binary>>, Opts) end,
-        {incomplete, fun(Stream) ->
+        fun() -> try 
-            detect_encoding(<<X, 0, Stream/binary>>, Opts)
+                {incomplete, _, Force} = jsx_utf16le:parse(<<X, 0>>, Opts),
-        end}
+                Force()
-    end;
+                catch error:function_clause -> {error, badjson} 
+            end 
+        end
+    };
     
 %% not enough input, request more
 detect_encoding(Bin, Opts) ->
     {incomplete, 
-        fun(Stream) -> 
+        fun(Stream) -> detect_encoding(<<Bin/binary, Stream/binary>>, Opts) end,
-            detect_encoding(<<Bin/binary, Stream/binary>>, Opts) 
+        fun() -> {error, badjson} end
-        end
     }.

src/jsx_decoder.erl

@@ -217,7 +217,7 @@ string(<<S/?encoding, Rest/binary>>, Stack, Opts, Acc) when ?is_noncontrol(S) ->
 string(Bin, Stack, Opts, Acc) ->
     case partial_utf(Bin) of
         false -> {error, badjson}
-        ; _ -> {incomplete, fun(Stream) -> string(<<Bin/binary, Stream/binary>>, Stack, Opts, Acc) end}
+        ; _ -> {incomplete, fun(Stream) -> string(<<Bin/binary, Stream/binary>>, Stack, Opts, Acc) end, ?ferror}
     end.
 
     
@@ -436,7 +436,7 @@ integer(<<?solidus/?encoding, Rest/binary>>, Stack, ?comments_enabled(Opts), Acc
     maybe_comment(Rest, fun(Resume) -> integer(Resume, Stack, Opts, Acc) end);
 integer(<<>>, [], Opts, Acc) ->
     {incomplete, 
-        fun(Stream) -> zero(Stream, [], Opts, Acc) end,
+        fun(Stream) -> integer(Stream, [], Opts, Acc) end,
         fun() -> {event, {integer, lists:reverse(Acc)}, fun() -> maybe_done(<<>>, [], Opts) end} end
     };
 integer(Bin, Stack, Opts, Acc) ->  

test/cases/naked_number_g.json

@@ -0,0 +1 @@
+7

test/cases/naked_number_g.test

@@ -0,0 +1 @@
+[{integer, "7"}].