converted jsx_test to etap, fixed resulting bugs with autodetection

src/jsx.erl

@@ -99,9 +99,6 @@ detect_encoding(<<16#ff, 16#fe, Rest/binary>>, Stack, Callbacks, Opts) ->
 detect_encoding(<<0, 0, 16#fe, 16#ff, Rest/binary>>, Stack, Callbacks, Opts) ->
     jsx_utf32:start(Rest, Stack, Callbacks, Opts);
     
-%% utf8 null order detection
-detect_encoding(<<X, Y, _Rest/binary>> = JSON, Stack, Callbacks, Opts) when X =/= 0, Y =/= 0 ->
-    jsx_utf8:start(JSON, Stack, Callbacks, Opts);
 
 %% utf32-little null order detection
 detect_encoding(<<X, 0, 0, 0, _Rest/binary>> = JSON, Stack, Callbacks, Opts) when X =/= 0 ->
@@ -119,17 +116,38 @@ detect_encoding(<<X, 0, Y, 0, _Rest/binary>> = JSON, Stack, Callbacks, Opts) whe
 detect_encoding(<<0, 0, 0, X, _Rest/binary>> = JSON, Stack, Callbacks, Opts) when X =/= 0 ->
     jsx_utf32:start(JSON, Stack, Callbacks, Opts);
     
-%% trying to parse a json string of a single character encoded in utf8 will fail
+%% utf8 null order detection
-%%   unless special cased
+detect_encoding(<<X, Y, _Rest/binary>> = JSON, Stack, Callbacks, Opts) when X =/= 0, Y =/= 0 ->
-detect_encoding(<<X>> = JSON, Stack, Callbacks, Opts) when X =/= 0 ->
+    jsx_utf8:start(JSON, Stack, Callbacks, Opts);
-    try jsx_utf8:start(JSON, Stack, Callbacks, Opts)
+    
-    catch error:function_clause ->
+%% a problem, to autodetect naked single digits' encoding, there is not enough data
-        {incomplete, 
+%%   to conclusively determine the encoding correctly. below is an attempt to solve
-            fun(Stream) -> 
+%%   the problem
-                detect_encoding(<<X, Stream/binary>>, Stack, Callbacks, Opts) 
+
-            end
+detect_encoding(<<X>>, Stack, Callbacks, Opts) when X =/= 0 ->
-        }
+    {
-    end;
+        try {Result, _} = jsx_utf8:start(<<X>>, [], Callbacks, Opts), Result 
+        catch error:function_clause -> incomplete end,
+        fun(Stream) ->
+            detect_encoding(<<X, Stream/binary>>, Stack, Callbacks, Opts)
+        end
+    };
+detect_encoding(<<0, X>>, Stack, Callbacks, Opts) when X =/= 0 ->
+    {
+        try {Result, _} = jsx_utf16:start(<<0, X>>, [], Callbacks, Opts), Result 
+        catch error:function_clause -> incomplete end,
+        fun(Stream) ->
+            detect_encoding(<<0, X, Stream/binary>>, Stack, Callbacks, Opts)
+        end
+    };
+detect_encoding(<<X, 0>>, Stack, Callbacks, Opts) when X =/= 0 ->
+    {
+        try {Result, _} = jsx_utf16le:start(<<X, 0>>, [], Callbacks, Opts), Result 
+        catch error:function_clause -> incomplete end,
+        fun(Stream) ->
+            detect_encoding(<<X, 0, Stream/binary>>, Stack, Callbacks, Opts)
+        end
+    };
     
 %% not enough input, request more
 detect_encoding(Bin, Stack, Callbacks, Opts) ->

src/jsx_utf16.erl

@@ -371,7 +371,7 @@ initial_decimal(<<S/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) when
 initial_decimal(<<?zero/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) ->
     decimal(Rest, Stack, Callbacks, Opts, [?zero] ++ Acc);
 initial_decimal(Bin, Stack, Callbacks, Opts, Acc) when byte_size(Bin) < 2 ->
-    {incomplete, fun(Stream) -> initial_decimal(Stream, Stack, Callbacks, Opts, Acc) end}.
+    {incomplete, fun(Stream) -> initial_decimal(<<Bin/binary, Stream/binary>>, Stack, Callbacks, Opts, Acc) end}.
 
 
 decimal(<<S/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) when ?is_nonzero(S) ->

src/jsx_utf16le.erl

@@ -371,7 +371,7 @@ initial_decimal(<<S/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) when
 initial_decimal(<<?zero/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) ->
     decimal(Rest, Stack, Callbacks, Opts, [?zero] ++ Acc);
 initial_decimal(Bin, Stack, Callbacks, Opts, Acc) when byte_size(Bin) < 2 ->
-    {incomplete, fun(Stream) -> initial_decimal(Stream, Stack, Callbacks, Opts, Acc) end}.
+    {incomplete, fun(Stream) -> initial_decimal(<<Bin/binary, Stream/binary>>, Stack, Callbacks, Opts, Acc) end}.
 
 
 decimal(<<S/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) when ?is_nonzero(S) ->

src/jsx_utf32.erl

@@ -358,7 +358,7 @@ initial_decimal(<<S/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) when
 initial_decimal(<<?zero/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) ->
     decimal(Rest, Stack, Callbacks, Opts, [?zero] ++ Acc);
 initial_decimal(Bin, Stack, Callbacks, Opts, Acc) when byte_size(Bin) < 4 ->
-    {incomplete, fun(Stream) -> initial_decimal(Stream, Stack, Callbacks, Opts, Acc) end}.
+    {incomplete, fun(Stream) -> initial_decimal(<<Bin/binary, Stream/binary>>, Stack, Callbacks, Opts, Acc) end}.
 
 
 decimal(<<S/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) when ?is_nonzero(S) ->

src/jsx_utf32le.erl

@@ -358,7 +358,7 @@ initial_decimal(<<S/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) when
 initial_decimal(<<?zero/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) ->
     decimal(Rest, Stack, Callbacks, Opts, [?zero] ++ Acc);
 initial_decimal(Bin, Stack, Callbacks, Opts, Acc) when byte_size(Bin) < 4 ->
-    {incomplete, fun(Stream) -> initial_decimal(Stream, Stack, Callbacks, Opts, Acc) end}.
+    {incomplete, fun(Stream) -> initial_decimal(<<Bin/binary, Stream/binary>>, Stack, Callbacks, Opts, Acc) end}.
 
 
 decimal(<<S/?encoding, Rest/binary>>, Stack, Callbacks, Opts, Acc) when ?is_nonzero(S) ->

test/jsx_test.erl

@@ -24,14 +24,18 @@
 -module(jsx_test).
 -author("alisdairsullivan@yahoo.ca").
 
--export([test/1, test_event/2, incremental_decode/2, decode/2]).
+-export([test/0, test/1, test_event/2]).
-
--include_lib("eunit/include/eunit.hrl").
 
+test() ->
+    F = decoder([]),
+    incremental_decode(F, unicode:characters_to_binary(<<"0">>, utf8, utf16)).
 
 test(Dir) ->
-    Tests = gen_tests(Dir),
+    ValidJSONTests = load_tests(Dir),
-    eunit:test(Tests, [verbose]).
+    
+    etap:plan(length(ValidJSONTests) * 10),
+    run_tests(ValidJSONTests),
+    etap:end_tests().
 
 
 decoder(Flags) ->
@@ -43,33 +47,40 @@ test_event(Event, Acc) ->
     [Event] ++ Acc.
 
 
-
+load_tests(Dir) ->
-gen_tests(Dir) ->
     TestSpecs = filelib:wildcard("*.test", Dir),
-    gen_tests(TestSpecs, Dir, []).
+    load_tests(TestSpecs, Dir, []).
     
-gen_tests([], _, Acc) ->
+load_tests([], _Dir, Acc) ->
-    lists:reverse(Acc); 
+    lists:reverse(Acc);
-    
+load_tests([Test|Rest], Dir, Acc) ->
-gen_tests([Test|Rest], Dir, Acc) ->
-    gen_tests(Rest, Dir, test_body(Test, Dir) ++ Acc).
-    
-test_body(TestSpec, Dir) ->
     try
-        TestName = filename:basename(TestSpec, ".test"),
+        TestName = filename:basename(Test, ".test"),
         {ok, JSON} = file:read_file(Dir ++ "/" ++ TestName ++ ".json"),
-        case file:consult(Dir ++ "/" ++ TestSpec) of
+        case file:consult(Dir ++ "/" ++ Test) of
             {ok, [Events]} ->
-                Decoder = jsx:decoder(),
+                load_tests(Rest, Dir, [{TestName, JSON, Events, []}] ++ Acc)
-                [{TestName ++ "_incremental", ?_assertEqual(incremental_decode(Decoder, JSON), Events)}] ++
-                [{TestName, ?_assertEqual(decode(Decoder, JSON), Events)}]
             ; {ok, [Events, Flags]} ->
-                Decoder = jsx:decoder(Flags),
+                load_tests(Rest, Dir, [{TestName, JSON, Events, Flags}] ++ Acc)
-                [{TestName ++ "_incremental", ?_assertEqual(incremental_decode(Decoder, JSON), Events)}] ++
-                [{TestName, ?_assertEqual(decode(Decoder, JSON), Events)}]
         end
-    catch _:_ -> []
+    catch _:_ -> load_tests(Rest, Dir, Acc) end.
-    end.
+    
+run_tests([]) ->
+    ok;
+run_tests([{TestName, JSON, Events, Flags}|Rest]) ->
+    F = decoder(Flags),
+    etap:is(decode(F, JSON), Events, TestName ++ ": utf8"),
+    etap:is(incremental_decode(F, JSON), Events, TestName ++ ": incremental utf8"),
+    etap:is(decode(F, to_utf16(JSON)), Events, TestName ++ ": utf16"),
+    etap:is(incremental_decode(F, to_utf16(JSON)), Events, TestName ++ ": incremental utf16"),
+    etap:is(decode(F, to_utf16le(JSON)), Events, TestName ++ ": utf16le"),
+    etap:is(incremental_decode(F, to_utf16le(JSON)), Events, TestName ++ ": incremental utf16le"),
+    etap:is(decode(F, to_utf32(JSON)), Events, TestName ++ ": utf32"),
+    etap:is(incremental_decode(F, to_utf32(JSON)), Events, TestName ++ ": incremental utf32"),
+    etap:is(decode(F, to_utf32le(JSON)), Events, TestName ++ ": utf32le"),
+    etap:is(incremental_decode(F, to_utf32le(JSON)), Events, TestName ++ ": incremental utf32le"),
+    run_tests(Rest).
+    
     
 incremental_decode(F, <<>>) ->
     case F(<<>>) of
@@ -87,4 +98,10 @@ decode(F, JSON) ->
         ; {Result, _} ->
             Result
     end.
+    
+to_utf16(Bin) -> unicode:characters_to_binary(Bin, utf8, utf16).
+to_utf16le(Bin) -> unicode:characters_to_binary(Bin, utf8, {utf16,little}).
+to_utf32(Bin) -> unicode:characters_to_binary(Bin, utf8, utf32).
+to_utf32le(Bin) -> unicode:characters_to_binary(Bin, utf8, {utf32,little}).
+