Merge branch 'feature/inline_ops' into develop

This commit is contained in:
alisdair sullivan 2011-08-10 00:36:06 -07:00
commit 3fdf2b28c8
7 changed files with 645 additions and 625 deletions

View file

@ -27,7 +27,7 @@
%% the core parser api
-export([parser/0, parser/1]).
-export([decoder/0, decoder/1]).
-export([encoder/0]).
-export([encoder/0, encoder/1]).
-export([term_to_json/1, term_to_json/2]).
-export([json_to_term/1, json_to_term/2]).
-export([is_json/1, is_json/2]).
@ -37,11 +37,6 @@
-include("jsx_common.hrl").
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-endif.
-spec parser() -> jsx_decoder().
parser() -> decoder([]).
@ -52,7 +47,6 @@ parser() -> decoder([]).
parser(OptsList) -> decoder(OptsList).
-spec decoder() -> jsx_decoder().
decoder() -> decoder([]).
@ -77,7 +71,7 @@ decoder(OptsList) ->
encoder() -> encoder([]).
-spec encoder(OptsList::jsx_opts()) -> jsx_encoder().
-spec encoder(Opts::jsx_opts()) -> jsx_encoder().
encoder(Opts) -> jsx_encoder:encoder(Opts).
@ -146,6 +140,8 @@ format(JSON, Opts) ->
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
jsx_decoder_test_() ->
jsx_decoder_gen(load_tests(?eunit_test_path)).
@ -174,13 +170,19 @@ jsx_decoder_gen([Test|_] = Tests, [Encoding|Encodings]) ->
Flags = proplists:get_value(jsx_flags, Test, []),
{generator,
fun() ->
[{Name, ?_assertEqual(decode(JSON, Flags), JSX)}
| {generator,
[{Name ++ " iterative",
?_assertEqual(iterative_decode(JSON, Flags), JSX)}
| {generator,
fun() -> [{Name ++ " incremental", ?_assertEqual(
incremental_decode(JSON, Flags), JSX)
} | jsx_decoder_gen(Tests, Encodings)]
end
}
} | {generator,
fun() ->
[{Name, ?_assertEqual(
decode(JSON, Flags), JSX)
} | jsx_decoder_gen(Tests, Encodings)]
end}
]
end}
]
end
}.
@ -219,20 +221,33 @@ parse_tests([], _Dir, Acc) ->
decode(JSON, Flags) ->
P = jsx:decoder(Flags),
decode_loop(P(JSON), []).
case P(JSON) of
{error, {badjson, _}} -> {error, badjson}
; {jsx, incomplete, More} ->
case More(end_stream) of
{error, {badjson, _}} -> {error, badjson}
; {jsx, T, _} -> T
end
; {jsx, T, _} -> T
end.
decode_loop({jsx, end_json, _Next}, Acc) ->
iterative_decode(JSON, Flags) ->
P = jsx:decoder([iterate] ++ Flags),
iterative_decode_loop(P(JSON), []).
iterative_decode_loop({jsx, end_json, _Next}, Acc) ->
lists:reverse([end_json] ++ Acc);
decode_loop({jsx, incomplete, More}, Acc) ->
decode_loop(More(end_stream), Acc);
decode_loop({jsx, E, Next}, Acc) ->
decode_loop(Next(), [E] ++ Acc);
decode_loop({error, {badjson, _Error}}, _Acc) ->
iterative_decode_loop({jsx, incomplete, More}, Acc) ->
iterative_decode_loop(More(end_stream), Acc);
iterative_decode_loop({jsx, E, Next}, Acc) ->
iterative_decode_loop(Next(), [E] ++ Acc);
iterative_decode_loop({error, {badjson, _Error}}, _Acc) ->
{error, badjson}.
incremental_decode(<<C:1/binary, Rest/binary>>, Flags) ->
P = jsx:decoder(Flags),
P = jsx:decoder([iterate] ++ Flags),
incremental_decode_loop(P(C), Rest, []).
incremental_decode_loop({jsx, incomplete, Next}, <<>>, Acc) ->
@ -246,47 +261,5 @@ incremental_decode_loop({jsx, Event, Next}, Rest, Acc) ->
incremental_decode_loop({error, {badjson, _Error}}, _Rest, _Acc) ->
{error, badjson}.
multi_decode_test_() ->
[
{"multiple values in a single stream", ?_assert(
multi_decode(multi_json_body(), []) =:= multi_test_result()
)}
].
multi_decode(JSON, Flags) ->
P = jsx:decoder(Flags ++ [multi_term]),
multi_decode_loop(P(JSON), [[]]).
multi_decode_loop({jsx, incomplete, _Next}, [[]|Acc]) ->
lists:reverse(Acc);
multi_decode_loop({jsx, end_json, Next}, [S|Acc]) ->
multi_decode_loop(Next(), [[]|[lists:reverse(S)] ++ Acc]);
multi_decode_loop({jsx, E, Next}, [S|Acc]) ->
multi_decode_loop(Next(), [[E] ++ S] ++ Acc).
multi_json_body() ->
<<"0 1 -1 1e1 0.7 0.7e-1 truefalsenull {} {\"key\": \"value\"}[] [1, 2, 3]\"hope this works\"">>.
multi_test_result() ->
[[{integer, 0}],
[{integer, 1}],
[{integer, -1}],
[{float, 1.0e1}],
[{float, 0.7}],
[{float, 0.7e-1}],
[{literal, true}],
[{literal, false}],
[{literal, null}],
[start_object, end_object],
[start_object, {key, "key"}, {string, "value"}, end_object],
[start_array, end_array],
[start_array, {integer, 1}, {integer, 2}, {integer, 3}, end_array],
[{string, "hope this works"}]
].
-endif.

File diff suppressed because it is too large Load diff

View file

@ -23,116 +23,125 @@
-module(jsx_encoder).
-export([start/3,
list_or_object/4,
key/4,
value/4,
maybe_done/4,
bad_json/2
]).
-export([encoder/1]).
-include("jsx_common.hrl").
-include("jsx_opts.hrl").
-record(opts, {
multi_term = false,
encoding = auto,
loose_unicode = false, %% does nothing, used by decoder
escape_forward_slash = false
}).
-spec encoder(OptsList::jsx_opts()) -> jsx_encoder().
encoder(OptsList) ->
Opts = parse_opts(OptsList),
case Opts#opts.iterate of
true ->
fun(Forms) -> start(Forms, iterate, Opts) end
; false ->
fun(Forms) -> start(Forms, [], Opts) end
end.
-spec encoder(Opts::jsx_opts()) -> jsx_encoder().
encoder(Opts) -> fun(Forms) -> start(Forms, Opts) end.
-define(ENDJSON,
{jsx, end_json, fun() ->
{jsx, incomplete, fun(Forms) -> {error, {badjson, Forms}} end}
end}
).
start({string, String}, Opts) when is_binary(String) ->
{jsx, {string, json_escape(String, Opts)}, fun() -> ?ENDJSON end};
start({float, Float}, _Opts) when is_float(Float) ->
{jsx, {float, Float}, fun() -> ?ENDJSON end};
start({integer, Int}, _Opts) when is_integer(Int) ->
{jsx, {integer, Int}, fun() -> ?ENDJSON end};
start({literal, Atom}, _Opts) when Atom == true; Atom == false; Atom == null ->
{jsx, {literal, Atom}, fun() -> ?ENDJSON end};
%% second parameter is a stack to match end_foos to start_foos
start(Forms, Opts) -> list_or_object(Forms, [], Opts).
list_or_object([start_object|Forms], Stack, Opts) ->
{jsx, start_object, fun() -> key(Forms, [object] ++ Stack, Opts) end};
list_or_object([start_array|Forms], Stack, Opts) ->
{jsx, start_array, fun() -> value(Forms, [array] ++ Stack, Opts) end};
list_or_object([], Stack, Opts) ->
{jsx, incomplete, fun(end_stream) ->
%% emit takes a list of `events` to present to client code and formats them
%% appropriately
emit([], {State, Rest, T, Args}) ->
erlang:apply(?MODULE, State, [Rest, T] ++ Args);
emit([incomplete], {State, Rest, T, Args}) ->
{jsx, incomplete, fun(end_stream) ->
{error, {badjson, []}}
; (Stream) ->
list_or_object(Stream, Stack, Opts)
; (Stream) ->
erlang:apply(?MODULE,
State,
[Rest ++ Stream, T] ++ Args
)
end};
list_or_object(Forms, _, _) -> {error, {badjson, Forms}}.
emit([Event|Events], {_State, _Rest, iterate, _Args} = Next) ->
{jsx, Event, fun() -> emit(Events, Next) end};
emit([end_json|Events], {_State, _Rest, T, _Args} = Next) ->
{jsx, lists:reverse([end_json] ++ T), fun() -> emit(Events, Next) end};
emit([Event|Events], {State, Rest, T, Args}) ->
emit(Events, {State, Rest, [Event] ++ T, Args}).
bad_json(Stream, _) -> {error, {badjson, Stream}}.
start({string, String}, T, Opts) when is_binary(String) ->
emit([{string, json_escape(String, Opts)}, end_json, incomplete],
{bad_json, [], T, []}
);
start({float, Float}, T, _Opts) when is_float(Float) ->
emit([{float, Float}, end_json, incomplete], {bad_json, [], T, []});
start({integer, Int}, T, _Opts) when is_integer(Int) ->
emit([{integer, Int}, end_json, incomplete], {bad_json, [], T, []});
start({literal, Atom}, T, _Opts) when Atom == true; Atom == false; Atom == null ->
emit([{literal, Atom}, end_json, incomplete], {bad_json, [], T, []});
%% third parameter is a stack to match end_foos to start_foos
start(Forms, T, Opts) -> list_or_object(Forms, T, [], Opts).
list_or_object([start_object|Forms], T, Stack, Opts) ->
emit([start_object], {key, Forms, T, [[object] ++ Stack, Opts]});
list_or_object([start_array|Forms], T, Stack, Opts) ->
emit([start_array], {value, Forms, T, [[array] ++ Stack, Opts]});
list_or_object([], T, Stack, Opts) ->
emit([incomplete], {list_or_object, [], T, [Stack, Opts]});
list_or_object(Forms, _, _, _) -> {error, {badjson, Forms}}.
key([{key, Key}|Forms], Stack, Opts) when is_binary(Key) ->
{jsx, {key, json_escape(Key, Opts)}, fun() ->
value(Forms, Stack, Opts)
end};
key([end_object|Forms], [object|Stack], Opts) ->
{jsx, end_object, fun() -> maybe_done(Forms, Stack, Opts) end};
key([], Stack, Opts) ->
{jsx, incomplete, fun(end_stream) ->
{error, {badjson, []}}
; (Stream) ->
key(Stream, Stack, Opts)
end};
key(Forms, _, _) -> {error, {badjson, Forms}}.
key([{key, Key}|Forms], T, Stack, Opts) when is_binary(Key) ->
emit([{key, json_escape(Key, Opts)}], {value, Forms, T, [Stack, Opts]});
key([end_object|Forms], T, [object|Stack], Opts) ->
emit([end_object], {maybe_done, Forms, T, [Stack, Opts]});
key([], T, Stack, Opts) ->
emit([incomplete], {key, [], T, [Stack, Opts]});
key(Forms, _, _, _) -> {error, {badjson, Forms}}.
value([{string, S}|Forms], Stack, Opts) when is_binary(S) ->
{jsx, {string, json_escape(S, Opts)}, fun() ->
maybe_done(Forms, Stack, Opts)
end};
value([{float, F}|Forms], Stack, Opts) when is_float(F) ->
{jsx, {float, F}, fun() -> maybe_done(Forms, Stack, Opts) end};
value([{integer, I}|Forms], Stack, Opts) when is_integer(I) ->
{jsx, {integer, I}, fun() -> maybe_done(Forms, Stack, Opts) end};
value([{literal, L}|Forms], Stack, Opts)
value([{string, S}|Forms], T, Stack, Opts) when is_binary(S) ->
emit([{string, json_escape(S, Opts)}],
{maybe_done, Forms, T, [Stack, Opts]}
);
value([{float, F}|Forms], T, Stack, Opts) when is_float(F) ->
emit([{float, F}], {maybe_done, Forms, T, [Stack, Opts]});
value([{integer, I}|Forms], T, Stack, Opts) when is_integer(I) ->
emit([{integer, I}], {maybe_done, Forms, T, [Stack, Opts]});
value([{literal, L}|Forms], T, Stack, Opts)
when L == true; L == false; L == null ->
{jsx, {literal, L}, fun() -> maybe_done(Forms, Stack, Opts) end};
value([start_object|Forms], Stack, Opts) ->
{jsx, start_object, fun() -> key(Forms, [object] ++ Stack, Opts) end};
value([start_array|Forms], Stack, Opts) ->
{jsx, start_array, fun() -> value(Forms, [array] ++ Stack, Opts) end};
value([end_array|Forms], [array|Stack], Opts) ->
{jsx, end_array, fun() -> maybe_done(Forms, Stack, Opts) end};
value([], Stack, Opts) ->
{jsx, incomplete, fun(end_stream) ->
{error, {badjson, []}}
; (Stream) ->
value(Stream, Stack, Opts)
end};
value(Forms, _, _) -> {error, {badjson, Forms}}.
emit([{literal, L}], {maybe_done, Forms, T, [Stack, Opts]});
value([start_object|Forms], T, Stack, Opts) ->
emit([start_object], {key, Forms, T, [[object] ++ Stack, Opts]});
value([start_array|Forms], T, Stack, Opts) ->
emit([start_array], {value, Forms, T, [[array] ++ Stack, Opts]});
value([end_array|Forms], T, [array|Stack], Opts) ->
emit([end_array], {maybe_done, Forms, T, [Stack, Opts]});
value([], T, Stack, Opts) ->
emit([incomplete], {value, [], T, [Stack, Opts]});
value(Forms, _, _, _) -> {error, {badjson, Forms}}.
maybe_done([], [], _) -> ?ENDJSON;
maybe_done([end_json], [], _) -> ?ENDJSON;
maybe_done([end_json|Forms], [], #opts{multi_term=true}=Opts) ->
{jsx, end_json, fun() -> start(Forms, Opts) end};
maybe_done([end_object|Forms], [object|Stack], Opts) ->
{jsx, end_object, fun() -> maybe_done(Forms, Stack, Opts) end};
maybe_done([end_array|Forms], [array|Stack], Opts) ->
{jsx, end_array, fun() -> maybe_done(Forms, Stack, Opts) end};
maybe_done(Forms, [object|_] = Stack, Opts) -> key(Forms, Stack, Opts);
maybe_done(Forms, [array|_] = Stack, Opts) -> value(Forms, Stack, Opts);
maybe_done([], Stack, Opts) ->
{jsx, incomplete, fun(end_stream) ->
{error, {badjson, []}}
; (Stream) ->
maybe_done(Stream, Stack, Opts)
end};
maybe_done(Forms, _, _) -> {error, {badjson, Forms}}.
maybe_done([], T, [], _Opts) ->
emit([end_json, incomplete], {bad_json, [], T, []});
maybe_done([end_json], T, [], _Opts) ->
emit([end_json, incomplete], {bad_json, [], T, []});
maybe_done([end_object|Forms], T, [object|Stack], Opts) ->
emit([end_object], {maybe_done, Forms, T, [Stack, Opts]});
maybe_done([end_array|Forms], T, [array|Stack], Opts) ->
emit([end_array], {maybe_done, Forms, T, [Stack, Opts]});
maybe_done(Forms, T, [object|_] = Stack, Opts) -> key(Forms, T, Stack, Opts);
maybe_done(Forms, T, [array|_] = Stack, Opts) -> value(Forms, T, Stack, Opts);
maybe_done([], T, Stack, Opts) ->
emit([incomplete], {maybe_done, [], T, [Stack, Opts]});
maybe_done(Forms, _, _, _) -> {error, {badjson, Forms}}.
@ -199,58 +208,49 @@ to_hex(X) -> X + $0.
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
encode(Terms) -> encode_whole(Terms) andalso encode_incremental(Terms).
encode(Terms) ->
encode_simple(Terms) andalso encode_iterative(Terms).
encode_whole(Terms) ->
case loop((encoder([]))(Terms), []) of
%% unwrap naked values
{ok, [Terms]} -> true
; {ok, Terms} -> true
; _ -> false
end.
encode_incremental(Terms) when is_list(Terms) ->
encode_incremental(Terms, encoder([]), Terms, []);
%% we could feed naked terms to the regular encoder, but we already do that, so
%% cheat instead
encode_incremental(_) -> true.
encode_incremental([Term], F, Expected, Acc) ->
case loop(F([Term]), []) of
{ok, R} -> Expected =:= Acc ++ R
; _ -> false
end;
encode_incremental([Term|Terms], F, Expected, Acc) ->
case loop(F([Term]), []) of
{jsx, incomplete, Next, R} ->
encode_incremental(Terms, Next, Expected, Acc ++ R)
; _ ->
encode_simple(Terms) ->
case (encoder([]))(Terms) of
{jsx, Terms, _} ->
true
%% matches [foo, end_json], aka naked terms
; {jsx, [Terms, end_json], _} ->
true
; {error, _} ->
false
end.
loop({error, _}, _) -> error;
loop({jsx, incomplete, Next}, Acc) ->
{jsx, incomplete, Next, lists:reverse(Acc)};
encode_iterative(Terms) ->
case loop((encoder([iterate]))(Terms), []) of
{ok, Terms} ->
true
%% matches naked terms
; {ok, [Terms, end_json]} ->
true
; {error, _} ->
false
end.
loop({jsx, end_json, Next}, Acc) ->
{jsx, incomplete, F} = Next(),
{error, {badjson, []}} = F([]),
{ok, lists:reverse(Acc)};
loop({jsx, Event, Next}, Acc) -> loop(Next(), [Event] ++ Acc).
{error, _} = F([]),
{ok, lists:reverse([end_json] ++ Acc)};
loop({jsx, Event, Next}, Acc) ->
loop(Next(), [Event] ++ Acc).
encode_test_() ->
[
{"empty object", ?_assert(encode([start_object, end_object]))},
{"empty array", ?_assert(encode([start_array, end_array]) =:= true)},
{"empty object", ?_assert(encode([start_object, end_object, end_json]))},
{"empty array", ?_assert(encode([start_array, end_array, end_json]))},
{"nested empty objects", ?_assert(encode([start_object,
{key, <<"empty object">>},
start_object,
@ -258,14 +258,16 @@ encode_test_() ->
start_object,
end_object,
end_object,
end_object
end_object,
end_json
]))},
{"nested empty arrays", ?_assert(encode([start_array,
start_array,
start_array,
end_array,
end_array,
end_array
end_array,
end_json
]))},
{"simple object", ?_assert(encode([start_object,
{key, <<"a">>},
@ -276,18 +278,21 @@ encode_test_() ->
{float, 1.0},
{key, <<"d">>},
{literal, true},
end_object
end_object,
end_json
]))},
{"simple array", ?_assert(encode([start_array,
{string, <<"hello">>},
{integer, 1},
{float, 1.0},
{literal, true},
end_array
end_array,
end_json
]))},
{"unbalanced array", ?_assertNot(encode([start_array,
end_array,
end_array
end_array,
end_json
]))},
{"naked string", ?_assert(encode({string, <<"hello">>}))},
{"naked literal", ?_assert(encode({literal, true}))},
@ -324,5 +329,4 @@ escape_test_() ->
}
].
-endif.
-endif.

View file

@ -41,10 +41,10 @@
binary() | iolist().
format(JSON, OptsList) when is_binary(JSON) ->
P = jsx:decoder(extract_parser_opts(OptsList)),
P = jsx:decoder([iterate] ++ extract_parser_opts(OptsList)),
format(fun() -> P(JSON) end, OptsList);
format(Terms, OptsList) when is_list(Terms); is_tuple(Terms) ->
P = jsx:encoder(),
P = jsx:encoder([iterate]),
format(fun() -> P(Terms) end, OptsList);
format(F, OptsList) when is_function(F) ->
Opts = parse_opts(OptsList, #format_opts{}),

48
src/jsx_opts.hrl Normal file
View file

@ -0,0 +1,48 @@
%% The MIT License
%% Copyright (c) 2010 Alisdair Sullivan <alisdairsullivan@yahoo.ca>
%% Permission is hereby granted, free of charge, to any person obtaining a copy
%% of this software and associated documentation files (the "Software"), to deal
%% in the Software without restriction, including without limitation the rights
%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
%% copies of the Software, and to permit persons to whom the Software is
%% furnished to do so, subject to the following conditions:
%% The above copyright notice and this permission notice shall be included in
%% all copies or substantial portions of the Software.
%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
%% THE SOFTWARE.
%% opts record for decoder/encoder
-record(opts, {
loose_unicode = false,
encoding = auto,
escape_forward_slash = false, %% does nothing, used by encoder
iterate = false
}).
parse_opts(Opts) ->
parse_opts(Opts, #opts{}).
parse_opts([], Opts) ->
Opts;
parse_opts([loose_unicode|Rest], Opts) ->
parse_opts(Rest, Opts#opts{loose_unicode=true});
parse_opts([iterate|Rest], Opts) ->
parse_opts(Rest, Opts#opts{iterate=true});
parse_opts([escape_forward_slash|Rest], Opts) ->
parse_opts(Rest, Opts#opts{escape_forward_slash=true});
parse_opts([{encoding, _}|Rest], Opts) ->
parse_opts(Rest, Opts);
parse_opts(_, _) ->
{error, badarg}.

View file

@ -39,7 +39,7 @@
jsx_term() | {jsx, incomplete, fun()}.
json_to_term(JSON, Opts) ->
P = jsx:decoder(extract_parser_opts(Opts)),
P = jsx:decoder([iterate] ++ extract_parser_opts(Opts)),
case proplists:get_value(strict, Opts, false) of
true -> collect_strict(P(JSON), [[]], Opts)
; false -> collect(P(JSON), [[]], Opts)

View file

@ -38,10 +38,10 @@
; (F::jsx_iterator(), Opts::verify_opts()) -> true | false.
is_json(JSON, OptsList) when is_binary(JSON) ->
P = jsx:decoder(extract_parser_opts(OptsList)),
P = jsx:decoder([iterate] ++ extract_parser_opts(OptsList)),
is_json(fun() -> P(JSON) end, OptsList);
is_json(Terms, OptsList) when is_list(Terms) ->
P = jsx:encoder(),
P = jsx:encoder([iterate]),
is_json(fun() -> P(Terms) end, OptsList);
is_json(F, OptsList) when is_function(F) ->
Opts = parse_opts(OptsList, #verify_opts{}),