jsx/README.markdown

jsx
===

basically [yajl][1], but in erlang. born from a need for a stream based, incremental parser capable of outputting a number of representations



overview
--------

jsx is a json parsing toolkit, to be used to convert json streams to arbitrary output representations. see:


an eep0018 (erlang extension proposal for json decoding) json decoder
    
    %% this is nearly complete, ignoring only the options json_to_term should accept
    
    %% [true,[{<<"key">>, <<"value">>}],1] = json_to_term(<<"[ true, { \"key\": \"value\" }, 1 ]">>).
    
    json_to_term(JSON) ->
        P = jsx:parser(),
        collect_start(P(JSON), [[]]).
    
	collect_start({event, Event, Next}, Acc) when Event =:= start_object; Event =:= start_array ->
	    collect(Next(), [[]|Acc]);
	collect_start(_, _) ->
	    erlang:error(badarg).
    
	collect({event, Event, Next}, Acc) when Event =:= start_object; Event =:= start_array ->
	    collect(Next(), [[]|Acc]);

	%% special case for empty object
	collect({event, end_object, Next}, [[], Parent|Rest]) ->
	    collect(Next(), [[[{}]] ++ Parent] ++ Rest);
	%% reverse the array/object accumulator before prepending it to it's parent
	collect({event, end_object, Next}, [Current, Parent|Rest]) when is_list(Parent) ->
	    collect(Next(), [[lists:reverse(Current)] ++ Parent] ++ Rest);
	collect({event, end_array, Next}, [Current, Parent|Rest]) when is_list(Parent) ->
	    collect(Next(), [[lists:reverse(Current)] ++ Parent] ++ Rest);
	collect({event, Start, Next}, [Current, Key, Parent|Rest])
	        when Start =:= end_object; Start =:= end_array ->
	    collect(Next(), [[{Key, lists:reverse(Current)}] ++ Parent] ++ Rest);
    
	%% end of json is emitted asap (at close of array/object), Next() should return {incomplete, More}
	%%   and then More(end_stream) ensures the tail of the json binary is clean (whitespace only)     
	collect({event, end_json, Next}, [[Acc]]) ->
	    case Next() of
	        {incomplete, More} -> case More(end_stream) of
	            ok -> Acc
	            ; _ -> erlang:error(badarg)
	        end
	        ; _ -> erlang:error(badarg)
	    end;
    
	%% key can only be emitted inside of a json object, so just insert it directly into
	%%   the head of the accumulator and deal with it when we receive it's paired value    
	collect({event, {key, _} = PreKey, Next}, [Current|_] = Acc) ->
	    Key = event(PreKey),
	    case key_repeats(Key, Current) of
	        true -> erlang:error(badarg)
	        ; false -> collect(Next(), [Key] ++ Acc)
	    end;
    
	%% check acc to see if we're inside an object or an array. because inside an object
	%%   context the events that fall this far are always preceded by a key (which are
	%%   binaries or atoms), if Current is a list, we're inside an array, else, an
	%%   object
	collect({event, Event, Next}, [Current|Rest]) when is_list(Current) ->
	    collect(Next(), [[event(Event)] ++ Current] ++ Rest);
	collect({event, Event, Next}, [Key, Current|Rest]) ->
	    collect(Next(), [[{Key, event(Event)}] ++ Current] ++ Rest);
    
	%% any other event is an error
	collect(_, _) -> erlang:error(badarg).
    
    
	event({string, String}) ->
	    unicode:characters_to_binary(String);
	event({key, Key}) ->
	    unicode:characters_to_binary(Key);
	%% special case for negative zero
	event({integer, "-0"}) ->
	    erlang:float(erlang:list_to_integer("-0"));
	event({integer, Integer}) ->
	    erlang:list_to_integer(Integer);
	event({float, Float}) ->
	    erlang:list_to_float(Float);
	event({literal, Literal}) ->
	    Literal.
    
    
	key_repeats(Key, [{Key, _Value}|_Rest]) -> true;
	key_repeats(Key, [_|Rest]) -> key_repeats(Key, Rest);
	key_repeats(_Key, []) -> false.



the twitter json API has a ton of detail in it's stream, most of which is ignorable. this parser extracts just the user screenname, the text and the time and throws the rest away
    
    %% {"talentdeficit", "use jsx!", "Fri Jul 30 04:25:50 +0000 2010"} = tweet_to_tuple(...).
    
    tweet_to_tuple(Tweet) ->
        P = jsx:parser(),
        scan(P(Tweet), {name, text, time}).
    
    scan({event, {key, "screen_name"}, Next}, Acc) ->
        collect_name(Next(), Acc);
    scan({event, {key, "created_at"}, Next}, Acc) ->
        collect_time(Next(), Acc);
    scan({event, {key, "text"}, Next}, Acc) ->
        collect_text(Next(), Acc);
    scan({event, end_json, Next}, Acc) ->
        Acc;
    scan({_, _, Next}, Acc) ->
        scan(Next(), Acc).
    
    collect_name({event, {string, Name}, Next}, {_, Text, Time}) ->
        scan(Next(), {Name, Text, Time}).
    collect_time({event, {string, Time}, Next}, {Name, Text, _}) ->
        scan(Next(), {Name, Text, Time}).
    collect_text({event, {string, Text}, Next}, {Name, _, Time}) ->
        scan(Next(), {Name, Text, Time}).



api
---

`jsx:parser/0` returns a function with arity 1. call it with a binary containing a properly encoded json stream and it returns one of the following values:

* `{event, Event, Next}`: Event is described below and Next is a zero arity function that returns the next value
* `{incomplete, More}`: More is described below
* `{error, badjson}`: the parser has decided the json stream is not a valid json document

`jsx:parser/1` has the same return signature as `jsx:parser/0` but accepts a proplist containing the following options:

* `{comments, true | false}`: determines whether (/\* c style \*/) comments are allowed. default is false
* `{escaped_unicode, ascii | codepoint | none}`: determines what escaped unicode sequences like `"\ua123"` are converted to in key/string events. ascii converts any sequences in the range 0-127 to their ascii value (`"\u0021"` would become 33), codepoint converts all valid sequences to their unicode codepoint value (`"\uabcd"` would become 43981), none does no conversion (`"\u0021"` would become the sequence 92, 117, 48, 48, 50, 49)
* `{encoding, utf8 | utf16 | {utf16, little} | utf32 | {utf32, little} | auto}`: determines which encoding to expect the binary to use. a contrary encoding will result in an error. auto will auto detect the encoding. auto is the default
* `{multi_term, true | false}`: normally, after the end of a json document only whitespace is allowed, passing this option with true alters parsing so after the end of a json document, another json document is permitted, with any amount of whitespace in between. default is false

More is a new parser returned when parsing encounters the end of the stream supplied to the parser. calling it with a new stream resumes parsing as if the stream was not interrupted. because json documents may be followed by arbitrary whitespace, there is no unambiguous ending to a json stream. Next will always eventually return `{incomplete, More}`. to ensure the stream is clean and contains no garbage in the tail, call `More(end_stream)`. if `ok` is returned, parsing is complete. otherwise `{error, badjson}` will be returned



events
------

a complete list of events. Next is described under *api* above:

* `{event, start_object | end_object | start_array | end_array, Next}`: emitted when `{`, `}`, `[`, `]` are encountered by the parser in a legal position
* `{event, end_json, Next}`: emitted when the json document has been completed (the root object/array has been closed). note that this does NOT ensure the json document is valid, the tail must be checked to be free of invalid characters as described above under *api*
* `{event, {key, Key}, Next}`: an object key has been encountered. Key has the same format as String below
* `{event, {string, String}, Next}`: a string has been encountered. String is a list of unicode codepoints
* `{event, {integer, Integer}, Next}`: an integer had been encountered. Integer is a list of unicode codepoints that can be passed to `erlang:list_to_integer/1` to convert to an integer
* `{event, {float, Float}, Next}`: a float has been encountered. Float is a list of unicode codepoints that can be passed to `erlang:list_to_float/1` to convert to a float
* `{event, {literal, true | false | null}, Next}`: a json literal has been encountered. it will be the atom `true`, the atom `false` or the atom `null`


notes
-----

to compile and install, run `make && make install` from the root of the project directory

jsx supports utf8, utf16 (little and big endian) and utf32 (little and big endian). future support is planned for erlang iolists







[1]: http://github.com/lloyd/yajl