diff --git a/benchmark/poison.patch b/benchmark/poison.patch index c54ccf0..52e98c3 100644 --- a/benchmark/poison.patch +++ b/benchmark/poison.patch @@ -63,9 +63,10 @@ index c07cf0d..8426e22 100644 {:jason, "~> 1.1", only: [:test, :bench]}, {:exjsx, "~> 4.0", only: :bench}, {:tiny, "~> 1.0", only: :bench}, - {:jsone, "~> 1.4", only: :bench}, +- {:jsone, "~> 1.4", only: :bench}, - {:jiffy, "~> 0.15", only: :bench}, - {:json, "~> 1.2", only: :bench} ++ {:jsone, path: "../..", only: :bench}, + {:jiffy, "~> 1.0", only: :bench}, + {:json, "~> 1.4", only: :bench} ] diff --git a/doc/jsone.md b/doc/jsone.md index 1fefd61..f787f9c 100644 --- a/doc/jsone.md +++ b/doc/jsone.md @@ -120,7 +120,7 @@ the last such instance. 

-encode_option() = native_utf8 | native_forward_slash | canonical_form | {float_format, [float_format_option()]} | {datetime_format, datetime_encode_format()} | {object_key_type, string | scalar | value} | {space, non_neg_integer()} | {indent, non_neg_integer()} | {map_unknown_value, fun((term()) -> {ok, json_value()} | error)} | common_option()
+encode_option() = native_utf8 | native_forward_slash | canonical_form | {float_format, [float_format_option()]} | {datetime_format, datetime_encode_format()} | {object_key_type, string | scalar | value} | {space, non_neg_integer()} | {indent, non_neg_integer()} | {map_unknown_value, undefined | fun((term()) -> {ok, json_value()} | error)} | skip_undefined | common_option()
`native_utf8`:
@@ -156,8 +156,13 @@ encode_option() = native_utf8 | native_forward_slash | canonical_form | {float_f - Inserts a newline and `N` spaces for each level of indentation
- default: `0`
+`skip_undefined`:
+- If specified, each entry having `undefined` value in a object isn't included in the result JSON
+ `{map_unknown_value, Fun}`:
-- If specified, unknown values encountered during an encoding process are converted to `json_value()` by applying `Fun`. +- If `Fun` is a function, unknown values encountered during an encoding process are converted to `json_value()` by applying `Fun`.
+- If `Fun` is `undefined`, the encoding results in an error if there are unknown values.
+- default: `term_to_json_string/1`
@@ -175,7 +180,7 @@ float_format_option() = {scientific, Decimals::0..249} | {decimals, Decimals::0. - The encoded string will contain at most `Decimals` number of digits past the decimal point.
- If `compact` is provided the trailing zeros at the end of the string are truncated.
-For more details, see [erlang:float_to_list/2](http://erlang.org/doc/man/erlang.html#float_to_list-2). +For more details, see [erlang:float_to_list/2](http://erlang.org/doc/man/erlang.md#float_to_list-2). ``` > jsone:encode(1.23). @@ -192,6 +197,46 @@ For more details, see [erlang:float_to_list/2](http://erlang.org/doc/man/erlang. +### incomplete() ### + + +

+incomplete() = {incomplete, incomplete_fun()}
+
+ + + + +### incomplete_fun() ### + + +

+incomplete_fun() = fun((binary() | end_stream) -> incomplete() | json_value())
+
+ + + + +### incomplete_try() ### + + +

+incomplete_try() = {incomplete, incomplete_try_fun()}
+
+ + + + +### incomplete_try_fun() ### + + +

+incomplete_try_fun() = fun((binary() | end_stream) -> incomplete_try() | {ok, json_value(), Remainings::binary()} | {error, {Reason::term(), [stack_item()]}})
+
+ + + + ### json_array() ### @@ -236,7 +281,7 @@ json_object() = json_object_format_tupl 

-json_object_format_map() = #{}
+json_object_format_map() = map()
@@ -392,7 +437,7 @@ utc_offset_seconds() = -86399..86399 ## Function Index ## -
decode/1Equivalent to decode(Json, []).
decode/2Decodes an erlang term from json text (a utf8 encoded binary).
encode/1Equivalent to encode(JsonValue, []).
encode/2Encodes an erlang term into json text (a utf8 encoded binary).
try_decode/1Equivalent to try_decode(Json, []).
try_decode/2Decodes an erlang term from json text (a utf8 encoded binary).
try_encode/1Equivalent to try_encode(JsonValue, []).
try_encode/2Encodes an erlang term into json text (a utf8 encoded binary).
+
decode/1Equivalent to decode(Json, []).
decode/2Decodes an erlang term from json text (a utf8 encoded binary).
decode_stream/1Equivalent to decode_stream(Json, []).
decode_stream/2Decodes an Erlang from JSON text in chunks.
encode/1Equivalent to encode(JsonValue, []).
encode/2Encodes an erlang term into json text (a utf8 encoded binary).
ip_address_to_json_string/1Convert an IP address into a text representation.
term_to_json_string/1Converts the given term X to its string representation (i.e., the result of io_lib:format("~p", [X])).
try_decode/1Equivalent to try_decode(Json, []).
try_decode/2Decodes an erlang term from json text (a utf8 encoded binary).
try_decode_stream/1Equivalent to try_decode_stream(Json, []).
try_decode_stream/2Decodes an Erlang from JSON text in chunks.
try_encode/1Equivalent to try_encode(JsonValue, []).
try_encode/2Encodes an erlang term into json text (a utf8 encoded binary).
 @@ -433,6 +478,43 @@ Raises an error exception if input is not valid json in call from jsone:decode/1 (src/jsone.erl, line 71) ``` + + +### decode_stream/1 ### + +

+decode_stream(Json::binary()) -> incomplete()
+
+
+ +Equivalent to [`decode_stream(Json, [])`](#decode_stream-2). + + + +### decode_stream/2 ### + +

+decode_stream(Json::binary(), Options::[decode_option()]) -> incomplete()
+
+
+ +Decodes an Erlang from JSON text in chunks. + +Instead of returning a result, `{incomplete, fun()}` is returned. The +returned fun takes a single argument and it should called to continue the +decoding. When all the input has been provided, the fun should be called with +`end_stream` to signal the end of input and then the fun returns a result or +an error, as `decode/2` would do. + +``` + 1> {incomplete, F1} = jsone:decode(<<"[1,2,">>, []). + {incomplete, #Fun} + 2> {incomplete, F2} = F1(<<"3]>>). + {incomplete, #Fun} + 3> F2(end_stream). + [1,2,3] +``` + ### encode/1 ### @@ -467,6 +549,47 @@ Raises an error exception if input is not an instance of type `json_value()` in call from jsone:encode/1 (src/jsone.erl, line 97) ``` + + +### ip_address_to_json_string/1 ### + +

+ip_address_to_json_string(X::inet:ip_address() | any()) -> {ok, json_string()} | error
+
+
+ +Convert an IP address into a text representation. + +This function can be specified as the value of the `map_unknown_value` encoding option. + +This function formats IPv6 addresses by following the recommendation defined in RFC 5952. +Note that the trailing 32 bytes of special IPv6 addresses such as IPv4-Compatible (::X.X.X.X), +IPv4-Mapped (::ffff:X.X.X.X), IPv4-Translated (::ffff:0:X.X.X.X) and IPv4/IPv6 translation +(64:ff9b::X.X.X.X and 64:ff9b:1::X.X.X.X ~ 64:ff9b:1:ffff:ffff:ffff:X.X.X.X) are formatted +using the IPv4 format. + +``` + > EncodeOpt = [{map_unknown_value, fun jsone:ip_address_to_json_string/1}]. + > jsone:encode(#{ip => {127, 0, 0, 1}}, EncodeOpt). + <<"{\"ip\":\"127.0.0.1\"}">> + > {ok, Addr} = inet:parse_address("2001:DB8:0000:0000:0001:0000:0000:0001"). + > jsone:encode(Addr, EncodeOpt). + <<"\"2001:db8::1:0:0:1\"">> + > jsone:encode([foo, {0, 0, 0, 0, 0, 16#FFFF, 16#7F00, 16#0001}], EncodeOpt). + <<"[\"foo\",\"::ffff:127.0.0.1\"]">> +``` + + + +### term_to_json_string/1 ### + +

+term_to_json_string(X::term()) -> {ok, json_string()} | error
+
+
+ +Converts the given term `X` to its string representation (i.e., the result of `io_lib:format("~p", [X])`). + ### try_decode/1 ### @@ -498,6 +621,37 @@ Decodes an erlang term from json text (a utf8 encoded binary) [{line,208}]}]}} ``` + + +### try_decode_stream/1 ### + +

+try_decode_stream(Json::binary()) -> incomplete_try()
+
+
+ +Equivalent to [`try_decode_stream(Json, [])`](#try_decode_stream-2). + + + +### try_decode_stream/2 ### + +

+try_decode_stream(Json::binary(), Options::[decode_option()]) -> incomplete_try()
+
+
+ +Decodes an Erlang from JSON text in chunks. + +``` + 1> {incomplete, F1} = jsone:try_decode(<<"[1,2,">>, []). + {incomplete, #Fun} + 2> {incomplete, F2} = F1(<<"3] \"next value\"">>). + {incomplete, #Fun} + 2> F2(end_stream). + {ok,[1,2,3],<<" \"next value\"">>} +``` + ### try_encode/1 ### @@ -528,3 +682,4 @@ Encodes an erlang term into json text (a utf8 encoded binary) [hoge,[{array_values,[2]}],<<"[1,">>], [{line,86}]}]}} ``` + diff --git a/src/jsone.erl b/src/jsone.erl index 725fc60..4e1d146 100644 --- a/src/jsone.erl +++ b/src/jsone.erl @@ -31,6 +31,8 @@ %%-------------------------------------------------------------------------------- -export([decode/1, decode/2, try_decode/1, try_decode/2, + decode_stream/1, decode_stream/2, + try_decode_stream/1, try_decode_stream/2, encode/1, encode/2, try_encode/1, try_encode/2, term_to_json_string/1, @@ -49,6 +51,8 @@ json_object_format_map/0, json_scalar/0, + incomplete/0, + incomplete_try/0, encode_option/0, decode_option/0, float_format_option/0, @@ -130,6 +134,17 @@ -type json_scalar() :: json_boolean() | json_number() | json_string(). +-type incomplete_fun() :: fun((binary() | end_stream) -> + incomplete() | json_value()). +-type incomplete() :: {incomplete, incomplete_fun()}. + +-type incomplete_try_fun() :: fun((binary() | end_stream) -> + incomplete_try() | + {ok, json_value(), Remainings :: binary()} | + {error, {Reason :: term(), [stack_item()]}}). + +-type incomplete_try() :: {incomplete, incomplete_try_fun()}. + -type float_format_option() :: {scientific, Decimals :: 0 .. 249} | {decimals, Decimals :: 0 .. 253} | compact. %% `scientific':
%% - The float will be formatted using scientific notation with `Decimals' digits of precision.
@@ -287,7 +302,6 @@ %% - If the value is `first' then the first duplicate key/value is returned.
%% - If the value is `last' then the last duplicate key/value is returned. %% - default: `first'
-%% -type stack_item() :: {Module :: module(), Function :: atom(), @@ -346,7 +360,9 @@ decode(Json, Options) -> %% @equiv try_decode(Json, []) --spec try_decode(binary()) -> {ok, json_value(), Remainings :: binary()} | {error, {Reason :: term(), [stack_item()]}}. +-spec try_decode(binary()) -> + {ok, json_value(), Remainings :: binary()} | + {error, {Reason :: term(), [stack_item()]}}. try_decode(Json) -> try_decode(Json, []). @@ -363,11 +379,67 @@ try_decode(Json) -> %% [{line,208}]}]}} %% ''' -spec try_decode(binary(), [decode_option()]) -> - {ok, json_value(), Remainings :: binary()} | {error, {Reason :: term(), [stack_item()]}}. + {ok, json_value(), Remainings :: binary()} | + {error, {Reason :: term(), [stack_item()]}}. try_decode(Json, Options) -> jsone_decode:decode(Json, Options). +%% @equiv decode_stream(Json, []) +-spec decode_stream(binary()) -> incomplete(). +decode_stream(Json) -> + decode_stream(Json, []). + + +%% @doc Decodes an Erlang from JSON text in chunks. +%% +%% Instead of returning a result, `{incomplete, fun()}' is returned. The +%% returned fun takes a single argument and it should called to continue the +%% decoding. When all the input has been provided, the fun should be called with +%% `end_stream' to signal the end of input and then the fun returns a result or +%% an error, as `decode/2' would do. +%% +%% ``` +%% 1> {incomplete, F1} = jsone:decode(<<"[1,2,">>, []). +%% {incomplete, #Fun} +%% 2> {incomplete, F2} = F1(<<"3]>>). +%% {incomplete, #Fun} +%% 3> F2(end_stream). +%% [1,2,3] +%% ''' + +-spec decode_stream(binary(), [decode_option()]) -> incomplete(). +decode_stream(Json, Options) -> + try + {incomplete, ContinueFun} = try_decode_stream(Json, Options), + {incomplete, replace_incomplete_fun(ContinueFun)} + catch + error:{badmatch, {error, {Reason, [StackItem]}}} ?CAPTURE_STACKTRACE-> + erlang:raise(error, Reason, [StackItem | ?GET_STACKTRACE]) + end. + + +%% @equiv try_decode_stream(Json, []) +-spec try_decode_stream(binary()) -> incomplete_try(). +try_decode_stream(Json) -> + try_decode_stream(Json, []). + + +%% @doc Decodes an Erlang from JSON text in chunks. +%% +%% ``` +%% 1> {incomplete, F1} = jsone:try_decode(<<"[1,2,">>, []). +%% {incomplete, #Fun} +%% 2> {incomplete, F2} = F1(<<"3] \"next value\"">>). +%% {incomplete, #Fun} +%% 2> F2(end_stream). +%% {ok,[1,2,3],<<" \"next value\"">>} +%% ''' +-spec try_decode_stream(binary(), [decode_option()]) -> incomplete_try(). +try_decode_stream(Json, Options) -> + jsone_decode:decode_stream(Json, Options). + + %% @equiv encode(JsonValue, []) -spec encode(json_value()) -> binary(). encode(JsonValue) -> @@ -471,3 +543,25 @@ check_decode_remainings(<<$\n, Bin/binary>>) -> check_decode_remainings(Bin); check_decode_remainings(<>) -> erlang:error(badarg, [Bin]). + +%% Replace the fun in `{incomplete, Fun}' from `jsone_decode:decode_steram/2' +%% with a fun that returns the same result as `jsone:decode/1,2'. +replace_incomplete_fun(Fun) -> + fun (end_stream) -> + try + {ok, Value, Remainings} = Fun(end_stream), + check_decode_remainings(Remainings), + Value + catch + error:{badmatch, {error, {Reason, [StackItem]}}} ?CAPTURE_STACKTRACE-> + erlang:raise(error, Reason, [StackItem | ?GET_STACKTRACE]) + end; + (Input) -> + try + {incomplete, ContinueFun} = Fun(Input), + {incomplete, replace_incomplete_fun(ContinueFun)} + catch + error:{badmatch, {error, {Reason, [StackItem]}}} ?CAPTURE_STACKTRACE-> + erlang:raise(error, Reason, [StackItem | ?GET_STACKTRACE]) + end + end. diff --git a/src/jsone_decode.erl b/src/jsone_decode.erl index 33160e0..d44aeb1 100644 --- a/src/jsone_decode.erl +++ b/src/jsone_decode.erl @@ -34,7 +34,7 @@ %%-------------------------------------------------------------------------------- %% Exported API %%-------------------------------------------------------------------------------- --export([decode/1, decode/2]). +-export([decode/1, decode/2, decode_stream/2]). %%-------------------------------------------------------------------------------- %% Macros & Records & Types @@ -61,37 +61,57 @@ {object_value, jsone:json_string(), jsone:json_object_members()} | {object_next, jsone:json_object_members()}. --type decode_result() :: {ok, jsone:json_value(), Rest :: binary()} | {error, {Reason :: term(), [jsone:stack_item()]}}. +-type incomplete() :: jsone:incomplete_try(). --record(decode_opt_v2, { +-type decode_result() :: {ok, jsone:json_value(), Rest :: binary()} | + incomplete() | + {error, {Reason :: term(), [jsone:stack_item()]}}. + +-record(decode_opt_v3, { object_format = ?DEFAULT_OBJECT_FORMAT :: tuple | proplist | map, allow_ctrl_chars = false :: boolean(), reject_invalid_utf8 = false :: boolean(), keys = binary :: 'binary' | 'atom' | 'existing_atom' | 'attempt_atom', undefined_as_null = false :: boolean(), - duplicate_map_keys = first :: first | last + duplicate_map_keys = first :: first | last, + stream = false :: boolean() }). --define(OPT, #decode_opt_v2). --type opt() :: #decode_opt_v2{}. +-define(OPT, #decode_opt_v3). + +-type opt() :: #decode_opt_v3{}. %%-------------------------------------------------------------------------------- %% Exported Functions %%-------------------------------------------------------------------------------- --spec decode(binary()) -> decode_result(). +-spec decode(binary()) -> + {ok, jsone:json_value(), Rest :: binary()} | + {error, {Reason :: term(), [jsone:stack_item()]}}. decode(Json) -> decode(Json, []). --spec decode(binary(), [jsone:decode_option()]) -> decode_result(). +-spec decode(binary(), [jsone:decode_option()]) -> + {ok, jsone:json_value(), Rest :: binary()} | + {error, {Reason :: term(), [jsone:stack_item()]}}. decode(<>, Options) -> Opt = parse_options(Options), whitespace(Json, value, [], <<"">>, Opt). +-spec decode_stream(binary(), [jsone:decode_option()]) -> incomplete(). +decode_stream(<>, Options) -> + Opt0 = parse_options(Options), + Opt = Opt0?OPT{stream = true}, + Result = whitespace(Json, value, [], <<"">>, Opt), + incomplete_result(Result). %%-------------------------------------------------------------------------------- %% Internal Functions %%-------------------------------------------------------------------------------- + +%% The functions which return decode_result() can only return incomplete() if +%% the stream option is set. + -spec next(binary(), jsone:json_value(), [next()], binary(), opt()) -> decode_result(). next(<>, Value, [], _Buf, _Opt) -> {ok, Value, Bin}; @@ -115,6 +135,8 @@ whitespace(<<$\r, Bin/binary>>, Next, Nexts, Buf, Opt) -> whitespace(Bin, Next, Nexts, Buf, Opt); whitespace(<<$\n, Bin/binary>>, Next, Nexts, Buf, Opt) -> whitespace(Bin, Next, Nexts, Buf, Opt); +whitespace(<<>>, Next, Nexts, Buf, Opt) when Opt?OPT.stream -> + incomplete(fun whitespace/5, [<<>>, Next, Nexts, Buf, Opt]); whitespace(<>, Next, Nexts, Buf, Opt) -> case Next of value -> @@ -149,6 +171,12 @@ value(<<${, Bin/binary>>, Nexts, Buf, Opt) -> whitespace(Bin, object, Nexts, Buf, Opt); value(<<$", Bin/binary>>, Nexts, Buf, Opt) -> string(Bin, byte_size(Buf), Nexts, Buf, Opt); +value(<> = Bin, Nexts, Buf, Opt) when + Opt?OPT.stream, + (C =:= $f andalso byte_size(Bin) < 5) orelse % incomplete "false" + (C =:= $t andalso byte_size(Bin) < 4) orelse % incomplete "true" + (C =:= $n andalso byte_size(Bin) < 4) -> % incomplete "null" + incomplete(fun value/4, [Bin, Nexts, Buf, Opt]); value(<>, Nexts, Buf, Opt) -> number(Bin, Nexts, Buf, Opt). @@ -156,6 +184,8 @@ value(<>, Nexts, Buf, Opt) -> -spec array(binary(), [next()], binary(), opt()) -> decode_result(). array(<<$], Bin/binary>>, Nexts, Buf, Opt) -> next(Bin, [], Nexts, Buf, Opt); +array(<<>>, Nexts, Buf, Opt) when Opt?OPT.stream -> + incomplete(fun array/4, [<<>>, Nexts, Buf, Opt]); array(<>, Nexts, Buf, Opt) -> value(Bin, [{array_next, []} | Nexts], Buf, Opt). @@ -165,6 +195,8 @@ array_next(<<$], Bin/binary>>, Values, Nexts, Buf, Opt) -> next(Bin, lists:reverse(Values), Nexts, Buf, Opt); array_next(<<$,, Bin/binary>>, Values, Nexts, Buf, Opt) -> whitespace(Bin, value, [{array_next, Values} | Nexts], Buf, Opt); +array_next(<<>>, Values, Nexts, Buf, Opt) when Opt?OPT.stream -> + incomplete(fun array_next/5, [<<>>, Values, Nexts, Buf, Opt]); array_next(Bin, Values, Nexts, Buf, Opt) -> ?ERROR(array_next, [Bin, Values, Nexts, Buf, Opt]). @@ -172,6 +204,8 @@ array_next(Bin, Values, Nexts, Buf, Opt) -> -spec object(binary(), [next()], binary(), opt()) -> decode_result(). object(<<$}, Bin/binary>>, Nexts, Buf, Opt) -> next(Bin, make_object([], Opt), Nexts, Buf, Opt); +object(<<>>, Nexts, Buf, Opt) when Opt?OPT.stream -> + incomplete(fun object/4, [<<>>, Nexts, Buf, Opt]); object(<>, Nexts, Buf, Opt) -> object_key(Bin, [], Nexts, Buf, Opt). @@ -179,6 +213,8 @@ object(<>, Nexts, Buf, Opt) -> -spec object_key(binary(), jsone:json_object_members(), [next()], binary(), opt()) -> decode_result(). object_key(<<$", Bin/binary>>, Members, Nexts, Buf, Opt) -> string(Bin, byte_size(Buf), [{object_value, Members} | Nexts], Buf, Opt); +object_key(<<>>, Members, Nexts, Buf, Opt) when Opt?OPT.stream -> + incomplete(fun object_key/5, [<<>>, Members, Nexts, Buf, Opt]); object_key(<>, Members, Nexts, Buf, Opt) -> ?ERROR(object_key, [Bin, Members, Nexts, Buf, Opt]). @@ -187,6 +223,8 @@ object_key(<>, Members, Nexts, Buf, Opt) -> decode_result(). object_value(<<$:, Bin/binary>>, Key, Members, Nexts, Buf, Opt) -> whitespace(Bin, value, [{object_next, object_key(Key, Opt), Members} | Nexts], Buf, Opt); +object_value(<<>>, Key, Members, Nexts, Buf, Opt) when Opt?OPT.stream -> + incomplete(fun object_value/6, [<<>>, Key, Members, Nexts, Buf, Opt]); object_value(Bin, Key, Members, Nexts, Buf, Opt) -> ?ERROR(object_value, [Bin, Key, Members, Nexts, Buf, Opt]). @@ -213,6 +251,8 @@ object_next(<<$}, Bin/binary>>, Members, Nexts, Buf, Opt) -> next(Bin, make_object(Members, Opt), Nexts, Buf, Opt); object_next(<<$,, Bin/binary>>, Members, Nexts, Buf, Opt) -> whitespace(Bin, {object_key, Members}, Nexts, Buf, Opt); +object_next(<<>>, Members, Nexts, Buf, Opt) when Opt?OPT.stream -> + incomplete(fun object_next/5, [<<>>, Members, Nexts, Buf, Opt]); object_next(Bin, Members, Nexts, Buf, Opt) -> ?ERROR(object_next, [Bin, Members, Nexts, Buf, Opt]). @@ -232,6 +272,8 @@ string(<<$", Bin/binary>>, Base, Start, Nexts, Buf, Opt) -> Buf2 = <>, next(Bin, binary:part(Buf2, Start, byte_size(Buf2) - Start), Nexts, Buf2, Opt) end; +string(<<$\\>> = Bin, Base, Start, Nexts, Buf, Opt) when Opt?OPT.stream -> + incomplete_string(Bin, Base, Start, Nexts, Buf, Opt); string(<<$\\, B/binary>>, Base, Start, Nexts, Buf, Opt) -> Prefix = binary:part(Base, 0, byte_size(Base) - byte_size(B) - 1), case B of @@ -265,12 +307,20 @@ string(<<_/utf8, Bin/binary>>, Base, Start, Nexts, Buf, Opt) when Opt?OPT.allow_ string(Bin, Base, Start, Nexts, Buf, Opt); string(<>, Base, Start, Nexts, Buf, Opt) when 16#20 =< C -> string(Bin, Base, Start, Nexts, Buf, Opt); +string(<>, Base, Start, Nexts, Buf, Opt = ?OPT{stream = true}) + when C >= 16#c0, C =< 16#df, byte_size(Bin) < 1; % 110xxxxx + C >= 16#e0, C =< 16#ef, byte_size(Bin) < 2; % 1110xxxx + C >= 16#f0, C =< 16#f7, byte_size(Bin) < 3 -> % 11110xxx + %% Partial UTF-8 character. The first byte determines the number of following bytes. + incomplete_string(<>, Base, Start, Nexts, Buf, Opt); +string(<<>>, Base, Start, Nexts, Buf, Opt) when Opt?OPT.stream -> + incomplete_string(<<>>, Base, Start, Nexts, Buf, Opt); string(Bin, Base, Start, Nexts, Buf, Opt) -> ?ERROR(string, [Bin, Base, Start, Nexts, Buf, Opt]). -spec unicode_string(binary(), non_neg_integer(), [next()], binary(), opt()) -> decode_result(). -unicode_string(<>, Start, Nexts, Buf, Opt) -> +unicode_string(<> = Bin0, Start, Nexts, Buf, Opt) -> try binary_to_integer(N, 16) of @@ -290,6 +340,10 @@ unicode_string(<>, Start, Nexts, Buf, Opt) -> error:badarg -> ?ERROR(unicode_string, [<>, Start, Nexts, Buf, Opt]) end; + <<$\\, $u, N2/binary>> when byte_size(N2) < 4, Opt?OPT.stream -> + incomplete(fun unicode_string/5, [Bin0, Start, Nexts, Buf, Opt]); + _ when Bin =:= <<$\\>> orelse Bin =:= <<>>, Opt?OPT.stream -> + incomplete(fun unicode_string/5, [Bin0, Start, Nexts, Buf, Opt]); _ -> ?ERROR(unicode_string, [<>, Start, Nexts, Buf, Opt]) end; @@ -304,6 +358,8 @@ unicode_string(<>, Start, Nexts, Buf, Opt) -> error:badarg -> ?ERROR(unicode_string, [<>, Start, Nexts, Buf, Opt]) end; +unicode_string(Bin, Start, Nexts, Buf, Opt) when byte_size(Bin) < 4, Opt?OPT.stream -> + incomplete(fun unicode_string/5, [Bin, Start, Nexts, Buf, Opt]); unicode_string(Bin, Start, Nexts, Buf, Opt) -> ?ERROR(unicode_string, [Bin, Start, Nexts, Buf, Opt]). @@ -316,10 +372,14 @@ number(<>, Nexts, Buf, Opt) -> -spec number_integer_part(binary(), 1 | -1, [next()], binary(), opt()) -> decode_result(). +number_integer_part(<<$0>>, Sign, Nexts, Buf, Opt) when Opt?OPT.stream -> + incomplete(fun number_fraction_part/6, [<<>>, Sign, 0, Nexts, Buf, Opt]); number_integer_part(<<$0, Bin/binary>>, Sign, Nexts, Buf, Opt) -> number_fraction_part(Bin, Sign, 0, Nexts, Buf, Opt); number_integer_part(<>, Sign, Nexts, Buf, Opt) when $1 =< C, C =< $9 -> number_integer_part_rest(Bin, C - $0, Sign, Nexts, Buf, Opt); +number_integer_part(<<>>, Sign, Nexts, Buf, Opt) when Opt?OPT.stream -> + incomplete(fun number_integer_part/5, [<<>>, Sign, Nexts, Buf, Opt]); number_integer_part(Bin, Sign, Nexts, Buf, Opt) -> ?ERROR(number_integer_part, [Bin, Sign, Nexts, Buf, Opt]). @@ -327,6 +387,8 @@ number_integer_part(Bin, Sign, Nexts, Buf, Opt) -> -spec number_integer_part_rest(binary(), non_neg_integer(), 1 | -1, [next()], binary(), opt()) -> decode_result(). number_integer_part_rest(<>, N, Sign, Nexts, Buf, Opt) when $0 =< C, C =< $9 -> number_integer_part_rest(Bin, N * 10 + C - $0, Sign, Nexts, Buf, Opt); +number_integer_part_rest(<<>>, N, Sign, Nexts, Buf, Opt) when Opt?OPT.stream -> + incomplete(fun number_integer_part_rest/6, [<<>>, N, Sign, Nexts, Buf, Opt]); number_integer_part_rest(<>, N, Sign, Nexts, Buf, Opt) -> number_fraction_part(Bin, Sign, N, Nexts, Buf, Opt). @@ -342,6 +404,8 @@ number_fraction_part(<>, Sign, Int, Nexts, Buf, Opt) -> decode_result(). number_fraction_part_rest(<>, Sign, N, DecimalOffset, Nexts, Buf, Opt) when $0 =< C, C =< $9 -> number_fraction_part_rest(Bin, Sign, N * 10 + C - $0, DecimalOffset + 1, Nexts, Buf, Opt); +number_fraction_part_rest(<<>>, Sign, N, DecimalOffset, Nexts, Buf, Opt) when Opt?OPT.stream -> + incomplete(fun number_fraction_part_rest/7, [<<>>, Sign, N, DecimalOffset, Nexts, Buf, Opt]); number_fraction_part_rest(<>, Sign, N, DecimalOffset, Nexts, Buf, Opt) when DecimalOffset > 0 -> number_exponation_part(Bin, Sign * N, DecimalOffset, Nexts, Buf, Opt); number_fraction_part_rest(Bin, Sign, N, DecimalOffset, Nexts, Buf, Opt) -> @@ -357,10 +421,15 @@ number_exponation_part(<<$e, $-, Bin/binary>>, N, DecimalOffset, Nexts, Buf, Opt number_exponation_part(Bin, N, DecimalOffset, -1, 0, true, Nexts, Buf, Opt); number_exponation_part(<<$E, $-, Bin/binary>>, N, DecimalOffset, Nexts, Buf, Opt) -> number_exponation_part(Bin, N, DecimalOffset, -1, 0, true, Nexts, Buf, Opt); +number_exponation_part(<>, N, DecimalOffset, Nexts, Buf, Opt) when E == $e orelse E == $E, + Opt?OPT.stream -> + incomplete(fun number_exponation_part/6, [<>, N, DecimalOffset, Nexts, Buf, Opt]); number_exponation_part(<<$e, Bin/binary>>, N, DecimalOffset, Nexts, Buf, Opt) -> number_exponation_part(Bin, N, DecimalOffset, 1, 0, true, Nexts, Buf, Opt); number_exponation_part(<<$E, Bin/binary>>, N, DecimalOffset, Nexts, Buf, Opt) -> number_exponation_part(Bin, N, DecimalOffset, 1, 0, true, Nexts, Buf, Opt); +number_exponation_part(<<>>, N, DecimalOffset, Nexts, Buf, Opt) when Opt?OPT.stream -> + incomplete(fun number_exponation_part/6, [<<>>, N, DecimalOffset, Nexts, Buf, Opt]); number_exponation_part(<>, N, DecimalOffset, Nexts, Buf, Opt) -> case DecimalOffset of 0 -> @@ -381,6 +450,8 @@ number_exponation_part(<>, N, DecimalOffset, Nexts, Buf, Opt) -> opt()) -> decode_result(). number_exponation_part(<>, N, DecimalOffset, ExpSign, Exp, _, Nexts, Buf, Opt) when $0 =< C, C =< $9 -> number_exponation_part(Bin, N, DecimalOffset, ExpSign, Exp * 10 + C - $0, false, Nexts, Buf, Opt); +number_exponation_part(<<>>, N, DecimalOffset, ExpSign, Exp, IsFirst, Nexts, Buf, Opt) when Opt?OPT.stream -> + incomplete(fun number_exponation_part/9, [<<>>, N, DecimalOffset, ExpSign, Exp, IsFirst, Nexts, Buf, Opt]); number_exponation_part(<>, N, DecimalOffset, ExpSign, Exp, false, Nexts, Buf, Opt) -> Pos = ExpSign * Exp - DecimalOffset, try @@ -400,6 +471,52 @@ number_exponation_part(<>, N, DecimalOffset, ExpSign, Exp, false, Ne number_exponation_part(Bin, N, DecimalOffset, ExpSign, Exp, IsFirst, Nexts, Buf, Opt) -> ?ERROR(number_exponation_part, [Bin, N, DecimalOffset, ExpSign, Exp, IsFirst, Nexts, Buf, Opt]). +incomplete_result({ok, Value, Rest} = Result) -> + %% The user needs to call Fun(end_stream) to get the ok tuple. + {incomplete, + fun (end_stream) -> + Result; + (<>) -> + incomplete_result({ok, Value, <>}) + end}; +incomplete_result({error, _} = Error) -> + {incomplete, + fun (end_stream) -> + Error; + (<<_More/binary>>) -> + incomplete_result(Error) + end}; +incomplete_result({incomplete, _} = Incomplete) -> + Incomplete. + +-spec incomplete(fun(), list()) -> jsone:incomplete_try(). +incomplete(Fun, [Remains | Args]) -> + {incomplete, + fun F(end_stream) -> + %% The last arg is always Opts + [Opt | RevArgs] = lists:reverse(Args), + Args1 = lists:reverse([Opt?OPT{stream = false} | RevArgs]), + apply(Fun, [Remains | Args1]); + F(<<>>) -> + {incomplete, F}; + F(More) when is_binary(More) -> + Bin = <>, + incomplete_result(apply(Fun, [Bin | Args])) + end}. + +-spec incomplete_string(binary(), binary(), non_neg_integer(), [next()], binary(), opt()) -> + jsone:incomplete(). +incomplete_string(Remains, Base, Start, Nexts, Buf, Opt) -> + {incomplete, + fun F(end_stream) -> + string(Remains, Base, Start, Nexts, Buf, Opt?OPT{stream = false}); + F(<<>>) -> + {incomplete, F}; + F(More) when is_binary(More) -> + incomplete_result(string(<>, + <>, + Start, Nexts, Buf, Opt)) + end}. -spec make_object(jsone:json_object_members(), opt()) -> jsone:json_object(). make_object(Members, ?OPT{object_format = tuple}) -> @@ -434,5 +551,7 @@ parse_option([undefined_as_null | T], Opt) -> parse_option(T, Opt?OPT{undefined_as_null = true}); parse_option([{duplicate_map_keys, V} | T], Opt) when V =:= first; V =:= last -> parse_option(T, Opt?OPT{duplicate_map_keys = V}); +parse_option([stream | T], Opt) -> + parse_option(T, Opt?OPT{stream = true}); parse_option(List, Opt) -> error(badarg, [List, Opt]). diff --git a/test/jsone_decode_tests.erl b/test/jsone_decode_tests.erl index 81c153e..087494f 100644 --- a/test/jsone_decode_tests.erl +++ b/test/jsone_decode_tests.erl @@ -6,7 +6,7 @@ -ifdef('NO_MAP_TYPE'). -define(MAP_OBJECT_TYPE, tuple). --define(OBJ0, {[]}). +-define(OBJ0, {[]}). -define(OBJ1(K, V), {[{K, V}]}). -define(OBJ2(K1, V1, K2, V2), {[{K1, V1}, {K2, V2}]}). -define(OBJ2_DUP_KEY(K1, V1, K2, V2), ?OBJ2(K1, V1, K2, V2)). @@ -19,36 +19,35 @@ -define(OBJ2_DUP_KEY_LAST(_K1, _V1, K2, V2), #{K2 => V2}). % the last value is used -endif. - decode_test_() -> [ %% Symbols - {"false", fun() -> ?assertEqual({ok, false, <<"">>}, jsone_decode:decode(<<"false">>)) end}, - {"true", fun() -> ?assertEqual({ok, true, <<"">>}, jsone_decode:decode(<<"true">>)) end}, - {"null", fun() -> ?assertEqual({ok, null, <<"">>}, jsone_decode:decode(<<"null">>)) end}, + {"false", fun () -> ?assertEqual({ok, false, <<"">>}, jsone_decode:decode(<<"false">>)) end}, + {"true", fun () -> ?assertEqual({ok, true, <<"">>}, jsone_decode:decode(<<"true">>)) end}, + {"null", fun () -> ?assertEqual({ok, null, <<"">>}, jsone_decode:decode(<<"null">>)) end}, %% Numbers: Integer - {"positive integer", fun() -> ?assertEqual({ok, 1, <<"">>}, jsone_decode:decode(<<"1">>)) end}, - {"zero", fun() -> ?assertEqual({ok, 0, <<"">>}, jsone_decode:decode(<<"0">>)) end}, - {"negative integer", fun() -> ?assertEqual({ok, -1, <<"">>}, jsone_decode:decode(<<"-1">>)) end}, + {"positive integer", fun () -> ?assertEqual({ok, 1, <<"">>}, jsone_decode:decode(<<"1">>)) end}, + {"zero", fun () -> ?assertEqual({ok, 0, <<"">>}, jsone_decode:decode(<<"0">>)) end}, + {"negative integer", fun () -> ?assertEqual({ok, -1, <<"">>}, jsone_decode:decode(<<"-1">>)) end}, {"large integer (no limit on size)", - fun() -> - ?assertEqual({ok, 111111111111111111111111111111111111111111111111111111111111111111111111111111, <<"">>}, - jsone_decode:decode(<<"111111111111111111111111111111111111111111111111111111111111111111111111111111">>)) + fun () -> + ?assertEqual({ok, 111111111111111111111111111111111111111111111111111111111111111111111111111111, <<"">>}, + jsone_decode:decode(<<"111111111111111111111111111111111111111111111111111111111111111111111111111111">>)) end}, {"integer with leading zero (interpreted as zero and remaining binary)", - fun() -> + fun () -> ?assertEqual({ok, 0, <<"0">>}, jsone_decode:decode(<<"00">>)), ?assertEqual({ok, 0, <<"1">>}, jsone_decode:decode(<<"01">>)), ?assertEqual({ok, 0, <<"1">>}, jsone_decode:decode(<<"-01">>)) end}, {"integer can't begin with an explicit plus sign", - fun() -> ?assertMatch({error, {badarg, _}}, jsone_decode:decode(<<"+1">>)) end}, + fun () -> ?assertMatch({error, {badarg, _}}, jsone_decode:decode(<<"+1">>)) end}, %% Numbers: Floats - {"float: decimal notation", fun() -> ?assertEqual({ok, 1.23, <<"">>}, jsone_decode:decode(<<"1.23">>)) end}, + {"float: decimal notation", fun () -> ?assertEqual({ok, 1.23, <<"">>}, jsone_decode:decode(<<"1.23">>)) end}, {"float: exponential notation", - fun() -> + fun () -> ?assertEqual({ok, 12.345, <<"">>}, jsone_decode:decode(<<"12345e-3">>)), % lower case 'e' ?assertEqual({ok, 12.345, <<"">>}, jsone_decode:decode(<<"12345E-3">>)), % upper case 'E' ?assertEqual({ok, 12.345, <<"">>}, jsone_decode:decode(<<"12345.0e-3">>)), @@ -59,7 +58,7 @@ decode_test_() -> ?assertEqual({ok, 123.0, <<"">>}, jsone_decode:decode(<<"1.23000000000000000000e+02">>)) end}, {"float: invalid format", - fun() -> + fun () -> ?assertMatch({error, {badarg, _}}, jsone_decode:decode(<<".123">>)), % omitted integer part ?assertMatch({error, {badarg, _}}, jsone_decode:decode(<<"0.">>)), % omitted fraction part: EOS ?assertMatch({error, {badarg, _}}, jsone_decode:decode(<<"0.e+3">>)), % omitted fraction part: with exponent part @@ -74,15 +73,15 @@ decode_test_() -> end}, %% Strings - {"simple string", fun() -> ?assertEqual({ok, <<"abc">>, <<"">>}, jsone_decode:decode(<<"\"abc\"">>)) end}, + {"simple string", fun () -> ?assertEqual({ok, <<"abc">>, <<"">>}, jsone_decode:decode(<<"\"abc\"">>)) end}, {"string: escaped characters", - fun() -> + fun () -> Input = list_to_binary([$", [[$\\, C] || C <- [$", $/, $\\, $b, $f, $n, $r, $t]], $"]), Expected = <<"\"\/\\\b\f\n\r\t">>, ?assertEqual({ok, Expected, <<"">>}, jsone_decode:decode(Input)) end}, {"string: escaped Unicode characters", - fun() -> + fun () -> %% japanese Input1 = <<"\"\\u3042\\u3044\\u3046\\u3048\\u304A\"">>, Expected1 = <<"あいうえお">>, % assumed that the encoding of this file is UTF-8 @@ -104,28 +103,28 @@ decode_test_() -> ?assertEqual({ok, Expected4, <<"">>}, jsone_decode:decode(Input4)) end}, {"string: surrogate pairs", - fun() -> + fun () -> Input = <<"\"\\ud848\\udc49\\ud848\\udc9a\\ud848\\udcfc\"">>, Expected = <<"𢁉𢂚𢃼">>, ?assertEqual({ok, Expected, <<"">>}, jsone_decode:decode(Input)) end}, {"string: control characters", - fun() -> + fun () -> Ctrls = lists:seq(0, 16#1f), - lists:foreach(fun(C) -> - %% Control characters are unacceptable - ?assertMatch({error, {badarg, _}}, jsone_decode:decode(<<$", C, $">>)) + lists:foreach(fun (C) -> + %% Control characters are unacceptable + ?assertMatch({error, {badarg, _}}, jsone_decode:decode(<<$", C, $">>)) end, Ctrls), - lists:foreach(fun(C) -> - %% `allow_ctrl_chars' option allows strings which contain unescaped control characters - ?assertEqual({ok, <>, <<"">>}, - jsone_decode:decode(<<$", C, $">>, [{allow_ctrl_chars, true}])) + lists:foreach(fun (C) -> + %% `allow_ctrl_chars' option allows strings which contain unescaped control characters + ?assertEqual({ok, <>, <<"">>}, + jsone_decode:decode(<<$", C, $">>, [{allow_ctrl_chars, true}])) end, Ctrls) end}, {"string: invalid escape characters", - fun() -> + fun () -> ?assertMatch({error, {badarg, _}}, jsone_decode:decode(<<"\"\\z\"">>)), % '\z' is undefined ?assertMatch({error, {badarg, _}}, jsone_decode:decode(<<"\"\\uab\"">>)), % too few hex characters ?assertMatch({error, {badarg, _}}, jsone_decode:decode(<<"\"\\ud848\"">>)), % high(first) surrogate only @@ -138,117 +137,117 @@ decode_test_() -> %% Arrays {"simple array", - fun() -> + fun () -> Input = <<"[1,2,\"abc\",null]">>, Expected = [1, 2, <<"abc">>, null], ?assertEqual({ok, Expected, <<"">>}, jsone_decode:decode(Input)) end}, {"array: contains whitespaces", - fun() -> + fun () -> Input = <<"[ 1,\t2, \n \"abc\",\r null]">>, Expected = [1, 2, <<"abc">>, null], ?assertEqual({ok, Expected, <<"">>}, jsone_decode:decode(Input)) end}, {"empty array", - fun() -> + fun () -> ?assertEqual({ok, [], <<"">>}, jsone_decode:decode(<<"[]">>)), ?assertEqual({ok, [], <<"">>}, jsone_decode:decode(<<"[ \t\r\n]">>)) end}, {"array: trailing comma is disallowed", - fun() -> + fun () -> Input = <<"[1, 2, \"abc\", null, ]">>, ?assertMatch({error, {badarg, _}}, jsone_decode:decode(Input)) end}, {"array: missing comma", - fun() -> + fun () -> Input = <<"[1 2, \"abc\", null]">>, % a missing comma between '1' and '2' ?assertMatch({error, {badarg, _}}, jsone_decode:decode(Input)) end}, {"array: missing closing bracket", - fun() -> + fun () -> Input = <<"[1, 2, \"abc\", null">>, ?assertMatch({error, {badarg, _}}, jsone_decode:decode(Input)) end}, %% Objects {"simple object", - fun() -> + fun () -> Input = <<"{\"1\":2,\"key\":\"value\"}">>, Expected = ?OBJ2(<<"1">>, 2, <<"key">>, <<"value">>), ?assertEqual({ok, Expected, <<"">>}, jsone_decode:decode(Input)), % `map' is the default format ?assertEqual({ok, Expected, <<"">>}, jsone_decode:decode(Input, [{object_format, ?MAP_OBJECT_TYPE}])) end}, {"simple object: tuple or proplist", - fun() -> + fun () -> Input = <<"{\"1\":2,\"key\":\"value\"}">>, Expected = {[{<<"1">>, 2}, {<<"key">>, <<"value">>}]}, ?assertEqual({ok, Expected, <<"">>}, jsone_decode:decode(Input, [{object_format, tuple}])), ?assertEqual({ok, element(1, Expected), <<"">>}, jsone_decode:decode(Input, [{object_format, proplist}])) end}, {"object: contains whitespaces", - fun() -> + fun () -> Input = <<"{ \"1\" :\t 2,\n\r\"key\" : \n \"value\"}">>, Expected = ?OBJ2(<<"1">>, 2, <<"key">>, <<"value">>), ?assertEqual({ok, Expected, <<"">>}, jsone_decode:decode(Input)) end}, {"empty object", - fun() -> + fun () -> ?assertEqual({ok, ?OBJ0, <<"">>}, jsone_decode:decode(<<"{}">>)), ?assertEqual({ok, ?OBJ0, <<"">>}, jsone_decode:decode(<<"{ \t\r\n}">>)), ?assertEqual({ok, {[]}, <<"">>}, jsone_decode:decode(<<"{}">>, [{object_format, tuple}])), ?assertEqual({ok, [{}], <<"">>}, jsone_decode:decode(<<"{}">>, [{object_format, proplist}])) end}, {"empty object: map", - fun() -> - ?assertEqual({ok, ?OBJ0, <<"">>}, jsone_decode:decode(<<"{}">>, [{object_format, ?MAP_OBJECT_TYPE}])) + fun () -> + ?assertEqual({ok, ?OBJ0, <<"">>}, jsone_decode:decode(<<"{}">>, [{object_format, ?MAP_OBJECT_TYPE}])) end}, {"duplicated members: map", - fun() -> + fun () -> Input = <<"{\"1\":\"first\",\"1\":\"second\"}">>, Expected = ?OBJ2_DUP_KEY(<<"1">>, <<"first">>, <<"1">>, <<"second">>), ?assertEqual({ok, Expected, <<"">>}, jsone_decode:decode(Input, [{object_format, ?MAP_OBJECT_TYPE}])) end}, {"duplicated members last: map", - fun() -> + fun () -> Input = <<"{\"1\":\"first\",\"1\":\"second\"}">>, Expected = ?OBJ2_DUP_KEY_LAST(<<"1">>, <<"first">>, <<"1">>, <<"second">>), ?assertEqual({ok, Expected, <<"">>}, jsone_decode:decode(Input, [{object_format, ?MAP_OBJECT_TYPE}, {duplicate_map_keys, last}])) end}, {"object: trailing comma is disallowed", - fun() -> + fun () -> Input = <<"{\"1\":2, \"key\":\"value\", }">>, io:format("~p\n", [catch jsone_decode:decode(Input)]), ?assertMatch({error, {badarg, _}}, jsone_decode:decode(Input, [{object_format, tuple}])) end}, {"object: missing comma", - fun() -> + fun () -> Input = <<"{\"1\":2 \"key\":\"value\"}">>, ?assertMatch({error, {badarg, _}}, jsone_decode:decode(Input)) end}, {"object: missing field key", - fun() -> + fun () -> Input = <<"{:2, \"key\":\"value\"}">>, ?assertMatch({error, {badarg, _}}, jsone_decode:decode(Input)) end}, {"object: non string key", - fun() -> + fun () -> Input = <<"{1:2, \"key\":\"value\"}">>, ?assertMatch({error, {badarg, _}}, jsone_decode:decode(Input)) end}, {"object: missing field value", - fun() -> + fun () -> Input = <<"{\"1\", \"key\":\"value\"}">>, ?assertMatch({error, {badarg, _}}, jsone_decode:decode(Input)) end}, {"object: missing closing brace", - fun() -> + fun () -> Input = <<"{\"1\":2 \"key\":\"value\"">>, ?assertMatch({error, {badarg, _}}, jsone_decode:decode(Input)) end}, {"atom keys", - fun() -> - KeyOpt = fun(Keys) -> [{keys, Keys}, {object_format, proplist}] end, + fun () -> + KeyOpt = fun (Keys) -> [{keys, Keys}, {object_format, proplist}] end, Input = <<"{\"foo\":\"ok\"}">>, ?assertEqual([{<<"foo">>, <<"ok">>}], jsone:decode(Input, KeyOpt(binary))), ?assertEqual([{foo, <<"ok">>}], jsone:decode(Input, KeyOpt(atom))), @@ -262,27 +261,134 @@ decode_test_() -> ?assertEqual(Value, atom_to_binary(Atom, latin1)) end}, {"garbage remainings chars", - fun() -> + fun () -> ?assertError(badarg, jsone:decode(<<"1@">>)), ?assertEqual(1, jsone:decode(<<"1 \n\t\r ">>)) % Whitespaces are OK end}, %% Others {"compound data", - fun() -> + fun () -> Input = <<" [true, {\"1\" : 2, \"array\":[[[[1]]], {\"ab\":\"cd\"}, false]}, null] ">>, Expected = [true, ?OBJ2(<<"1">>, 2, <<"array">>, [[[[1]]], ?OBJ1(<<"ab">>, <<"cd">>), false]), null], ?assertEqual({ok, Expected, <<" ">>}, jsone_decode:decode(Input)) end}, {"undefined_as_null option", - fun() -> + fun () -> ?assertEqual({ok, undefined, <<>>}, jsone_decode:decode(<<"null">>, [undefined_as_null])), % OK ?assertEqual({ok, null, <<>>}, jsone_decode:decode(<<"null">>, [])) % OK end}, {"Invalid UTF-8 characters", - fun() -> - Input = <<123, 34, 105, 100, 34, 58, 34, 190, 72, 94, 90, 253, 121, 94, 71, 73, 68, 91, 122, 211, 253, 32, - 94, 86, 67, 163, 253, 230, 34, 125>>, + fun () -> + Input = + <<123, 34, 105, 100, 34, 58, 34, 190, 72, 94, 90, 253, 121, 94, 71, 73, 68, 91, 122, 211, 253, 32, 94, + 86, 67, 163, 253, 230, 34, 125>>, ?assertMatch({ok, _, _}, jsone:try_decode(Input)), ?assertMatch({error, {badarg, _}}, jsone:try_decode(Input, [reject_invalid_utf8])) end}]. + +stream_decode_full_test_() -> + [{"stream one chunk", + fun () -> + {incomplete, Fun} = jsone_decode:decode_stream(<<"null">>, []), + ?assertEqual({ok, null, <<>>}, Fun(end_stream)) + end}, + {"stream two chunks", + fun () -> + {incomplete, Fun1} = jsone_decode:decode_stream(<<"nul">>, []), + {incomplete, Fun2} = Fun1(<<"l">>), + ?assertEqual({ok, null, <<>>}, Fun2(end_stream)) + end}]. + +stream_decode_byte_by_byte_test_() -> + %% A complex JSON document without errors. + %% Escaped strings include japanese, ascii, surrugate pairs and mixed. + Jsons = + [<<"true">>, + <<"false">>, + <<"null">>, % symbols + <<"1">>, + <<"0">>, + <<"-1">>, + <<"-0">>, % integers + %% large integer + <<"111111111111111111111111111111111111111111111111111111111111111111111111111111">>, + %% floats + <<"1.23">>, + <<"12345e-3">>, + <<"12345E-3">>, + <<"12345.0e-3">>, + <<"0.12345E2">>, + <<"0.12345e+2">>, + <<"0.12345E+2">>, + <<"-0.012345e3">>, + <<"1.23000000000000000000e+02">>, + %% strings + <<"\"abc\"">>, + <<"\"\\\"\\/\\\\\\b\\f\\n\\r\\t\"">>, + <<"\"\\u3042\\u3044\\u3046\\u3048\\u304A\"">>, + <<"\"\\u0061\\u0062\\u0063\"">>, + <<"\"\\u06DD\\u06DE\\u10AE\\u10AF\"">>, + <<"\"a\\u30421\\u3044bb\\u304622\\u3048ccc\\u304A333\"">>, + <<"\"\\ud848\\udc49\\ud848\\udc9a\\ud848\\udcfc\"">>, + <<"[1,2,\"abc\",null]">>, % simple array + <<"[ 1,\t2, \n \"abc\",\r null]">>, % array: contains whitespaces + <<"[]">>, + <<"[ \t\r\n]">>, % empty array + <<"{\"1\":2,\"key\":\"value\"}">>, % simple object + %% object: contains whitespaces + <<"{ \"1\" :\t 2,\n\r\"key\" : \n \"value\"}">>, + <<"{}">>, + <<"{ \t\r\n}">>, % empty object + <<"{\"1\":\"first\",\"1\":\"second\"}">>, % duplicated members + %% compound data + <<"[true, {\"1\" : 2, \"array\":[[[[1]]], {\"ab\":\"cd\"}, false]}, null]">>], + %% Test cases + [{"'" ++ binary_to_list(Json) ++ "'", fun () -> byte_by_byte(Json) end} || Json <- Jsons]. + +%% Parse one byte at a time, interleaved with parsing an empty binary +%% between every chunk. +byte_by_byte(Bin) -> + Result = + lists:foldl(fun (Byte, {incomplete, Continue1}) -> + {incomplete, Continue2} = Continue1(<>), + Continue2(<<>>) + end, + jsone_decode:decode_stream(<<>>, []), + binary_to_list(Bin)), + {incomplete, Fun} = Result, + {ok, Expected, <<>>} = jsone_decode:decode(Bin), + ?assertEqual({ok, Expected, <<>>}, Fun(end_stream)). + +stream_decode_byte_by_byte_using_jsone_test_() -> + Cases = + [{"trailing whitespace", <<"[true, {\"1\" : 2, \"array\":[[[[1]]], {\"ab\":\"cd\"}, false]}, null] ">>}, + {"trailing garbage", <<"[true, {\"1\" : 2, \"array\":[[[[1]]], {\"ab\":\"cd\"}, false]}, null] bla bla">>}], + [{Name, fun () -> byte_by_byte_using_jsone(Bin) end} || {Name, Bin} <- Cases]. + +%% Using jsone:decode_stream/2 as opposed to jsone_decode:decode_stream/2. +byte_by_byte_using_jsone(Bin) -> + %% Parse one byte at a time, interleaved with parsing an empty binary + %% between every chunk. + {incomplete, Fun} = + lists:foldl(fun (Byte, {incomplete, Continue1}) -> + {incomplete, Continue2} = Continue1(<>), + Continue2(<<>>) + end, + jsone:decode_stream(<<>>, []), + binary_to_list(Bin)), + Expected = + try + jsone:decode(Bin) + catch + C1:E1 -> + {C1, E1} + end, + Actual = + try + Fun(end_stream) + catch + C2:E2 -> + {C2, E2} + end, + ?assertEqual(Expected, Actual).