0
Fork 0
mirror of https://github.com/ninenines/cowboy.git synced 2025-07-14 04:10:24 +00:00

Rework and improve the decompress stream handler

The read buffer was changed into an iovec to avoid doing
too many binary concatenations and allocations.

Decompression happens transparently: when decoding gzip,
the content-encoding header is removed (we only decode
when "gzip" is the only encoding so nothing remains).

We always add a content_decoded key to the Req object.
This key contains a list of codings that were decoded,
in the reverse order in which they were. Currently it
can only be empty or contain <<"gzip">> but future
improvements or user handlers may see it contain more
values.

The option to disable decompression was renamed to
decompress_enabled and defaults to true.

It is no longer possible to enable/disable decompression
in the middle of reading the body: this ensures that the
data we pass forward is always valid.

Various smaller improvements were made to the code,
tests and manual pages.
This commit is contained in:
Loïc Hoguin 2024-01-04 15:15:41 +01:00
parent 3ed1b24dd6
commit fd9711d949
No known key found for this signature in database
GPG key ID: 8A9DF795F6FED764
6 changed files with 361 additions and 203 deletions

View file

@ -66,9 +66,9 @@ enabled by default. It is a good example for writing your
own handlers that will modify responses.
link:man:cowboy_decompress_h(3)[cowboy_decompress_h] will
automatically decompress requests when possible. It is not
enabled by default. It is a good example for writing your
own handlers that will modify requests.
automatically decompress request bodies when possible.
It is not enabled by default. It is a good example for
writing your own handlers that will modify requests.
link:man:cowboy_metrics_h(3)[cowboy_metrics_h] gathers
metrics about a stream then passes them to a configurable

View file

@ -36,6 +36,7 @@ Stream handlers:
* link:man:cowboy_stream_h(3)[cowboy_stream_h(3)] - Default stream handler
* link:man:cowboy_compress_h(3)[cowboy_compress_h(3)] - Compress stream handler
* link:man:cowboy_decompress_h(3)[cowboy_decompress_h(3)] - Decompress stream handler
* link:man:cowboy_metrics_h(3)[cowboy_metrics_h(3)] - Metrics stream handler
* link:man:cowboy_tracer_h(3)[cowboy_tracer_h(3)] - Tracer stream handler

View file

@ -7,18 +7,26 @@ cowboy_decompress_h - Decompress stream handler
== Description
The module `cowboy_decompress_h` decompresses request bodies
automatically when the server supports it. Requests will
only be decompressed when their compression ratio is lower
than the configured limit. Mismatch of the content and
`content-encoding` is rejected with `400 Bad Request`.
automatically when the server supports it.
The only compression algorithm currently supported is the
gzip algorithm. Another limitation is that decompression
is only attempted when gzip is the only content-encoding
in the request.
This stream handler always adds a field to the Req object
with the name `content_decoded` which is treated as a
list of decoded content-encoding values. Currently this
list may only contain the `<<"gzip">>` binary if content
was decoded; or be empty otherwise.
== Options
[source,erlang]
----
opts() :: #{
decompress_ratio_limit => non_neg_integer(),
decompress_ignore => boolean()
decompress_enabled => boolean(),
decompress_ratio_limit => non_neg_integer()
}
----
@ -28,17 +36,21 @@ The default value is given next to the option name:
decompress_ratio_limit (20)::
The max ratio of the compressed and decompressed body
before it is rejected with `413 Payload Too Large`.
before it is rejected with a `413 Payload Too Large`
error response.
+
This option can be updated at any time using the
`set_options` stream handler command.
decompress_ignore (false)::
decompress_enabled (true)::
Whether the handler will be ignored.
Whether the handler is enabled by default.
+
This option can be updated at any time using the
`set_options` stream handler command.
This option can be updated using the `set_options`
stream handler command. This allows disabling
decompression for the current stream. Attempts
to enable or disable decompression after starting
to read the body will be ignored.
== Events

View file

@ -1,3 +1,18 @@
%% Copyright (c) 2024, jdamanalo <joshuadavid.agustin@manalo.ph>
%% Copyright (c) 2024, Loïc Hoguin <essen@ninenines.eu>
%%
%% Permission to use, copy, modify, and/or distribute this software for any
%% purpose with or without fee is hereby granted, provided that the above
%% copyright notice and this permission notice appear in all copies.
%%
%% THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
%% WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
%% MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
%% ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
%% WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
%% ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
%% OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-module(cowboy_decompress_h).
-behavior(cowboy_stream).
@ -9,21 +24,27 @@
-record(state, {
next :: any(),
enabled :: boolean(),
ratio_limit :: non_neg_integer() | undefined,
ignore = false :: boolean(),
compress = undefined :: undefined | gzip,
inflate = undefined :: undefined | zlib:zstream(),
is_reading = false :: boolean(),
read_body_buffer = <<>> :: binary(),
%% We use a list of binaries to avoid doing unnecessary
%% memory allocations when inflating. We convert to binary
%% when we propagate the data. The data must be reversed
%% before converting to binary or inflating: this is done
%% via the buffer_to_binary/buffer_to_iovec functions.
read_body_buffer = [] :: [binary()],
read_body_is_fin = nofin :: nofin | {fin, non_neg_integer()}
}).
-spec init(cowboy_stream:streamid(), cowboy_req:req(), cowboy:opts())
-> {cowboy_stream:commands(), #state{}}.
init(StreamID, Req, Opts) ->
init(StreamID, Req0, Opts) ->
Enabled = maps:get(decompress_enabled, Opts, true),
RatioLimit = maps:get(decompress_ratio_limit, Opts, 20),
Ignore = maps:get(decompress_ignore, Opts, false),
State = check_req(Req),
{Req, State} = check_and_update_req(Req0),
Inflate = case State#state.compress of
undefined ->
undefined;
@ -33,48 +54,46 @@ init(StreamID, Req, Opts) ->
Z
end,
{Commands, Next} = cowboy_stream:init(StreamID, Req, Opts),
fold(Commands, State#state{next=Next, ratio_limit=RatioLimit, ignore=Ignore,
inflate=Inflate}).
fold(Commands, State#state{next=Next, enabled=Enabled,
ratio_limit=RatioLimit, inflate=Inflate}).
-spec data(cowboy_stream:streamid(), cowboy_stream:fin(), cowboy_req:resp_body(), State)
-> {cowboy_stream:commands(), State} when State::#state{}.
data(StreamID, IsFin, Data, State=#state{next=Next0, inflate=undefined}) ->
{Commands, Next} = cowboy_stream:data(StreamID, IsFin, Data, Next0),
fold(Commands, State#state{next=Next, read_body_is_fin=IsFin});
data(StreamID, IsFin, Data, State=#state{next=Next0, ignore=true, read_body_buffer=Buffer}) ->
data(StreamID, IsFin, Data, State=#state{next=Next0, enabled=false, read_body_buffer=Buffer}) ->
{Commands, Next} = cowboy_stream:data(StreamID, IsFin,
<< Buffer/binary, Data/binary >>, Next0),
buffer_to_binary([Data|Buffer]), Next0),
fold(Commands, State#state{next=Next, read_body_is_fin=IsFin});
data(StreamID, IsFin, Data, State0=#state{next=Next0, ratio_limit=RatioLimit,
inflate=Z, is_reading=true, read_body_buffer=Buffer0}) ->
Buffer = << Buffer0/binary, Data/binary >>,
case inflate(Z, RatioLimit, Buffer) of
{error, Type} ->
Status = case Type of
data -> 400;
size -> 413
inflate=Z, is_reading=true, read_body_buffer=Buffer}) ->
case inflate(Z, RatioLimit, buffer_to_iovec([Data|Buffer])) of
{error, ErrorType} ->
zlib:close(Z),
Status = case ErrorType of
data_error -> 400;
size_error -> 413
end,
Commands = [
{error_response, Status, #{<<"content-length">> => <<"0">>}, <<>>},
stop
],
fold(Commands, State0#state{inflate=undefined});
fold(Commands, State0#state{inflate=undefined, read_body_buffer=[]});
{ok, Inflated} ->
State = case IsFin of
nofin ->
State0;
fin ->
zlib:inflateEnd(Z),
zlib:close(Z),
State0#state{inflate=undefined}
end,
{Commands, Next} = cowboy_stream:data(StreamID, IsFin, Inflated, Next0),
fold(Commands, State#state{next=Next, read_body_buffer= <<>>,
fold(Commands, State#state{next=Next, read_body_buffer=[],
read_body_is_fin=IsFin})
end;
data(_, IsFin, Data, State=#state{read_body_buffer=Buffer0}) ->
Buffer = << Buffer0/binary, Data/binary >>,
{[], State#state{read_body_buffer=Buffer, read_body_is_fin=IsFin}}.
data(_, IsFin, Data, State=#state{read_body_buffer=Buffer}) ->
{[], State#state{read_body_buffer=[Data|Buffer], read_body_is_fin=IsFin}}.
-spec info(cowboy_stream:streamid(), any(), State)
-> {cowboy_stream:commands(), State} when State::#state{}.
@ -86,12 +105,19 @@ info(StreamID, Info={CommandTag, _, _, _, _}, State=#state{next=Next0, read_body
{Commands0, Next1} = cowboy_stream:info(StreamID, Info, Next0),
{Commands, Next} = data(StreamID, IsFin, <<>>, State#state{next=Next1, is_reading=true}),
fold(Commands ++ Commands0, Next);
info(StreamID, Info={set_options, Opts}, State=#state{next=Next0,
ignore=Ignore0, ratio_limit=RatioLimit0}) ->
Ignore = maps:get(decompress_ignore, Opts, Ignore0),
info(StreamID, Info={set_options, Opts}, State0=#state{next=Next0,
enabled=Enabled0, ratio_limit=RatioLimit0, is_reading=IsReading}) ->
Enabled = maps:get(decompress_enabled, Opts, Enabled0),
RatioLimit = maps:get(decompress_ratio_limit, Opts, RatioLimit0),
{Commands, Next} = cowboy_stream:info(StreamID, Info, Next0),
fold(Commands, State#state{next=Next, ignore=Ignore, ratio_limit=RatioLimit});
%% We can't change the enabled setting after we start reading,
%% otherwise the data becomes garbage. Changing the setting
%% is not treated as an error, it is just ignored.
State = case IsReading of
true -> State0;
false -> State0#state{enabled=Enabled}
end,
fold(Commands, State#state{next=Next, ratio_limit=RatioLimit});
info(StreamID, Info, State=#state{next=Next0}) ->
{Commands, Next} = cowboy_stream:info(StreamID, Info, Next0),
fold(Commands, State#state{next=Next}).
@ -112,31 +138,49 @@ early_error(StreamID, Reason, PartialReq, Resp, Opts) ->
%% Internal.
check_req(Req) ->
%% Check whether the request needs content decoding, and if it does
%% whether it fits our criteria for decoding. We also update the
%% Req to indicate whether content was decoded.
%%
%% We always set the content_decoded value in the Req because it
%% indicates whether content decoding was attempted.
%%
%% A malformed content-encoding header results in no decoding.
check_and_update_req(Req=#{headers := Headers}) ->
ContentDecoded = maps:get(content_decoded, Req, []),
try cowboy_req:parse_header(<<"content-encoding">>, Req) of
undefined ->
#state{compress=undefined};
Encodings ->
case [E || E=(<<"gzip">>) <- Encodings] of
[] ->
#state{compress=undefined};
_ ->
#state{compress=gzip}
end
catch
_:_ ->
#state{compress=undefined}
%% We only automatically decompress when gzip is the only
%% encoding used. Since it's the only encoding used, we
%% can remove the header entirely before passing the Req
%% forward.
[<<"gzip">>] ->
{Req#{
headers => maps:remove(<<"content-encoding">>, Headers),
content_decoded => [<<"gzip">>|ContentDecoded]
}, #state{compress=gzip}};
_ ->
{Req#{content_decoded => ContentDecoded},
#state{compress=undefined}}
catch _:_ ->
{Req#{content_decoded => ContentDecoded},
#state{compress=undefined}}
end.
buffer_to_iovec(Buffer) ->
lists:reverse(Buffer).
buffer_to_binary(Buffer) ->
iolist_to_binary(lists:reverse(Buffer)).
fold(Commands, State) ->
fold(Commands, State, []).
fold([], State, Acc) ->
{lists:reverse(Acc), State};
fold([{response, Status, Headers0, Body}|Tail], State=#state{ignore=false}, Acc) ->
fold([{response, Status, Headers0, Body}|Tail], State=#state{enabled=true}, Acc) ->
Headers = add_accept_encoding(Headers0),
fold(Tail, State, [{response, Status, Headers, Body}|Acc]);
fold([{headers, Status, Headers0} | Tail], State=#state{ignore=false}, Acc) ->
fold([{headers, Status, Headers0} | Tail], State=#state{enabled=true}, Acc) ->
Headers = add_accept_encoding(Headers0),
fold(Tail, State, [{headers, Status, Headers}|Acc]);
fold([Command|Tail], State, Acc) ->
@ -146,7 +190,7 @@ add_accept_encoding(Headers=#{<<"accept-encoding">> := AcceptEncoding}) ->
try cow_http_hd:parse_accept_encoding(iolist_to_binary(AcceptEncoding)) of
List ->
case lists:keyfind(<<"gzip">>, 1, List) of
%% gzip is excluded but this handler is not ignored; we replace.
%% gzip is excluded but this handler is enabled; we replace.
{_, 0} ->
Replaced = lists:keyreplace(<<"gzip">>, 1, List, {<<"gzip">>, 1000}),
Codings = build_accept_encoding(Replaced),
@ -167,18 +211,20 @@ add_accept_encoding(Headers=#{<<"accept-encoding">> := AcceptEncoding}) ->
end
end
catch _:_ ->
%% The accept-encoding header is invalid. Probably empty. We replace it with ours.
Headers#{<<"accept-encoding">> => <<"gzip">>}
end;
add_accept_encoding(Headers) ->
Headers#{<<"accept-encoding">> => <<"gzip">>}.
%% From cowlib, maybe expose?
%% @todo From cowlib, maybe expose?
qvalue_to_iodata(0) -> <<"0">>;
qvalue_to_iodata(Q) when Q < 10 -> [<<"0.00">>, integer_to_binary(Q)];
qvalue_to_iodata(Q) when Q < 100 -> [<<"0.0">>, integer_to_binary(Q)];
qvalue_to_iodata(Q) when Q < 1000 -> [<<"0.">>, integer_to_binary(Q)];
qvalue_to_iodata(1000) -> <<"1">>.
%% @todo Should be added to Cowlib.
build_accept_encoding([{ContentCoding, Q}|Tail]) ->
Weight = iolist_to_binary(qvalue_to_iodata(Q)),
Acc = <<ContentCoding/binary, ";q=", Weight/binary>>,
@ -195,20 +241,14 @@ inflate(Z, RatioLimit, Data) ->
try
{Status, Output} = zlib:safeInflate(Z, Data),
Size = iolist_size(Output),
do_inflate(Z, Size, byte_size(Data) * RatioLimit, Status, [Output])
do_inflate(Z, Size, iolist_size(Data) * RatioLimit, Status, [Output])
catch
error:data_error ->
zlib:close(Z),
{error, data}
{error, data_error}
end.
do_inflate(Z, Size, Limit, Status, _) when Size > Limit ->
case Status of
continue -> ok;
finished -> zlib:inflateEnd(Z)
end,
zlib:close(Z),
{error, size};
do_inflate(_, Size, Limit, _, _) when Size > Limit ->
{error, size_error};
do_inflate(Z, Size0, Limit, continue, Acc) ->
{Status, Output} = zlib:safeInflate(Z, []),
Size = Size0 + iolist_size(Output),

View file

@ -1,3 +1,17 @@
%% Copyright (c) 2024, jdamanalo <joshuadavid.agustin@manalo.ph>
%%
%% Permission to use, copy, modify, and/or distribute this software for any
%% purpose with or without fee is hereby granted, provided that the above
%% copyright notice and this permission notice appear in all copies.
%%
%% THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
%% WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
%% MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
%% ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
%% WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
%% ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
%% OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-module(decompress_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
@ -50,14 +64,8 @@ init_compress_opts(Config) ->
init_routes(_) ->
[{'_', [
{"/echo/:what", decompress_h, []},
{"/header", decompress_h, header_command},
{"/invalid-header", decompress_h, invalid_header},
{"/accept-identity", decompress_h, accept_identity},
{"/reject-explicit-header", decompress_h, reject_explicit_header},
{"/reject-implicit-header", decompress_h, reject_implicit_header},
{"/accept-explicit-header", decompress_h, accept_explicit_header},
{"/accept-implicit-header", decompress_h, accept_implicit_header}
{"/echo/:what", decompress_h, echo},
{"/test/:what", decompress_h, test}
]}].
%% Internal.
@ -92,49 +100,91 @@ do_create_gzip_bomb(Z, N) ->
%% Tests.
content_encoding_none(Config) ->
doc("Send no content-encoding; get echo."),
doc("Requests without content-encoding are processed normally."),
Body = <<"test">>,
{200, _, Body} = do_post("/echo/normal",
[{<<"content-encoding">>, <<";">>}], Body, Config),
{200, _, Body} = do_post("/echo/normal", [], Body, Config),
%% The content-encoding header would be propagated,
%% but there was no content-encoding header to propagate.
{200, _, <<"undefined">>} = do_post("/test/content-encoding", [], Body, Config),
%% The content_decoded list is empty.
{200, _, <<"[]">>} = do_post("/test/content-decoded", [], Body, Config),
ok.
content_encoding_malformed(Config) ->
doc("Send malformed content-encoding; get echo."),
doc("Requests with a malformed content-encoding are processed "
"as if no content-encoding was sent."),
Body = <<"test">>,
{200, _, Body} = do_post("/echo/normal",
[{<<"content-encoding">>, <<";">>}], Body, Config),
%% The content-encoding header is propagated.
{200, _, <<";">>} = do_post("/test/content-encoding",
[{<<"content-encoding">>, <<";">>}], Body, Config),
%% The content_decoded list is empty.
{200, _, <<"[]">>} = do_post("/test/content-decoded",
[{<<"content-encoding">>, <<";">>}], Body, Config),
ok.
content_encoding_not_supported(Config) ->
doc("Send content-encoding: compress (unsupported by Cowboy); get echo."),
doc("Requests with an unsupported content-encoding are processed "
"as if no content-encoding was sent."),
Body = <<"test">>,
{200, _, Body} = do_post("/echo/normal",
[{<<"content-encoding">>, <<"compress">>}], Body, Config),
%% The content-encoding header is propagated.
{200, _, <<"compress">>} = do_post("/test/content-encoding",
[{<<"content-encoding">>, <<"compress">>}], Body, Config),
%% The content_decoded list is empty.
{200, _, <<"[]">>} = do_post("/test/content-decoded",
[{<<"content-encoding">>, <<"compress">>}], Body, Config),
ok.
content_encoding_wrong(Config) ->
doc("Send content-encoding and unencoded body; get 400."),
content_encoding_multiple(Config) ->
doc("Requests with multiple content-encoding values are processed "
"as if no content-encoding was sent."),
Body = <<"test">>,
{200, _, Body} = do_post("/echo/normal",
[{<<"content-encoding">>, <<"gzip, compress">>}], Body, Config),
%% The content-encoding header is propagated.
{200, _, <<"gzip, compress">>} = do_post("/test/content-encoding",
[{<<"content-encoding">>, <<"gzip, compress">>}], Body, Config),
%% The content_decoded list is empty.
{200, _, <<"[]">>} = do_post("/test/content-decoded",
[{<<"content-encoding">>, <<"gzip, compress">>}], Body, Config),
ok.
decompress(Config) ->
doc("Requests with content-encoding set to gzip and gzipped data "
"are transparently decompressed."),
Data = <<"test">>,
Body = zlib:gzip(Data),
{200, _, Data} = do_post("/echo/normal",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
%% The content-encoding header is NOT propagated.
{200, _, <<"undefined">>} = do_post("/test/content-encoding",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
%% The content_decoded list contains <<"gzip">>.
{200, _, <<"[<<\"gzip\">>]">>} = do_post("/test/content-decoded",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
ok.
decompress_error(Config) ->
doc("Requests with content-encoding set to gzip but the data "
"cannot be decoded are rejected with a 400 Bad Request error."),
Body = <<"test">>,
{400, _, _} = do_post("/echo/normal",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
ok.
decompress(Config) ->
doc("Send content-encoding and encoded body; get decompressed response."),
Data = <<"test">>,
Body = zlib:gzip(Data),
{200, _, Data} = do_post("/echo/normal",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
ok.
decompress_stream(Config) ->
doc("Stream encoded body; get decompressed response."),
doc("Requests with content-encoding set to gzip and gzipped data "
"are transparently decompressed, even when the data is streamed."),
%% Handler read length 1KB. Compressing 3KB should be enough to trigger more.
Data = crypto:strong_rand_bytes(3000),
Body = zlib:gzip(Data),
Size = byte_size(Body),
ConnPid = gun_open(Config),
Ref = gun:post(ConnPid, "/echo/normal", [{<<"content-encoding">>, <<"gzip">>}]),
Ref = gun:post(ConnPid, "/echo/normal",
[{<<"content-encoding">>, <<"gzip">>}]),
gun:data(ConnPid, Ref, nofin, binary:part(Body, 0, Size div 2)),
timer:sleep(1000),
gun:data(ConnPid, Ref, fin, binary:part(Body, Size div 2, Size div 2 + Size rem 2)),
@ -144,12 +194,23 @@ decompress_stream(Config) ->
fin -> {ok, <<>>}
end,
gun:close(ConnPid),
{200, _, Data} = do_post("/echo/normal",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
ok.
%% The content-encoding header is NOT propagated.
ConnPid2 = gun_open(Config),
Ref2 = gun:post(ConnPid2, "/test/content-encoding",
[{<<"content-encoding">>, <<"gzip">>}]),
{response, nofin, 200, _} = gun:await(ConnPid2, Ref2),
{ok, <<"undefined">>} = gun:await_body(ConnPid2, Ref2),
gun:close(ConnPid2),
%% The content_decoded list contains <<"gzip">>.
ConnPid3 = gun_open(Config),
Ref3 = gun:post(ConnPid3, "/test/content-decoded",
[{<<"content-encoding">>, <<"gzip">>}]),
{response, nofin, 200, _} = gun:await(ConnPid3, Ref3),
{ok, <<"[<<\"gzip\">>]">>} = gun:await_body(ConnPid3, Ref3),
gun:close(ConnPid3).
opts_decompress_ignore(Config0) ->
doc("Confirm that the decompress_ignore option can be set."),
opts_decompress_enabled_false(Config0) ->
doc("Confirm that the decompress_enabled option can be set."),
Fun = case config(ref, Config0) of
HTTPS when HTTPS =:= https_compress; HTTPS =:= https -> init_https;
H2 when H2 =:= h2_compress; H2 =:= h2 -> init_http2;
@ -158,28 +219,75 @@ opts_decompress_ignore(Config0) ->
Config = cowboy_test:Fun(?FUNCTION_NAME, #{
env => #{dispatch => cowboy_router:compile(init_routes(Config0))},
stream_handlers => [cowboy_decompress_h, cowboy_stream_h],
decompress_ignore => true
decompress_enabled => false
}, Config0),
Data = <<"test">>,
Body = zlib:gzip(Data),
try
{200, _, Body} = do_post("/echo/normal",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config)
{200, Headers, Body} = do_post("/echo/normal",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
%% We do not set accept-encoding when we are disabled.
false = lists:keyfind(<<"accept-encoding">>, 1, Headers)
after
cowboy:stop_listener(?FUNCTION_NAME)
end.
set_options_decompress_ignore(Config) ->
doc("Confirm that the decompress_ignore option can be dynamically
set to true and the data received is not decompressed."),
set_options_decompress_enabled_false(Config) ->
doc("Confirm that the decompress_enabled option can be dynamically "
"set to false and the data received is not decompressed."),
Data = <<"test">>,
Body = zlib:gzip(Data),
{200, _, Body} = do_post("/echo/decompress_ignore",
{200, Headers, Body} = do_post("/echo/decompress_disable",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
%% We do not set accept-encoding when we are disabled.
false = lists:keyfind(<<"accept-encoding">>, 1, Headers),
ok.
set_options_decompress_disable_in_the_middle(Config) ->
doc("Confirm that setting the decompress_enabled option dynamically "
"to false after starting to read the body does not disable decompression "
"and the data received is decompressed."),
Data = rand:bytes(1000000),
Body = zlib:gzip(Data),
%% Since we were not ignoring before starting to read,
%% we receive the entire body decompressed.
{200, Headers, Data} = do_post("/test/disable-in-the-middle",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
%% We do set accept-encoding when we are enabled,
%% even if an attempt to disable in the middle is ignored.
{_, _} = lists:keyfind(<<"accept-encoding">>, 1, Headers),
ok.
set_options_decompress_enable_in_the_middle(Config0) ->
doc("Confirm that setting the decompress_enabled option dynamically "
"to true after starting to read the body does not enable decompression "
"and the data received is not decompressed."),
Fun = case config(ref, Config0) of
HTTPS when HTTPS =:= https_compress; HTTPS =:= https -> init_https;
H2 when H2 =:= h2_compress; H2 =:= h2 -> init_http2;
_ -> init_http
end,
Config = cowboy_test:Fun(?FUNCTION_NAME, #{
env => #{dispatch => cowboy_router:compile(init_routes(Config0))},
stream_handlers => [cowboy_decompress_h, cowboy_stream_h],
decompress_enabled => false
}, Config0),
Data = rand:bytes(1000000),
Body = zlib:gzip(Data),
try
%% Since we were ignoring before starting to read,
%% we receive the entire body compressed.
{200, Headers, Body} = do_post("/test/enable-in-the-middle",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
%% We do not set accept-encoding when we are disabled,
%% even if an attempt to enable in the middle is ignored.
false = lists:keyfind(<<"accept-encoding">>, 1, Headers)
after
cowboy:stop_listener(?FUNCTION_NAME)
end.
opts_decompress_ratio_limit(Config0) ->
doc("Confirm that the decompress_ignore option can be set"),
doc("Confirm that the decompress_ratio_limit option can be set."),
Fun = case config(ref, Config0) of
HTTPS when HTTPS =:= https_compress; HTTPS =:= https -> init_https;
H2 when H2 =:= h2_compress; H2 =:= h2 -> init_http2;
@ -190,7 +298,8 @@ opts_decompress_ratio_limit(Config0) ->
stream_handlers => [cowboy_decompress_h, cowboy_stream_h],
decompress_ratio_limit => 1
}, Config0),
%% Data must be big enough for compression to be effective, so that ratio_limit=1 will fail.
%% Data must be big enough for compression to be effective,
%% so that ratio_limit=1 will fail.
Data = <<0:800>>,
Body = zlib:gzip(Data),
try
@ -202,7 +311,8 @@ opts_decompress_ratio_limit(Config0) ->
set_options_decompress_ratio_limit(Config) ->
doc("Confirm that the decompress_ratio_limit option can be dynamically set."),
%% Data must be big enough for compression to be effective, so that ratio_limit=1 will fail.
%% Data must be big enough for compression to be effective,
%% so that ratio_limit=1 will fail.
Data = <<0:800>>,
Body = zlib:gzip(Data),
{413, _, _} = do_post("/echo/decompress_ratio_limit",
@ -210,19 +320,16 @@ set_options_decompress_ratio_limit(Config) ->
ok.
gzip_bomb(Config) ->
doc("Send body compressed with suspiciously large ratio; get 413."),
doc("Confirm that requests are rejected with a 413 Payload Too Large "
"error when the ratio limit is exceeded."),
Body = create_gzip_bomb(),
{413, _, _} = do_post("/echo/normal",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
ok.
%% RFC 9110. Section 12.5.3. 3. When sent by a server in a response,
%% Accept-Encoding provides information about which content codings are
%% preferred in the content of a subsequent request to the same resource.
%%
%% Set or add gzip
set_accept_encoding_response(Config) ->
doc("Header accept-encoding must be set on valid response command."),
doc("Header accept-encoding must be set on valid response command. "
"(RFC9110 12.5.3)"),
Data = <<"test">>,
Body = zlib:gzip(Data),
{200, Headers, Data} = do_post("/echo/normal",
@ -231,97 +338,79 @@ set_accept_encoding_response(Config) ->
ok.
set_accept_encoding_header(Config) ->
doc("Header accept-encoding must be set on valid header command."),
doc("Header accept-encoding must be set on valid header command. "
"(RFC9110 12.5.3)"),
Data = <<"test">>,
Body = zlib:gzip(Data),
{200, Headers, Data} = do_post("/header",
{200, Headers, Data} = do_post("/test/header-command",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
{_, <<"gzip">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers),
ok.
add_accept_encoding_header_valid(Config) ->
doc("Header accept-encoding must be added on valid accept-encoding."),
doc("Supported content codings must be added to the accept-encoding "
"header if it already exists. (RFC9110 12.5.3)"),
Data = <<"test">>,
Body = zlib:gzip(Data),
{200, Headers, Data} = do_post("/accept-identity",
{200, Headers, Data} = do_post("/test/accept-identity",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
{_, <<"identity, gzip">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers),
ok.
override_accept_encoding_header_invalid(Config) ->
doc("Header accept-encoding must override invalid accept-encoding."),
doc("When the stream handler cannot parse the accept-encoding header "
"found in the response, it overrides it."),
Data = <<"test">>,
Body = zlib:gzip(Data),
{200, Headers, Data} = do_post("/invalid-header",
{200, Headers, Data} = do_post("/test/invalid-header",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
{_, <<"gzip">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers),
ok.
%% RFC 9110. Section 12.5.3. 10.3. If the representation's content coding is
%% one of the content codings listed in the Accept-Encoding field value, then
%% it is acceptable unless it is accompanied by a qvalue of 0.
%%
%% gzip must not have a qvalue of 0 when the handler is used. Set to 1.
override_accept_encoding_excluded(Config) ->
doc("Header accept-encoding must override when explicitly excluded."),
doc("The stream handler must ensure that the content encodings "
"it supports are not marked as unsupported in response headers. "
"The stream handler enables gzip when explicitly excluded. "
"(RFC9110 12.5.3)"),
Data = <<"test">>,
Body = zlib:gzip(Data),
{200, Headers, Data} = do_post("/reject-explicit-header",
{200, Headers, Data} = do_post("/test/reject-explicit-header",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
{_, <<"identity;q=1, gzip;q=1">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers),
ok.
%% RFC 9110. Section 12.5.3. 10.2. If the representation has no content coding,
%% then it is acceptable by default unless specifically excluded by the
%% Accept-Encoding field stating either "identity;q=0" or "*;q=0" wihout a more
%% specific entry for "identity".
%%
%% *;q=0 will reject codings that are not listed. Specific entry gzip must
%% always be listed when the handler is used. Add gzip.
%% *;q=0 will reject codings that are not listed. Supported codings
%% must always be enabled when the handler is used.
add_accept_encoding_excluded(Config) ->
doc("Header accept-encoding must added when implicitly excluded."),
doc("The stream handler must ensure that the content encodings "
"it supports are not marked as unsupported in response headers. "
"The stream handler enables gzip when implicitly excluded (*;q=0). "
"(RFC9110 12.5.3)"),
Data = <<"test">>,
Body = zlib:gzip(Data),
{200, Headers, Data} = do_post("/reject-implicit-header",
{200, Headers, Data} = do_post("/test/reject-implicit-header",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
{_, <<"gzip;q=1, identity;q=1, *;q=0">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers),
ok.
no_override_accept_coding_set_explicit(Config) ->
doc("Confirm that accept-encoding is not overridden when explicitly set."),
doc("Confirm that accept-encoding is not overridden when the "
"content encodings it supports are explicitly set. "
"(RFC9110 12.5.3)"),
Data = <<"test">>,
Body = zlib:gzip(Data),
{200, Headers, Data} = do_post("/accept-explicit-header",
{200, Headers, Data} = do_post("/test/accept-explicit-header",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
{_, <<"identity, gzip;q=0.5">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers),
ok.
no_override_accept_coding_set_implicit(Config) ->
doc("Confirm that accept-encoding is not overridden when implicitly set."),
doc("Confirm that accept-encoding is not overridden when the "
"content encodings it supports are implicitly set. "
"(RFC9110 12.5.3)"),
Data = <<"test">>,
Body = zlib:gzip(Data),
{200, Headers, Data} = do_post("/accept-implicit-header",
{200, Headers, Data} = do_post("/test/accept-implicit-header",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
{_, <<"identity, *;q=0.5">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers),
ok.
%% RFC 9110. Section 12.5.3. 10.1. If no Accept-Encoding header field is in the
%% request, any content coding is considered acceptable by the user agent.
%%
%% Don't add anything on error or when that handler is not used.
no_set_accept_encoding(Config) ->
doc("No header accept-encoding on invalid responses."),
Body = <<"test">>,
{400, Headers, _} = do_post("/echo/normal",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
false = lists:keyfind(<<"accept-encoding">>, 1, Headers),
ok.
no_set_accept_encoding_ignore(Config) ->
doc("Confirm that no accept-encoding is set when stream is ignored."),
Data = <<"test">>,
Body = zlib:gzip(Data),
{200, Headers, Body} = do_post("/echo/decompress_ignore",
[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
false = lists:keyfind(<<"accept-encoding">>, 1, Headers),
ok.

View file

@ -5,10 +5,10 @@
-export([init/2]).
init(Req0, State=[]) ->
init(Req0, State=echo) ->
case cowboy_req:binding(what, Req0) of
<<"decompress_ignore">> ->
cowboy_req:cast({set_options, #{decompress_ignore => true}}, Req0);
<<"decompress_disable">> ->
cowboy_req:cast({set_options, #{decompress_enabled => false}}, Req0);
<<"decompress_ratio_limit">> ->
cowboy_req:cast({set_options, #{decompress_ratio_limit => 0.5}}, Req0);
<<"normal">> -> ok
@ -16,47 +16,63 @@ init(Req0, State=[]) ->
{ok, Body, Req1} = read_body(Req0),
Req = cowboy_req:reply(200, #{}, Body, Req1),
{ok, Req, State};
init(Req0, State=header_command) ->
{ok, Body, Req1} = read_body(Req0),
Req2 = cowboy_req:stream_reply(200, #{}, Req1),
Req = cowboy_req:stream_body(Body, fin, Req2),
{ok, Req, State};
init(Req0, State=accept_identity) ->
{ok, Body, Req1} = read_body(Req0),
Req = cowboy_req:reply(200, #{<<"accept-encoding">> => <<"identity">>}, Body, Req1),
{ok, Req, State};
init(Req0, State=invalid_header) ->
{ok, Body, Req1} = read_body(Req0),
Req = cowboy_req:reply(200, #{<<"accept-encoding">> => <<";">>}, Body, Req1),
{ok, Req, State};
init(Req0, State=reject_explicit_header) ->
{ok, Body, Req1} = read_body(Req0),
Req = cowboy_req:reply(200, #{<<"accept-encoding">> => <<"identity, gzip;q=0">>},
Body, Req1),
{ok, Req, State};
init(Req0, State=reject_implicit_header) ->
{ok, Body, Req1} = read_body(Req0),
Req = cowboy_req:reply(200, #{<<"accept-encoding">> => <<"identity, *;q=0">>},
Body, Req1),
{ok, Req, State};
init(Req0, State=accept_explicit_header) ->
{ok, Body, Req1} = read_body(Req0),
Req = cowboy_req:reply(200, #{<<"accept-encoding">> => <<"identity, gzip;q=0.5">>},
Body, Req1),
{ok, Req, State};
init(Req0, State=accept_implicit_header) ->
{ok, Body, Req1} = read_body(Req0),
Req = cowboy_req:reply(200, #{<<"accept-encoding">> => <<"identity, *;q=0.5">>},
Body, Req1),
init(Req0, State=test) ->
Req = test(Req0, cowboy_req:binding(what, Req0)),
{ok, Req, State}.
test(Req, <<"content-encoding">>) ->
cowboy_req:reply(200, #{},
cowboy_req:header(<<"content-encoding">>, Req, <<"undefined">>),
Req);
test(Req, <<"content-decoded">>) ->
cowboy_req:reply(200, #{},
io_lib:format("~0p", [maps:get(content_decoded, Req, undefined)]),
Req);
test(Req0, <<"disable-in-the-middle">>) ->
{Status, Data, Req1} = cowboy_req:read_body(Req0, #{length => 1000}),
cowboy_req:cast({set_options, #{decompress_enabled => false}}, Req1),
{ok, Body, Req} = do_read_body(Status, Req1, Data),
cowboy_req:reply(200, #{}, Body, Req);
test(Req0, <<"enable-in-the-middle">>) ->
{Status, Data, Req1} = cowboy_req:read_body(Req0, #{length => 1000}),
cowboy_req:cast({set_options, #{decompress_enabled => true}}, Req1),
{ok, Body, Req} = do_read_body(Status, Req1, Data),
cowboy_req:reply(200, #{}, Body, Req);
test(Req0, <<"header-command">>) ->
{ok, Body, Req1} = read_body(Req0),
Req = cowboy_req:stream_reply(200, #{}, Req1),
cowboy_req:stream_body(Body, fin, Req);
test(Req0, <<"accept-identity">>) ->
{ok, Body, Req} = read_body(Req0),
cowboy_req:reply(200,
#{<<"accept-encoding">> => <<"identity">>},
Body, Req);
test(Req0, <<"invalid-header">>) ->
{ok, Body, Req} = read_body(Req0),
cowboy_req:reply(200,
#{<<"accept-encoding">> => <<";">>},
Body, Req);
test(Req0, <<"reject-explicit-header">>) ->
{ok, Body, Req} = read_body(Req0),
cowboy_req:reply(200,
#{<<"accept-encoding">> => <<"identity, gzip;q=0">>},
Body, Req);
test(Req0, <<"reject-implicit-header">>) ->
{ok, Body, Req} = read_body(Req0),
cowboy_req:reply(200,
#{<<"accept-encoding">> => <<"identity, *;q=0">>},
Body, Req);
test(Req0, <<"accept-explicit-header">>) ->
{ok, Body, Req} = read_body(Req0),
cowboy_req:reply(200,
#{<<"accept-encoding">> => <<"identity, gzip;q=0.5">>},
Body, Req);
test(Req0, <<"accept-implicit-header">>) ->
{ok, Body, Req} = read_body(Req0),
cowboy_req:reply(200,
#{<<"accept-encoding">> => <<"identity, *;q=0.5">>},
Body, Req).
read_body(Req0) ->
{Status, Data, Req} = cowboy_req:read_body(Req0, #{length => 1000}),
do_read_body(Status, Req, Data).