Add the chunked option for HTTP/1.1

It allows disabling the chunked transfer-encoding. It
can also be disabled on a per-request basis, although
it will be ignored for responses that are not streamed.

doc/src/guide/migrating_from_2.5.asciidoc

@@ -24,23 +24,26 @@ experimental.
   data in order to compress them. This is the case for
   gzip compression.
 
-* Add an `http10_keepalive` option to allow disabling
+* Add the `chunked` option to allow disabling chunked
+  transfer-encoding for HTTP/1.1 connections.
+
+* Add the `http10_keepalive` option to allow disabling
   keep-alive for HTTP/1.0 connections.
 
-* Add an `idle_timeout` option for HTTP/2.
+* Add the `idle_timeout` option for HTTP/2.
 
-* Add a `sendfile` option to both HTTP/1.1 and HTTP/2.
+* Add the `sendfile` option to both HTTP/1.1 and HTTP/2.
   It allows disabling the sendfile syscall entirely for
   all connections. It is recommended to disable sendfile
   when using VirtualBox shared folders.
 
 * Add the `rate_limited/2` callback to REST handlers.
 
-* Add a `deflate_opts` option to Websocket handlers that
+* Add the `deflate_opts` option to Websocket handlers that
   allows configuring deflate options for the
   permessage-deflate extension.
 
-* Add a `charset` option to `cowboy_static`.
+* Add the `charset` option to `cowboy_static`.
 
 * Add support for the SameSite cookie attribute.
 
@@ -81,8 +84,9 @@ experimental.
   handlers and Websocket handlers. This can be used
   to update options on a per-request basis. Allow
   overriding the `idle_timeout` option for both
-  HTTP/1.1 and Websocket, and the `cowboy_compress_h`
+  HTTP/1.1 and Websocket, the `cowboy_compress_h`
-  options for HTTP/1.1 and HTTP/2.
+  options for HTTP/1.1 and HTTP/2 and the `chunked`
+  option for HTTP/1.1.
 
 === Bugs fixed
 

doc/src/manual/cowboy_http.asciidoc

@@ -17,6 +17,7 @@ as a Ranch protocol.
 [source,erlang]
 ----
 opts() :: #{
+    chunked                 => boolean(),
     connection_type         => worker | supervisor,
     env                     => cowboy_middleware:env(),
     http10_keepalive        => boolean(),
@@ -51,6 +52,13 @@ Ranch functions `ranch:get_protocol_options/1` and
 
 The default value is given next to the option name:
 
+chunked (true)::
+
+Whether chunked transfer-encoding is enabled for HTTP/1.1 connections.
+Note that a response streamed to the client without the chunked
+transfer-encoding and without a content-length header will result
+in the connection being closed at the end of the response body.
+
 connection_type (supervisor)::
 
 Whether the connection process also acts as a supervisor.
@@ -140,7 +148,7 @@ Ordered list of stream handlers that will handle all stream events.
 
 == Changelog
 
-* *2.6*: The `http10_keepalive`, `proxy_header` and `sendfile` options were added.
+* *2.6*: The `chunked`, `http10_keepalive`, `proxy_header` and `sendfile` options were added.
 * *2.5*: The `linger_timeout` option was added.
 * *2.2*: The `max_skip_body_length` option was added.
 * *2.0*: The `timeout` option was renamed `request_timeout`.

src/cowboy_http.erl

@@ -25,6 +25,7 @@
 -export([system_code_change/4]).
 
 -type opts() :: #{
+	chunked => boolean(),
 	compress_buffering => boolean(),
 	compress_threshold => non_neg_integer(),
 	connection_type => worker | supervisor,
@@ -963,21 +964,28 @@ commands(State0=#state{socket=Socket, transport=Transport, out_state=wait, strea
 	end,
 	commands(State, StreamID, Tail);
 %% Send response headers and initiate chunked encoding or streaming.
-commands(State0=#state{socket=Socket, transport=Transport, streams=Streams0, out_state=OutState},
+commands(State0=#state{socket=Socket, transport=Transport,
+		opts=Opts, overriden_opts=Override, streams=Streams0, out_state=OutState},
 		StreamID, [{headers, StatusCode, Headers0}|Tail]) ->
 	%% @todo Same as above (about the last stream in the list).
 	Stream = #stream{version=Version} = lists:keyfind(StreamID, #stream.id, Streams0),
 	Status = cow_http:status_to_integer(StatusCode),
 	ContentLength = maps:get(<<"content-length">>, Headers0, undefined),
+	%% Chunked transfer-encoding can be disabled on a per-request basis.
+	Chunked = case Override of
+		#{chunked := Chunked0} -> Chunked0;
+		_ -> maps:get(chunked, Opts, true)
+	end,
 	{State1, Headers1} = case {Status, ContentLength, Version} of
 		{204, _, 'HTTP/1.1'} ->
 			{State0#state{out_state=done}, Headers0};
 		{304, _, 'HTTP/1.1'} ->
 			{State0#state{out_state=done}, Headers0};
-		{_, undefined, 'HTTP/1.1'} ->
+		{_, undefined, 'HTTP/1.1'} when Chunked ->
 			{State0#state{out_state=chunked}, Headers0#{<<"transfer-encoding">> => <<"chunked">>}};
-		%% Close the connection after streaming without content-length to HTTP/1.0 client.
+		%% Close the connection after streaming without content-length
-		{_, undefined, 'HTTP/1.0'} ->
+		%% to all HTTP/1.0 clients and to HTTP/1.1 clients when chunked is disabled.
+		{_, undefined, _} ->
 			{State0#state{out_state=streaming, last_streamid=StreamID}, Headers0};
 		%% Stream the response body without chunked transfer-encoding.
 		_ ->
@@ -1099,12 +1107,18 @@ commands(State0=#state{ref=Ref, parent=Parent, socket=Socket, transport=Transpor
 %% Set options dynamically.
 commands(State0=#state{overriden_opts=Opts},
 		StreamID, [{set_options, SetOpts}|Tail]) ->
-	State = case SetOpts of
+	State1 = case SetOpts of
 		#{idle_timeout := IdleTimeout} ->
 			set_timeout(State0#state{overriden_opts=Opts#{idle_timeout => IdleTimeout}});
 		_ ->
 			State0
 	end,
+	State = case SetOpts of
+		#{chunked := Chunked} ->
+			State1#state{overriden_opts=Opts#{chunked => Chunked}};
+		_ ->
+			State1
+	end,
 	commands(State, StreamID, Tail);
 %% Stream shutdown.
 commands(State, StreamID, [stop|Tail]) ->

test/handlers/set_options_h.erl

@@ -8,6 +8,19 @@
 init(Req, State) ->
 	set_options(cowboy_req:binding(key, Req), Req, State).
 
+set_options(<<"chunked_false">>, Req0, State) ->
+	%% @todo This should be replaced by a cowboy_req:cast/cowboy_stream:cast.
+	#{pid := Pid, streamid := StreamID} = Req0,
+	Pid ! {{Pid, StreamID}, {set_options, #{chunked => false}}},
+	Req = cowboy_req:stream_reply(200, Req0),
+	cowboy_req:stream_body(<<0:8000000>>, fin, Req),
+	{ok, Req, State};
+set_options(<<"chunked_false_ignored">>, Req0, State) ->
+	%% @todo This should be replaced by a cowboy_req:cast/cowboy_stream:cast.
+	#{pid := Pid, streamid := StreamID} = Req0,
+	Pid ! {{Pid, StreamID}, {set_options, #{chunked => false}}},
+	Req = cowboy_req:reply(200, #{}, <<"Hello world!">>, Req0),
+	{ok, Req, State};
 set_options(<<"idle_timeout_short">>, Req0, State) ->
 	%% @todo This should be replaced by a cowboy_req:cast/cowboy_stream:cast.
 	#{pid := Pid, streamid := StreamID} = Req0,

test/http_SUITE.erl

@@ -24,6 +24,8 @@
 -import(cowboy_test, [raw_open/1]).
 -import(cowboy_test, [raw_send/2]).
 -import(cowboy_test, [raw_recv_head/1]).
+-import(cowboy_test, [raw_recv/3]).
+-import(cowboy_test, [raw_expect_recv/2]).
 
 all() -> [{group, clear}].
 
@@ -33,12 +35,39 @@ init_routes(_) -> [
 	{"localhost", [
 		{"/", hello_h, []},
 		{"/echo/:key", echo_h, []},
+		{"/resp/:key[/:arg]", resp_h, []},
 		{"/set_options/:key", set_options_h, []}
 	]}
 ].
 
+chunked_false(Config) ->
+	doc("Confirm the option chunked => false disables chunked "
+		"transfer-encoding for HTTP/1.1 connections."),
+	{ok, _} = cowboy:start_clear(name(), [{port, 0}], #{
+		env => #{dispatch => cowboy_router:compile(init_routes(Config))},
+		chunked => false
+	}),
+	Port = ranch:get_port(name()),
+	Request = "GET /resp/stream_reply2/200 HTTP/1.1\r\nhost: localhost\r\n\r\n",
+	Client = raw_open([{type, tcp}, {port, Port}, {opts, []}|Config]),
+	ok = raw_send(Client, Request),
+	Rest = case catch raw_recv_head(Client) of
+		{'EXIT', _} -> error(closed);
+		Data ->
+			%% Cowboy always advertises itself as HTTP/1.1.
+			{'HTTP/1.1', 200, _, Rest0} = cow_http:parse_status_line(Data),
+			{Headers, Rest1} = cow_http:parse_headers(Rest0),
+			false = lists:keyfind(<<"content-length">>, 1, Headers),
+			false = lists:keyfind(<<"transfer-encoding">>, 1, Headers),
+			Rest1
+	end,
+	Bits = 8000000 - bit_size(Rest),
+	raw_expect_recv(Client, <<0:Bits>>),
+	{error, closed} = raw_recv(Client, 1, 1000),
+	ok.
+
 http10_keepalive_false(Config) ->
-	doc("Confirm the option {http10_keepalive, false} disables keep-alive "
+	doc("Confirm the option http10_keepalive => false disables keep-alive "
 		"completely for HTTP/1.0 connections."),
 	{ok, _} = cowboy:start_clear(name(), [{port, 0}], #{
 		env => #{dispatch => cowboy_router:compile(init_routes(Config))},
@@ -101,6 +130,53 @@ request_timeout_infinity(Config) ->
 		ok
 	end.
 
+set_options_chunked_false(Config) ->
+	doc("Confirm the option chunked can be dynamically set to disable "
+		"chunked transfer-encoding. This results in the closing of the "
+		"connection after the current request."),
+	{ok, _} = cowboy:start_clear(name(), [{port, 0}], #{
+		env => #{dispatch => cowboy_router:compile(init_routes(Config))},
+		chunked => true
+	}),
+	Port = ranch:get_port(name()),
+	Request = "GET /set_options/chunked_false HTTP/1.1\r\nhost: localhost\r\n\r\n",
+	Client = raw_open([{type, tcp}, {port, Port}, {opts, []}|Config]),
+	ok = raw_send(Client, Request),
+	_ = case catch raw_recv_head(Client) of
+		{'EXIT', _} -> error(closed);
+		Data ->
+			%% Cowboy always advertises itself as HTTP/1.1.
+			{'HTTP/1.1', 200, _, Rest} = cow_http:parse_status_line(Data),
+			{Headers, <<>>} = cow_http:parse_headers(Rest),
+			false = lists:keyfind(<<"content-length">>, 1, Headers),
+			false = lists:keyfind(<<"transfer-encoding">>, 1, Headers)
+	end,
+	raw_expect_recv(Client, <<0:8000000>>),
+	{error, closed} = raw_recv(Client, 1, 1000),
+	ok.
+
+set_options_chunked_false_ignored(Config) ->
+	doc("Confirm the option chunked can be dynamically set to disable "
+		"chunked transfer-encoding, and that it is ignored if the "
+		"response is not streamed."),
+	{ok, _} = cowboy:start_clear(name(), [{port, 0}], #{
+		env => #{dispatch => cowboy_router:compile(init_routes(Config))},
+		chunked => true
+	}),
+	Port = ranch:get_port(name()),
+	ConnPid = gun_open([{type, tcp}, {protocol, http}, {port, Port}|Config]),
+	%% We do a first request setting the option but not
+	%% using chunked transfer-encoding in the response.
+	StreamRef1 = gun:get(ConnPid, "/set_options/chunked_false_ignored"),
+	{response, nofin, 200, _} = gun:await(ConnPid, StreamRef1),
+	{ok, <<"Hello world!">>} = gun:await_body(ConnPid, StreamRef1),
+	%% We then do a second request to confirm that chunked
+	%% is not disabled for that second request.
+	StreamRef2 = gun:get(ConnPid, "/resp/stream_reply2/200"),
+	{response, nofin, 200, Headers} = gun:await(ConnPid, StreamRef2),
+	{_, <<"chunked">>} = lists:keyfind(<<"transfer-encoding">>, 1, Headers),
+	ok.
+
 set_options_idle_timeout(Config) ->
 	doc("Confirm that the idle_timeout option can be dynamically "
 		"set to change how long Cowboy will wait before it closes the connection."),