From be8a573fe6898ce9b07e4ef9535e07d8e76a6518 Mon Sep 17 00:00:00 2001 From: Oneric Date: Fri, 10 Oct 2025 00:00:00 +0000 Subject: [PATCH 1/3] fix: stop decompressing response on first unknown codec If multiple Content-Encodings were applied restoring the original message requires undoing all steps in reverse order. Thus we cannot continue when encountering an unsupported codec. Just skipping over one step will most likely just lead to decoding errors in the next supported step or incorrect results otherwise. --- lib/tesla/middleware/compression.ex | 26 +++++++++++----------- test/tesla/middleware/compression_test.exs | 10 +++++++++ 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/lib/tesla/middleware/compression.ex b/lib/tesla/middleware/compression.ex index 93fe3a2c..b68e8654 100644 --- a/lib/tesla/middleware/compression.ex +++ b/lib/tesla/middleware/compression.ex @@ -69,7 +69,7 @@ defmodule Tesla.Middleware.Compression do def decompress(env) do codecs = compression_algorithms(Tesla.get_header(env, "content-encoding")) - {decompressed_body, unknown_codecs} = decompress_body(codecs, env.body, []) + {decompressed_body, unknown_codecs} = decompress_body(codecs, env.body) env |> put_decompressed_body(decompressed_body) @@ -84,28 +84,28 @@ defmodule Tesla.Middleware.Compression do Tesla.put_header(env, "content-encoding", Enum.join(unknown_codecs, ", ")) end - defp decompress_body(_codecs, "" = body, acc) do - {body, acc} + defp decompress_body(_codecs, "" = body) do + {body, []} end - defp decompress_body([gzip | rest], body, acc) when gzip in ["gzip", "x-gzip"] do - decompress_body(rest, :zlib.gunzip(body), acc) + defp decompress_body([gzip | rest], body) when gzip in ["gzip", "x-gzip"] do + decompress_body(rest, :zlib.gunzip(body)) end - defp decompress_body(["deflate" | rest], body, acc) do - decompress_body(rest, :zlib.unzip(body), acc) + defp decompress_body(["deflate" | rest], body) do + decompress_body(rest, :zlib.unzip(body)) end - defp decompress_body(["identity" | rest], body, acc) do - decompress_body(rest, body, acc) + defp decompress_body(["identity" | rest], body) do + decompress_body(rest, body) end - defp decompress_body([codec | rest], body, acc) do - decompress_body(rest, body, [codec | acc]) + defp decompress_body([codec | rest], body) do + {body, Enum.reverse([codec | rest])} end - defp decompress_body([], body, acc) do - {body, acc} + defp decompress_body([], body) do + {body, []} end defp compression_algorithms(nil) do diff --git a/test/tesla/middleware/compression_test.exs b/test/tesla/middleware/compression_test.exs index 44d2a739..be8b3624 100644 --- a/test/tesla/middleware/compression_test.exs +++ b/test/tesla/middleware/compression_test.exs @@ -59,6 +59,10 @@ defmodule Tesla.Middleware.CompressionTest do {200, [{"content-type", "text/plain"}, {"content-encoding", "deflate"}], :zlib.zip("decompressed deflate")} + "/multiple-encodings" -> + {200, [{"content-type", "text/plain"}, {"content-encoding", "gzip, zstd, gzip"}], + :zlib.gzip("decompressed gzip")} + "/response-identity" -> {200, [{"content-type", "text/plain"}, {"content-encoding", "identity"}], "unchanged"} @@ -81,6 +85,12 @@ defmodule Tesla.Middleware.CompressionTest do assert env.body == "decompressed deflate" end + test "stops decompressing on first unsupported content-encoding" do + assert {:ok, env} = CompressionResponseClient.get("/multiple-encodings") + assert env.body == "decompressed gzip" + assert env.headers == [{"content-type", "text/plain"}, {"content-encoding", "gzip, zstd"}] + end + test "return unchanged response for unsupported content-encoding" do assert {:ok, env} = CompressionResponseClient.get("/response-identity") assert env.body == "unchanged" From da159240826a3bf2b1e25dcb9a33ebd7c1ffc508 Mon Sep 17 00:00:00 2001 From: Oneric Date: Sat, 11 Oct 2025 00:00:00 +0000 Subject: [PATCH 2/3] fix: keep original content length and encoding headers for HEAD requests HEAD requests can be used to check the size of remote content to decide ahead of time whether it is worth fetching. Of course the size after decompression likely differs from the transfer size indicated in the content-length header, but depending on use case only the transfer size might be relevant. This obsoletes the empty-body special case in decompress_body previously added in 5bc9b82823b3238257619ea3d67f0985a3707d2b since HEAD requests are now handled earlier. If we get an invalid empty body in a non-HEAD request we want to fail loudly. --- lib/tesla/middleware/compression.ex | 9 ++++---- test/tesla/middleware/compression_test.exs | 25 +++++++++++++++++++--- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/lib/tesla/middleware/compression.ex b/lib/tesla/middleware/compression.ex index b68e8654..f49dadca 100644 --- a/lib/tesla/middleware/compression.ex +++ b/lib/tesla/middleware/compression.ex @@ -67,6 +67,11 @@ defmodule Tesla.Middleware.Compression do def decompress({:ok, env}), do: {:ok, decompress(env)} def decompress({:error, reason}), do: {:error, reason} + # HEAD requests may be used to obtain information on the transfer size and properties + # and their empty bodies are not actually valid for the possibly indicated encodings + # thus we want to preserve them unchanged. + def decompress(%Tesla.Env{method: :head} = env), do: env + def decompress(env) do codecs = compression_algorithms(Tesla.get_header(env, "content-encoding")) {decompressed_body, unknown_codecs} = decompress_body(codecs, env.body) @@ -84,10 +89,6 @@ defmodule Tesla.Middleware.Compression do Tesla.put_header(env, "content-encoding", Enum.join(unknown_codecs, ", ")) end - defp decompress_body(_codecs, "" = body) do - {body, []} - end - defp decompress_body([gzip | rest], body) when gzip in ["gzip", "x-gzip"] do decompress_body(rest, :zlib.gunzip(body)) end diff --git a/test/tesla/middleware/compression_test.exs b/test/tesla/middleware/compression_test.exs index be8b3624..9e3f7260 100644 --- a/test/tesla/middleware/compression_test.exs +++ b/test/tesla/middleware/compression_test.exs @@ -68,6 +68,14 @@ defmodule Tesla.Middleware.CompressionTest do "/response-empty" -> {200, [{"content-type", "text/plain"}, {"content-encoding", "gzip"}], ""} + + "/response-empty-with-content-length" -> + {200, + [ + {"content-type", "text/plain"}, + {"content-encoding", "gzip"}, + {"content-length", "4194304"} + ], ""} end {:ok, %{env | status: status, headers: headers, body: body}} @@ -97,10 +105,21 @@ defmodule Tesla.Middleware.CompressionTest do assert env.headers == [{"content-type", "text/plain"}] end - test "return unchanged response for empty body (gzip)" do - assert {:ok, env} = CompressionResponseClient.get("/response-empty") + test "raises on invalid empty-body response (gzip)" do + assert_raise(ErlangError, "Erlang error: :data_error", fn -> + CompressionResponseClient.get("/response-empty") + end) + end + + test "preserves compression headers for HEAD requests" do + assert {:ok, env} = CompressionResponseClient.head("/response-empty-with-content-length") assert env.body == "" - assert env.headers == [{"content-type", "text/plain"}] + + assert env.headers == [ + {"content-type", "text/plain"}, + {"content-encoding", "gzip"}, + {"content-length", "4194304"} + ] end defmodule CompressRequestDecompressResponseClient do From 3fda9abb6c763da512471c471b482a0138447425 Mon Sep 17 00:00:00 2001 From: Oneric Date: Sat, 11 Oct 2025 00:00:00 +0000 Subject: [PATCH 3/3] fix: update existing content-length header after decompression Depending on context presence of this header is mandatory or at least strongly encouraged in HTTP/1.0 and HTTP/1.1 and some later processing steps might rely on or profit from its presence --- lib/tesla/middleware/compression.ex | 22 +++++++++++++++++++++- test/tesla/middleware/compression_test.exs | 21 +++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/lib/tesla/middleware/compression.ex b/lib/tesla/middleware/compression.ex index f49dadca..ee5c819c 100644 --- a/lib/tesla/middleware/compression.ex +++ b/lib/tesla/middleware/compression.ex @@ -124,7 +124,27 @@ defmodule Tesla.Middleware.Compression do defp put_decompressed_body(env, body) do env |> Tesla.put_body(body) - |> Tesla.delete_header("content-length") + |> update_content_length(body) + end + + # The value of the content-length header wil be inaccurate after decompression. + # But setting it is mandatory or strongly encouraged in HTTP/1.0 and HTTP/1.1. + # Except, when transfer-encoding is used defining content-length is invalid. + # Thus we can neither just drop it nor indiscriminately add it, but will update it if it already exist. + # Furthermore, content-length is technically allowed to be specified mutliple times if all values match, + # to ensure consistency we must therefore make sure to drop any duplicate definitions while updating. + defp update_content_length(env, body) when is_binary(body) do + if Tesla.get_header(env, "content-length") != nil do + env + |> Tesla.delete_header("content-length") + |> Tesla.put_header("content-length", "#{byte_size(body)}") + else + env + end + end + + defp update_content_length(env, _) do + env end end diff --git a/test/tesla/middleware/compression_test.exs b/test/tesla/middleware/compression_test.exs index 9e3f7260..9b0f5be7 100644 --- a/test/tesla/middleware/compression_test.exs +++ b/test/tesla/middleware/compression_test.exs @@ -69,6 +69,16 @@ defmodule Tesla.Middleware.CompressionTest do "/response-empty" -> {200, [{"content-type", "text/plain"}, {"content-encoding", "gzip"}], ""} + "/response-with-content-length" -> + body = :zlib.gzip("decompressed gzip") + + {200, + [ + {"content-type", "text/plain"}, + {"content-encoding", "gzip"}, + {"content-length", "#{byte_size(body)}"} + ], body} + "/response-empty-with-content-length" -> {200, [ @@ -111,6 +121,17 @@ defmodule Tesla.Middleware.CompressionTest do end) end + test "updates existing content-length header" do + expected_body = "decompressed gzip" + assert {:ok, env} = CompressionResponseClient.get("/response-with-content-length") + assert env.body == expected_body + + assert env.headers == [ + {"content-type", "text/plain"}, + {"content-length", "#{byte_size(expected_body)}"} + ] + end + test "preserves compression headers for HEAD requests" do assert {:ok, env} = CompressionResponseClient.head("/response-empty-with-content-length") assert env.body == ""