diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d463f7..30f83a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,7 @@ jobs: - "feat.: blocking" - "feat.: gzip" - "feat.: brotli" + - "feat.: deflate" - "feat.: json" - "feat.: multipart" - "feat.: stream" @@ -89,21 +90,21 @@ jobs: - name: windows / stable-x86_64-msvc os: windows-latest target: x86_64-pc-windows-msvc - features: "--features blocking,gzip,brotli,json,multipart" + features: "--features blocking,gzip,brotli,deflate,json,multipart" - name: windows / stable-i686-msvc os: windows-latest target: i686-pc-windows-msvc - features: "--features blocking,gzip,brotli,json,multipart" + features: "--features blocking,gzip,brotli,deflate,json,multipart" - name: windows / stable-x86_64-gnu os: windows-latest rust: stable-x86_64-pc-windows-gnu target: x86_64-pc-windows-gnu - features: "--features blocking,gzip,brotli,json,multipart" + features: "--features blocking,gzip,brotli,deflate,json,multipart" - name: windows / stable-i686-gnu os: windows-latest rust: stable-i686-pc-windows-gnu target: i686-pc-windows-gnu - features: "--features blocking,gzip,brotli,json,multipart" + features: "--features blocking,gzip,brotli,deflate,json,multipart" - name: "feat.: default-tls disabled" features: "--no-default-features" @@ -123,6 +124,8 @@ jobs: features: "--features gzip" - name: "feat.: brotli" features: "--features brotli" + - name: "feat.: deflate" + features: "--features deflate" - name: "feat.: json" features: "--features json" - name: "feat.: multipart" diff --git a/Cargo.toml b/Cargo.toml index 50c4f38..c50df9f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,6 +49,8 @@ gzip = ["async-compression", "async-compression/gzip", "tokio-util"] brotli = ["async-compression", "async-compression/brotli", "tokio-util"] +deflate = ["async-compression", "async-compression/deflate", "tokio-util"] + json = ["serde_json"] multipart = ["mime_guess"] @@ -216,6 +218,11 @@ name = "brotli" path = "tests/brotli.rs" required-features = ["brotli"] +[[test]] +name = "deflate" +path = "tests/deflate.rs" +required-features = ["deflate"] + [[test]] name = "multipart" path = "tests/multipart.rs" diff --git a/src/async_impl/client.rs b/src/async_impl/client.rs index 4bdad5d..04830a6 100644 --- a/src/async_impl/client.rs +++ b/src/async_impl/client.rs @@ -536,6 +536,29 @@ impl ClientBuilder { self } + /// Enable auto deflate decompression by checking the `Content-Encoding` response header. + /// + /// If auto deflate decompression is turned on: + /// + /// - When sending a request and if the request's headers do not already contain + /// an `Accept-Encoding` **and** `Range` values, the `Accept-Encoding` header is set to `deflate`. + /// The request body is **not** automatically compressed. + /// - When receiving a response, if it's headers contain a `Content-Encoding` value that + /// equals to `deflate`, both values `Content-Encoding` and `Content-Length` are removed from the + /// headers' set. The response body is automatically decompressed. + /// + /// If the `deflate` feature is turned on, the default option is enabled. + /// + /// # Optional + /// + /// This requires the optional `deflate` feature to be enabled + #[cfg(feature = "deflate")] + #[cfg_attr(docsrs, doc(cfg(feature = "deflate")))] + pub fn deflate(mut self, enable: bool) -> ClientBuilder { + self.config.accepts.deflate = enable; + self + } + /// Disable auto response body gzip decompression. /// /// This method exists even if the optional `gzip` feature is not enabled. @@ -570,6 +593,23 @@ impl ClientBuilder { } } + /// Disable auto response body deflate decompression. + /// + /// This method exists even if the optional `deflate` feature is not enabled. + /// This can be used to ensure a `Client` doesn't use deflate decompression + /// even if another dependency were to enable the optional `deflate` feature. + pub fn no_deflate(self) -> ClientBuilder { + #[cfg(feature = "deflate")] + { + self.deflate(false) + } + + #[cfg(not(feature = "deflate"))] + { + self + } + } + // Redirect options /// Set a `RedirectPolicy` for this client. diff --git a/src/async_impl/decoder.rs b/src/async_impl/decoder.rs index 50f762f..38adeeb 100644 --- a/src/async_impl/decoder.rs +++ b/src/async_impl/decoder.rs @@ -9,15 +9,18 @@ use async_compression::tokio::bufread::GzipDecoder; #[cfg(feature = "brotli")] use async_compression::tokio::bufread::BrotliDecoder; +#[cfg(feature = "deflate")] +use async_compression::tokio::bufread::DeflateDecoder; + use bytes::Bytes; use futures_core::Stream; use futures_util::stream::Peekable; use http::HeaderMap; use hyper::body::HttpBody; -#[cfg(any(feature = "gzip", feature = "brotli"))] +#[cfg(any(feature = "gzip", feature = "brotli", feature = "deflate"))] use tokio_util::codec::{BytesCodec, FramedRead}; -#[cfg(any(feature = "gzip", feature = "brotli"))] +#[cfg(any(feature = "gzip", feature = "brotli", feature = "deflate"))] use tokio_util::io::StreamReader; use super::super::Body; @@ -29,6 +32,8 @@ pub(super) struct Accepts { pub(super) gzip: bool, #[cfg(feature = "brotli")] pub(super) brotli: bool, + #[cfg(feature = "deflate")] + pub(super) deflate: bool, } /// A response decompressor over a non-blocking stream of chunks. @@ -50,8 +55,12 @@ enum Inner { #[cfg(feature = "brotli")] Brotli(FramedRead, Bytes>>, BytesCodec>), + /// A `Deflate` decoder will uncompress the deflated response content before returning it. + #[cfg(feature = "deflate")] + Deflate(FramedRead, Bytes>>, BytesCodec>), + /// A decoder that doesn't have a value yet. - #[cfg(any(feature = "brotli", feature = "gzip"))] + #[cfg(any(feature = "brotli", feature = "gzip", feature = "deflate"))] Pending(Pending), } @@ -65,6 +74,8 @@ enum DecoderType { Gzip, #[cfg(feature = "brotli")] Brotli, + #[cfg(feature = "deflate")] + Deflate, } impl fmt::Debug for Decoder { @@ -120,68 +131,49 @@ impl Decoder { } } - #[cfg(feature = "gzip")] - fn detect_gzip(headers: &mut HeaderMap) -> bool { - use http::header::{CONTENT_ENCODING, CONTENT_LENGTH, TRANSFER_ENCODING}; - use log::warn; + /// A deflate decoder. + /// + /// This decoder will buffer and decompress chunks that are deflated. + #[cfg(feature = "deflate")] + fn deflate(body: Body) -> Decoder { + use futures_util::StreamExt; - let content_encoding_gzip: bool; - let mut is_gzip = { - content_encoding_gzip = headers - .get_all(CONTENT_ENCODING) - .iter() - .any(|enc| enc == "gzip"); - content_encoding_gzip - || headers - .get_all(TRANSFER_ENCODING) - .iter() - .any(|enc| enc == "gzip") - }; - if is_gzip { - if let Some(content_length) = headers.get(CONTENT_LENGTH) { - if content_length == "0" { - warn!("gzip response with content-length of 0"); - is_gzip = false; - } - } + Decoder { + inner: Inner::Pending(Pending( + IoStream(body.into_stream()).peekable(), + DecoderType::Deflate, + )), } - if is_gzip { - headers.remove(CONTENT_ENCODING); - headers.remove(CONTENT_LENGTH); - } - is_gzip } - #[cfg(feature = "brotli")] - fn detect_brotli(headers: &mut HeaderMap) -> bool { + #[cfg(any(feature = "brotli", feature = "gzip", feature = "deflate"))] + fn detect_encoding(headers: &mut HeaderMap, encoding_str: &str) -> bool { use http::header::{CONTENT_ENCODING, CONTENT_LENGTH, TRANSFER_ENCODING}; use log::warn; - let content_encoding_gzip: bool; - let mut is_brotli = { - content_encoding_gzip = headers + let mut is_content_encoded = { + headers .get_all(CONTENT_ENCODING) .iter() - .any(|enc| enc == "br"); - content_encoding_gzip + .any(|enc| enc == encoding_str) || headers .get_all(TRANSFER_ENCODING) .iter() - .any(|enc| enc == "br") + .any(|enc| enc == encoding_str) }; - if is_brotli { + if is_content_encoded { if let Some(content_length) = headers.get(CONTENT_LENGTH) { if content_length == "0" { - warn!("brotli response with content-length of 0"); - is_brotli = false; + warn!("{} response with content-length of 0", encoding_str); + is_content_encoded = false; } } } - if is_brotli { + if is_content_encoded { headers.remove(CONTENT_ENCODING); headers.remove(CONTENT_LENGTH); } - is_brotli + is_content_encoded } /// Constructs a Decoder from a hyper request. @@ -193,18 +185,25 @@ impl Decoder { pub(super) fn detect(_headers: &mut HeaderMap, body: Body, _accepts: Accepts) -> Decoder { #[cfg(feature = "gzip")] { - if _accepts.gzip && Decoder::detect_gzip(_headers) { + if _accepts.gzip && Decoder::detect_encoding(_headers, "gzip") { return Decoder::gzip(body); } } #[cfg(feature = "brotli")] { - if _accepts.brotli && Decoder::detect_brotli(_headers) { + if _accepts.brotli && Decoder::detect_encoding(_headers, "br") { return Decoder::brotli(body); } } + #[cfg(feature = "deflate")] + { + if _accepts.deflate && Decoder::detect_encoding(_headers, "deflate") { + return Decoder::deflate(body); + } + } + Decoder::plain_text(body) } } @@ -215,7 +214,7 @@ impl Stream for Decoder { fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { // Do a read or poll for a pending decoder value. match self.inner { - #[cfg(any(feature = "brotli", feature = "gzip"))] + #[cfg(any(feature = "brotli", feature = "gzip", feature = "deflate"))] Inner::Pending(ref mut future) => match Pin::new(future).poll(cx) { Poll::Ready(Ok(inner)) => { self.inner = inner; @@ -243,6 +242,14 @@ impl Stream for Decoder { None => Poll::Ready(None), }; } + #[cfg(feature = "deflate")] + Inner::Deflate(ref mut decoder) => { + return match futures_core::ready!(Pin::new(decoder).poll_next(cx)) { + Some(Ok(bytes)) => Poll::Ready(Some(Ok(bytes.freeze()))), + Some(Err(err)) => Poll::Ready(Some(Err(crate::error::decode_io(err)))), + None => Poll::Ready(None), + }; + } } } } @@ -269,7 +276,7 @@ impl HttpBody for Decoder { match self.inner { Inner::PlainText(ref body) => HttpBody::size_hint(body), // the rest are "unknown", so default - #[cfg(any(feature = "brotli", feature = "gzip"))] + #[cfg(any(feature = "brotli", feature = "gzip", feature = "deflate"))] _ => http_body::SizeHint::default(), } } @@ -312,6 +319,11 @@ impl Future for Pending { GzipDecoder::new(StreamReader::new(_body)), BytesCodec::new(), )))), + #[cfg(feature = "deflate")] + DecoderType::Deflate => Poll::Ready(Ok(Inner::Deflate(FramedRead::new( + DeflateDecoder::new(StreamReader::new(_body)), + BytesCodec::new(), + )))), } } } @@ -337,15 +349,21 @@ impl Accepts { gzip: false, #[cfg(feature = "brotli")] brotli: false, + #[cfg(feature = "deflate")] + deflate: false, } } pub(super) fn as_str(&self) -> Option<&'static str> { - match (self.is_gzip(), self.is_brotli()) { - (true, true) => Some("gzip, br"), - (true, false) => Some("gzip"), - (false, true) => Some("br"), - _ => None, + match (self.is_gzip(), self.is_brotli(), self.is_deflate()) { + (true, true, true) => Some("gzip, br, deflate"), + (true, true, false) => Some("gzip, br"), + (true, false, true) => Some("gzip, deflate"), + (false, true, true) => Some("br, deflate"), + (true, false, false) => Some("gzip"), + (false, true, false) => Some("br"), + (false, false, true) => Some("deflate"), + (false, false, false) => None, } } @@ -372,6 +390,18 @@ impl Accepts { false } } + + fn is_deflate(&self) -> bool { + #[cfg(feature = "deflate")] + { + self.deflate + } + + #[cfg(not(feature = "deflate"))] + { + false + } + } } impl Default for Accepts { @@ -381,6 +411,8 @@ impl Default for Accepts { gzip: true, #[cfg(feature = "brotli")] brotli: true, + #[cfg(feature = "deflate")] + deflate: true, } } } diff --git a/src/blocking/client.rs b/src/blocking/client.rs index 568a741..0dce1d2 100644 --- a/src/blocking/client.rs +++ b/src/blocking/client.rs @@ -257,6 +257,28 @@ impl ClientBuilder { self.with_inner(|inner| inner.brotli(enable)) } + /// Enable auto deflate decompression by checking the `Content-Encoding` response header. + /// + /// If auto deflate decompresson is turned on: + /// + /// - When sending a request and if the request's headers do not already contain + /// an `Accept-Encoding` **and** `Range` values, the `Accept-Encoding` header is set to `deflate`. + /// The request body is **not** automatically compressed. + /// - When receiving a response, if it's headers contain a `Content-Encoding` value that + /// equals to `deflate`, both values `Content-Encoding` and `Content-Length` are removed from the + /// headers' set. The response body is automatically decompressed. + /// + /// If the `deflate` feature is turned on, the default option is enabled. + /// + /// # Optional + /// + /// This requires the optional `deflate` feature to be enabled + #[cfg(feature = "deflate")] + #[cfg_attr(docsrs, doc(cfg(feature = "deflate")))] + pub fn deflate(self, enable: bool) -> ClientBuilder { + self.with_inner(|inner| inner.deflate(enable)) + } + /// Disable auto response body gzip decompression. /// /// This method exists even if the optional `gzip` feature is not enabled. @@ -275,6 +297,15 @@ impl ClientBuilder { self.with_inner(|inner| inner.no_brotli()) } + /// Disable auto response body deflate decompression. + /// + /// This method exists even if the optional `deflate` feature is not enabled. + /// This can be used to ensure a `Client` doesn't use deflate decompression + /// even if another dependency were to enable the optional `deflate` feature. + pub fn no_deflate(self) -> ClientBuilder { + self.with_inner(|inner| inner.no_deflate()) + } + // Redirect options /// Set a `redirect::Policy` for this client. diff --git a/src/lib.rs b/src/lib.rs index f158f78..f8b4c74 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -182,6 +182,7 @@ //! - **cookies**: Provides cookie session support. //! - **gzip**: Provides response body gzip decompression. //! - **brotli**: Provides response body brotli decompression. +//! - **deflate**: Provides response body deflate decompression. //! - **json**: Provides serialization and deserialization for JSON bodies. //! - **multipart**: Provides functionality for multipart forms. //! - **stream**: Adds support for `futures::Stream`. diff --git a/tests/client.rs b/tests/client.rs index dc53a3a..2fbddc2 100644 --- a/tests/client.rs +++ b/tests/client.rs @@ -24,6 +24,12 @@ async fn auto_headers() { .unwrap() .contains("br")); } + if cfg!(feature = "deflate") { + assert!(req.headers()["accept-encoding"] + .to_str() + .unwrap() + .contains("deflate")); + } http::Response::default() }); diff --git a/tests/deflate.rs b/tests/deflate.rs new file mode 100644 index 0000000..8b16530 --- /dev/null +++ b/tests/deflate.rs @@ -0,0 +1,151 @@ +mod support; +use std::io::Write; +use support::*; + +#[tokio::test] +async fn deflate_response() { + deflate_case(10_000, 4096).await; +} + +#[tokio::test] +async fn deflate_single_byte_chunks() { + deflate_case(10, 1).await; +} + +#[tokio::test] +async fn test_deflate_empty_body() { + let server = server::http(move |req| async move { + assert_eq!(req.method(), "HEAD"); + + http::Response::builder() + .header("content-encoding", "deflate") + .header("content-length", 100) + .body(Default::default()) + .unwrap() + }); + + let client = reqwest::Client::new(); + let res = client + .head(&format!("http://{}/deflate", server.addr())) + .send() + .await + .unwrap(); + + let body = res.text().await.unwrap(); + + assert_eq!(body, ""); +} + +#[tokio::test] +async fn test_accept_header_is_not_changed_if_set() { + let server = server::http(move |req| async move { + assert_eq!(req.headers()["accept"], "application/json"); + assert!(req.headers()["accept-encoding"] + .to_str() + .unwrap() + .contains("deflate")); + http::Response::default() + }); + + let client = reqwest::Client::new(); + + let res = client + .get(&format!("http://{}/accept", server.addr())) + .header( + reqwest::header::ACCEPT, + reqwest::header::HeaderValue::from_static("application/json"), + ) + .send() + .await + .unwrap(); + + assert_eq!(res.status(), reqwest::StatusCode::OK); +} + +#[tokio::test] +async fn test_accept_encoding_header_is_not_changed_if_set() { + let server = server::http(move |req| async move { + assert_eq!(req.headers()["accept"], "*/*"); + assert_eq!(req.headers()["accept-encoding"], "identity"); + http::Response::default() + }); + + let client = reqwest::Client::new(); + + let res = client + .get(&format!("http://{}/accept-encoding", server.addr())) + .header( + reqwest::header::ACCEPT_ENCODING, + reqwest::header::HeaderValue::from_static("identity"), + ) + .send() + .await + .unwrap(); + + assert_eq!(res.status(), reqwest::StatusCode::OK); +} + +async fn deflate_case(response_size: usize, chunk_size: usize) { + use futures_util::stream::StreamExt; + + let content: String = (0..response_size) + .into_iter() + .map(|i| format!("test {}", i)) + .collect(); + let mut encoder = libflate::deflate::Encoder::new(Vec::new()); + match encoder.write(content.as_bytes()) { + Ok(n) => assert!(n > 0, "Failed to write to encoder."), + _ => panic!("Failed to deflate encode string."), + }; + + let deflated_content = encoder.finish().into_result().unwrap(); + + let mut response = format!( + "\ + HTTP/1.1 200 OK\r\n\ + Server: test-accept\r\n\ + Content-Encoding: deflate\r\n\ + Content-Length: {}\r\n\ + \r\n", + &deflated_content.len() + ) + .into_bytes(); + response.extend(&deflated_content); + + let server = server::http(move |req| { + assert!(req.headers()["accept-encoding"] + .to_str() + .unwrap() + .contains("deflate")); + + let deflated = deflated_content.clone(); + async move { + let len = deflated.len(); + let stream = + futures_util::stream::unfold((deflated, 0), move |(deflated, pos)| async move { + let chunk = deflated.chunks(chunk_size).nth(pos)?.to_vec(); + + Some((chunk, (deflated, pos + 1))) + }); + + let body = hyper::Body::wrap_stream(stream.map(Ok::<_, std::convert::Infallible>)); + + http::Response::builder() + .header("content-encoding", "deflate") + .header("content-length", len) + .body(body) + .unwrap() + } + }); + + let client = reqwest::Client::new(); + + let res = client + .get(&format!("http://{}/deflate", server.addr())) + .send() + .await + .expect("response"); + + let body = res.text().await.expect("text"); + assert_eq!(body, content); +}