forked/reqwest-impersonate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support Deflate decoding (#1250)

Browse Source
This commit is contained in:
Mohamed Daahir
2021-04-22 18:35:29 +01:00
committed by GitHub
parent 42b31600c3
commit 77ee0df7c5
8 changed files with 328 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
.github/workflows/ci.yml vendored
View File

@@ -66,6 +66,7 @@ jobs:
- "feat.: blocking"
- "feat.: gzip"
- "feat.: brotli"
- "feat.: deflate"
- "feat.: json"
- "feat.: multipart"
- "feat.: stream"
@@ -89,21 +90,21 @@ jobs:
- name: windows / stable-x86_64-msvc
os: windows-latest
target: x86_64-pc-windows-msvc
features: "--features blocking,gzip,brotli,json,multipart"
features: "--features blocking,gzip,brotli,deflate,json,multipart"
- name: windows / stable-i686-msvc
os: windows-latest
target: i686-pc-windows-msvc
features: "--features blocking,gzip,brotli,json,multipart"
features: "--features blocking,gzip,brotli,deflate,json,multipart"
- name: windows / stable-x86_64-gnu
os: windows-latest
rust: stable-x86_64-pc-windows-gnu
target: x86_64-pc-windows-gnu
features: "--features blocking,gzip,brotli,json,multipart"
features: "--features blocking,gzip,brotli,deflate,json,multipart"
- name: windows / stable-i686-gnu
os: windows-latest
rust: stable-i686-pc-windows-gnu
target: i686-pc-windows-gnu
features: "--features blocking,gzip,brotli,json,multipart"
features: "--features blocking,gzip,brotli,deflate,json,multipart"
- name: "feat.: default-tls disabled"
features: "--no-default-features"
@@ -123,6 +124,8 @@ jobs:
features: "--features gzip"
- name: "feat.: brotli"
features: "--features brotli"
- name: "feat.: deflate"
features: "--features deflate"
- name: "feat.: json"
features: "--features json"
- name: "feat.: multipart"

7
Cargo.toml
View File

@@ -49,6 +49,8 @@ gzip = ["async-compression", "async-compression/gzip", "tokio-util"]
brotli = ["async-compression", "async-compression/brotli", "tokio-util"]
deflate = ["async-compression", "async-compression/deflate", "tokio-util"]
json = ["serde_json"]
multipart = ["mime_guess"]
@@ -216,6 +218,11 @@ name = "brotli"
path = "tests/brotli.rs"
required-features = ["brotli"]
[[test]]
name = "deflate"
path = "tests/deflate.rs"
required-features = ["deflate"]
[[test]]
name = "multipart"
path = "tests/multipart.rs"

40
src/async_impl/client.rs
View File

@@ -536,6 +536,29 @@ impl ClientBuilder {
self
}
/// Enable auto deflate decompression by checking the `Content-Encoding` response header.
///
/// If auto deflate decompression is turned on:
///
/// - When sending a request and if the request's headers do not already contain
/// an `Accept-Encoding` **and** `Range` values, the `Accept-Encoding` header is set to `deflate`.
/// The request body is **not** automatically compressed.
/// - When receiving a response, if it's headers contain a `Content-Encoding` value that
/// equals to `deflate`, both values `Content-Encoding` and `Content-Length` are removed from the
/// headers' set. The response body is automatically decompressed.
///
/// If the `deflate` feature is turned on, the default option is enabled.
///
/// # Optional
///
/// This requires the optional `deflate` feature to be enabled
#[cfg(feature = "deflate")]
#[cfg_attr(docsrs, doc(cfg(feature = "deflate")))]
pub fn deflate(mut self, enable: bool) -> ClientBuilder {
self.config.accepts.deflate = enable;
self
}
/// Disable auto response body gzip decompression.
///
/// This method exists even if the optional `gzip` feature is not enabled.
@@ -570,6 +593,23 @@ impl ClientBuilder {
}
}
/// Disable auto response body deflate decompression.
///
/// This method exists even if the optional `deflate` feature is not enabled.
/// This can be used to ensure a `Client` doesn't use deflate decompression
/// even if another dependency were to enable the optional `deflate` feature.
pub fn no_deflate(self) -> ClientBuilder {
#[cfg(feature = "deflate")]
{
self.deflate(false)
}
#[cfg(not(feature = "deflate"))]
{
self
}
}
// Redirect options
/// Set a `RedirectPolicy` for this client.

138
src/async_impl/decoder.rs
View File

@@ -9,15 +9,18 @@ use async_compression::tokio::bufread::GzipDecoder;
#[cfg(feature = "brotli")]
use async_compression::tokio::bufread::BrotliDecoder;
#[cfg(feature = "deflate")]
use async_compression::tokio::bufread::DeflateDecoder;
use bytes::Bytes;
use futures_core::Stream;
use futures_util::stream::Peekable;
use http::HeaderMap;
use hyper::body::HttpBody;
#[cfg(any(feature = "gzip", feature = "brotli"))]
#[cfg(any(feature = "gzip", feature = "brotli", feature = "deflate"))]
use tokio_util::codec::{BytesCodec, FramedRead};
#[cfg(any(feature = "gzip", feature = "brotli"))]
#[cfg(any(feature = "gzip", feature = "brotli", feature = "deflate"))]
use tokio_util::io::StreamReader;
use super::super::Body;
@@ -29,6 +32,8 @@ pub(super) struct Accepts {
pub(super) gzip: bool,
#[cfg(feature = "brotli")]
pub(super) brotli: bool,
#[cfg(feature = "deflate")]
pub(super) deflate: bool,
}
/// A response decompressor over a non-blocking stream of chunks.
@@ -50,8 +55,12 @@ enum Inner {
#[cfg(feature = "brotli")]
Brotli(FramedRead<BrotliDecoder<StreamReader<Peekable<IoStream>, Bytes>>, BytesCodec>),
/// A `Deflate` decoder will uncompress the deflated response content before returning it.
#[cfg(feature = "deflate")]
Deflate(FramedRead<DeflateDecoder<StreamReader<Peekable<IoStream>, Bytes>>, BytesCodec>),
/// A decoder that doesn't have a value yet.
#[cfg(any(feature = "brotli", feature = "gzip"))]
#[cfg(any(feature = "brotli", feature = "gzip", feature = "deflate"))]
Pending(Pending),
}
@@ -65,6 +74,8 @@ enum DecoderType {
Gzip,
#[cfg(feature = "brotli")]
Brotli,
#[cfg(feature = "deflate")]
Deflate,
}
impl fmt::Debug for Decoder {
@@ -120,68 +131,49 @@ impl Decoder {
}
}
#[cfg(feature = "gzip")]
fn detect_gzip(headers: &mut HeaderMap) -> bool {
use http::header::{CONTENT_ENCODING, CONTENT_LENGTH, TRANSFER_ENCODING};
use log::warn;
/// A deflate decoder.
///
/// This decoder will buffer and decompress chunks that are deflated.
#[cfg(feature = "deflate")]
fn deflate(body: Body) -> Decoder {
use futures_util::StreamExt;
let content_encoding_gzip: bool;
let mut is_gzip = {
content_encoding_gzip = headers
.get_all(CONTENT_ENCODING)
.iter()
.any(|enc| enc == "gzip");
content_encoding_gzip
|| headers
.get_all(TRANSFER_ENCODING)
.iter()
.any(|enc| enc == "gzip")
};
if is_gzip {
if let Some(content_length) = headers.get(CONTENT_LENGTH) {
if content_length == "0" {
warn!("gzip response with content-length of 0");
is_gzip = false;
}
}
Decoder {
inner: Inner::Pending(Pending(
IoStream(body.into_stream()).peekable(),
DecoderType::Deflate,
)),
}
if is_gzip {
headers.remove(CONTENT_ENCODING);
headers.remove(CONTENT_LENGTH);
}
is_gzip
}
#[cfg(feature = "brotli")]
fn detect_brotli(headers: &mut HeaderMap) -> bool {
#[cfg(any(feature = "brotli", feature = "gzip", feature = "deflate"))]
fn detect_encoding(headers: &mut HeaderMap, encoding_str: &str) -> bool {
use http::header::{CONTENT_ENCODING, CONTENT_LENGTH, TRANSFER_ENCODING};
use log::warn;
let content_encoding_gzip: bool;
let mut is_brotli = {
content_encoding_gzip = headers
let mut is_content_encoded = {
headers
.get_all(CONTENT_ENCODING)
.iter()
.any(|enc| enc == "br");
content_encoding_gzip
.any(|enc| enc == encoding_str)
|| headers
.get_all(TRANSFER_ENCODING)
.iter()
.any(|enc| enc == "br")
.any(|enc| enc == encoding_str)
};
if is_brotli {
if is_content_encoded {
if let Some(content_length) = headers.get(CONTENT_LENGTH) {
if content_length == "0" {
warn!("brotli response with content-length of 0");
is_brotli = false;
warn!("{} response with content-length of 0", encoding_str);
is_content_encoded = false;
}
}
}
if is_brotli {
if is_content_encoded {
headers.remove(CONTENT_ENCODING);
headers.remove(CONTENT_LENGTH);
}
is_brotli
is_content_encoded
}
/// Constructs a Decoder from a hyper request.
@@ -193,18 +185,25 @@ impl Decoder {
pub(super) fn detect(_headers: &mut HeaderMap, body: Body, _accepts: Accepts) -> Decoder {
#[cfg(feature = "gzip")]
{
if _accepts.gzip && Decoder::detect_gzip(_headers) {
if _accepts.gzip && Decoder::detect_encoding(_headers, "gzip") {
return Decoder::gzip(body);
}
}
#[cfg(feature = "brotli")]
{
if _accepts.brotli && Decoder::detect_brotli(_headers) {
if _accepts.brotli && Decoder::detect_encoding(_headers, "br") {
return Decoder::brotli(body);
}
}
#[cfg(feature = "deflate")]
{
if _accepts.deflate && Decoder::detect_encoding(_headers, "deflate") {
return Decoder::deflate(body);
}
}
Decoder::plain_text(body)
}
}
@@ -215,7 +214,7 @@ impl Stream for Decoder {
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
// Do a read or poll for a pending decoder value.
match self.inner {
#[cfg(any(feature = "brotli", feature = "gzip"))]
#[cfg(any(feature = "brotli", feature = "gzip", feature = "deflate"))]
Inner::Pending(ref mut future) => match Pin::new(future).poll(cx) {
Poll::Ready(Ok(inner)) => {
self.inner = inner;
@@ -243,6 +242,14 @@ impl Stream for Decoder {
None => Poll::Ready(None),
};
}
#[cfg(feature = "deflate")]
Inner::Deflate(ref mut decoder) => {
return match futures_core::ready!(Pin::new(decoder).poll_next(cx)) {
Some(Ok(bytes)) => Poll::Ready(Some(Ok(bytes.freeze()))),
Some(Err(err)) => Poll::Ready(Some(Err(crate::error::decode_io(err)))),
None => Poll::Ready(None),
};
}
}
}
}
@@ -269,7 +276,7 @@ impl HttpBody for Decoder {
match self.inner {
Inner::PlainText(ref body) => HttpBody::size_hint(body),
// the rest are "unknown", so default
#[cfg(any(feature = "brotli", feature = "gzip"))]
#[cfg(any(feature = "brotli", feature = "gzip", feature = "deflate"))]
_ => http_body::SizeHint::default(),
}
}
@@ -312,6 +319,11 @@ impl Future for Pending {
GzipDecoder::new(StreamReader::new(_body)),
BytesCodec::new(),
)))),
#[cfg(feature = "deflate")]
DecoderType::Deflate => Poll::Ready(Ok(Inner::Deflate(FramedRead::new(
DeflateDecoder::new(StreamReader::new(_body)),
BytesCodec::new(),
)))),
}
}
}
@@ -337,15 +349,21 @@ impl Accepts {
gzip: false,
#[cfg(feature = "brotli")]
brotli: false,
#[cfg(feature = "deflate")]
deflate: false,
}
}
pub(super) fn as_str(&self) -> Option<&'static str> {
match (self.is_gzip(), self.is_brotli()) {
(true, true) => Some("gzip, br"),
(true, false) => Some("gzip"),
(false, true) => Some("br"),
_ => None,
match (self.is_gzip(), self.is_brotli(), self.is_deflate()) {
(true, true, true) => Some("gzip, br, deflate"),
(true, true, false) => Some("gzip, br"),
(true, false, true) => Some("gzip, deflate"),
(false, true, true) => Some("br, deflate"),
(true, false, false) => Some("gzip"),
(false, true, false) => Some("br"),
(false, false, true) => Some("deflate"),
(false, false, false) => None,
}
}
@@ -372,6 +390,18 @@ impl Accepts {
false
}
}
fn is_deflate(&self) -> bool {
#[cfg(feature = "deflate")]
{
self.deflate
}
#[cfg(not(feature = "deflate"))]
{
false
}
}
}
impl Default for Accepts {
@@ -381,6 +411,8 @@ impl Default for Accepts {
gzip: true,
#[cfg(feature = "brotli")]
brotli: true,
#[cfg(feature = "deflate")]
deflate: true,
}
}
}

31
src/blocking/client.rs
View File

@@ -257,6 +257,28 @@ impl ClientBuilder {
self.with_inner(|inner| inner.brotli(enable))
}
/// Enable auto deflate decompression by checking the `Content-Encoding` response header.
///
/// If auto deflate decompresson is turned on:
///
/// - When sending a request and if the request's headers do not already contain
/// an `Accept-Encoding` **and** `Range` values, the `Accept-Encoding` header is set to `deflate`.
/// The request body is **not** automatically compressed.
/// - When receiving a response, if it's headers contain a `Content-Encoding` value that
/// equals to `deflate`, both values `Content-Encoding` and `Content-Length` are removed from the
/// headers' set. The response body is automatically decompressed.
///
/// If the `deflate` feature is turned on, the default option is enabled.
///
/// # Optional
///
/// This requires the optional `deflate` feature to be enabled
#[cfg(feature = "deflate")]
#[cfg_attr(docsrs, doc(cfg(feature = "deflate")))]
pub fn deflate(self, enable: bool) -> ClientBuilder {
self.with_inner(|inner| inner.deflate(enable))
}
/// Disable auto response body gzip decompression.
///
/// This method exists even if the optional `gzip` feature is not enabled.
@@ -275,6 +297,15 @@ impl ClientBuilder {
self.with_inner(|inner| inner.no_brotli())
}
/// Disable auto response body deflate decompression.
///
/// This method exists even if the optional `deflate` feature is not enabled.
/// This can be used to ensure a `Client` doesn't use deflate decompression
/// even if another dependency were to enable the optional `deflate` feature.
pub fn no_deflate(self) -> ClientBuilder {
self.with_inner(|inner| inner.no_deflate())
}
// Redirect options
/// Set a `redirect::Policy` for this client.

1
src/lib.rs
View File

@@ -182,6 +182,7 @@
//! - **cookies**: Provides cookie session support.
//! - **gzip**: Provides response body gzip decompression.
//! - **brotli**: Provides response body brotli decompression.
//! - **deflate**: Provides response body deflate decompression.
//! - **json**: Provides serialization and deserialization for JSON bodies.
//! - **multipart**: Provides functionality for multipart forms.
//! - **stream**: Adds support for `futures::Stream`.

6
tests/client.rs
View File

@@ -24,6 +24,12 @@ async fn auto_headers() {
.unwrap()
.contains("br"));
}
if cfg!(feature = "deflate") {
assert!(req.headers()["accept-encoding"]
.to_str()
.unwrap()
.contains("deflate"));
}
http::Response::default()
});

151
tests/deflate.rs Normal file
View File

@@ -0,0 +1,151 @@
mod support;
use std::io::Write;
use support::*;
#[tokio::test]
async fn deflate_response() {
deflate_case(10_000, 4096).await;
}
#[tokio::test]
async fn deflate_single_byte_chunks() {
deflate_case(10, 1).await;
}
#[tokio::test]
async fn test_deflate_empty_body() {
let server = server::http(move |req| async move {
assert_eq!(req.method(), "HEAD");
http::Response::builder()
.header("content-encoding", "deflate")
.header("content-length", 100)
.body(Default::default())
.unwrap()
});
let client = reqwest::Client::new();
let res = client
.head(&format!("http://{}/deflate", server.addr()))
.send()
.await
.unwrap();
let body = res.text().await.unwrap();
assert_eq!(body, "");
}
#[tokio::test]
async fn test_accept_header_is_not_changed_if_set() {
let server = server::http(move |req| async move {
assert_eq!(req.headers()["accept"], "application/json");
assert!(req.headers()["accept-encoding"]
.to_str()
.unwrap()
.contains("deflate"));
http::Response::default()
});
let client = reqwest::Client::new();
let res = client
.get(&format!("http://{}/accept", server.addr()))
.header(
reqwest::header::ACCEPT,
reqwest::header::HeaderValue::from_static("application/json"),
)
.send()
.await
.unwrap();
assert_eq!(res.status(), reqwest::StatusCode::OK);
}
#[tokio::test]
async fn test_accept_encoding_header_is_not_changed_if_set() {
let server = server::http(move |req| async move {
assert_eq!(req.headers()["accept"], "*/*");
assert_eq!(req.headers()["accept-encoding"], "identity");
http::Response::default()
});
let client = reqwest::Client::new();
let res = client
.get(&format!("http://{}/accept-encoding", server.addr()))
.header(
reqwest::header::ACCEPT_ENCODING,
reqwest::header::HeaderValue::from_static("identity"),
)
.send()
.await
.unwrap();
assert_eq!(res.status(), reqwest::StatusCode::OK);
}
async fn deflate_case(response_size: usize, chunk_size: usize) {
use futures_util::stream::StreamExt;
let content: String = (0..response_size)
.into_iter()
.map(|i| format!("test {}", i))
.collect();
let mut encoder = libflate::deflate::Encoder::new(Vec::new());
match encoder.write(content.as_bytes()) {
Ok(n) => assert!(n > 0, "Failed to write to encoder."),
_ => panic!("Failed to deflate encode string."),
};
let deflated_content = encoder.finish().into_result().unwrap();
let mut response = format!(
"\
HTTP/1.1 200 OK\r\n\
Server: test-accept\r\n\
Content-Encoding: deflate\r\n\
Content-Length: {}\r\n\
\r\n",
&deflated_content.len()
)
.into_bytes();
response.extend(&deflated_content);
let server = server::http(move |req| {
assert!(req.headers()["accept-encoding"]
.to_str()
.unwrap()
.contains("deflate"));
let deflated = deflated_content.clone();
async move {
let len = deflated.len();
let stream =
futures_util::stream::unfold((deflated, 0), move |(deflated, pos)| async move {
let chunk = deflated.chunks(chunk_size).nth(pos)?.to_vec();
Some((chunk, (deflated, pos + 1)))
});
let body = hyper::Body::wrap_stream(stream.map(Ok::<_, std::convert::Infallible>));
http::Response::builder()
.header("content-encoding", "deflate")
.header("content-length", len)
.body(body)
.unwrap()
}
});
let client = reqwest::Client::new();
let res = client
.get(&format!("http://{}/deflate", server.addr()))
.send()
.await
.expect("response");
let body = res.text().await.expect("text");
assert_eq!(body, content);
}