forked/reqwest-impersonate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Brotli support (#791)

Browse Source
This commit is contained in:
EnokMan
2020-02-19 14:49:11 -06:00
committed by GitHub
parent 7622c75064
commit f831d62da0
9 changed files with 400 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
src/async_impl/client.rs
View File

@@ -11,8 +11,8 @@ use http::header::{
Entry, HeaderMap, HeaderValue, ACCEPT, ACCEPT_ENCODING, CONTENT_ENCODING, CONTENT_LENGTH,
CONTENT_TYPE, LOCATION, PROXY_AUTHORIZATION, RANGE, REFERER, TRANSFER_ENCODING, USER_AGENT,
};
use http::Uri;
use http::uri::Scheme;
use http::Uri;
use hyper::client::ResponseFuture;
#[cfg(feature = "native-tls-crate")]
use native_tls_crate::TlsConnector;
@@ -58,6 +58,7 @@ pub struct ClientBuilder {
struct Config {
// NOTE: When adding a new field, update `fmt::Debug for ClientBuilder`
gzip: bool,
brotli: bool,
headers: HeaderMap,
#[cfg(feature = "native-tls")]
hostname_verification: bool,
@@ -106,6 +107,7 @@ impl ClientBuilder {
config: Config {
error: None,
gzip: cfg!(feature = "gzip"),
brotli: cfg!(feature = "brotli"),
headers,
#[cfg(feature = "native-tls")]
hostname_verification: true,
@@ -179,7 +181,6 @@ impl ClientBuilder {
cert.add_to_native_tls(&mut tls);
}
#[cfg(feature = "native-tls")]
{
if let Some(id) = config.identity {
@@ -246,7 +247,9 @@ impl ClientBuilder {
if let Some(http2_initial_stream_window_size) = config.http2_initial_stream_window_size {
builder.http2_initial_stream_window_size(http2_initial_stream_window_size);
}
if let Some(http2_initial_connection_window_size) = config.http2_initial_connection_window_size {
if let Some(http2_initial_connection_window_size) =
config.http2_initial_connection_window_size
{
builder.http2_initial_connection_window_size(http2_initial_connection_window_size);
}
@@ -265,6 +268,7 @@ impl ClientBuilder {
#[cfg(feature = "cookies")]
cookie_store: config.cookie_store.map(RwLock::new),
gzip: config.gzip,
brotli: config.brotli,
hyper: hyper_client,
headers: config.headers,
redirect_policy: config.redirect_policy,
@@ -278,7 +282,6 @@ impl ClientBuilder {
// Higher-level options
/// Sets the `User-Agent` header to be used by this client.
///
/// # Example
@@ -360,7 +363,6 @@ impl ClientBuilder {
self
}
/// Enable a persistent cookie store for the client.
///
/// Cookies received in responses will be preserved and included in
@@ -383,7 +385,7 @@ impl ClientBuilder {
/// Enable auto gzip decompression by checking the `Content-Encoding` response header.
///
/// If auto gzip decompresson is turned on:
/// If auto gzip decompression is turned on:
///
/// - When sending a request and if the request's headers do not already contain
/// an `Accept-Encoding` **and** `Range` values, the `Accept-Encoding` header is set to `gzip`.
@@ -403,6 +405,28 @@ impl ClientBuilder {
self
}
/// Enable auto brotli decompression by checking the `Content-Encoding` response header.
///
/// If auto brotli decompression is turned on:
///
/// - When sending a request and if the request's headers do not already contain
/// an `Accept-Encoding` **and** `Range` values, the `Accept-Encoding` header is set to `br`.
/// The request body is **not** automatically compressed.
/// - When receiving a response, if it's headers contain a `Content-Encoding` value that
/// equals to `br`, both values `Content-Encoding` and `Content-Length` are removed from the
/// headers' set. The response body is automatically decompressed.
///
/// If the `brotli` feature is turned on, the default option is enabled.
///
/// # Optional
///
/// This requires the optional `brotli` feature to be enabled
#[cfg(feature = "brotli")]
pub fn brotli(mut self, enable: bool) -> ClientBuilder {
self.config.brotli = enable;
self
}
/// Disable auto response body gzip decompression.
///
/// This method exists even if the optional `gzip` feature is not enabled.
@@ -420,6 +444,23 @@ impl ClientBuilder {
}
}
/// Disable auto response body brotli decompression.
///
/// This method exists even if the optional `brotli` feature is not enabled.
/// This can be used to ensure a `Client` doesn't use brotli decompression
/// even if another dependency were to enable the optional `brotli` feature.
pub fn no_brotli(self) -> ClientBuilder {
#[cfg(feature = "brotli")]
{
self.brotli(false)
}
#[cfg(not(feature = "brotli"))]
{
self
}
}
// Redirect options
/// Set a `RedirectPolicy` for this client.
@@ -534,7 +575,10 @@ impl ClientBuilder {
/// Sets the max connection-level flow control for HTTP2
///
/// Default is currently 65,535 but may change internally to optimize for common uses.
pub fn http2_initial_connection_window_size(mut self, sz: impl Into<Option<u32>>) -> ClientBuilder {
pub fn http2_initial_connection_window_size(
mut self,
sz: impl Into<Option<u32>>,
) -> ClientBuilder {
self.config.http2_initial_connection_window_size = sz.into();
self
}
@@ -654,7 +698,6 @@ impl ClientBuilder {
self
}
/// Force using the Rustls TLS backend.
///
/// Since multiple TLS backends can be optionally enabled, this option will
@@ -807,9 +850,21 @@ impl Client {
}
}
if self.inner.gzip && !headers.contains_key(ACCEPT_ENCODING) && !headers.contains_key(RANGE)
let accept_encoding = match (self.inner.gzip, self.inner.brotli) {
(true, true) => Some("gzip, br"),
(true, false) => Some("gzip"),
(false, true) => Some("br"),
_ => None,
};
if accept_encoding.is_some()
&& !headers.contains_key(ACCEPT_ENCODING)
&& !headers.contains_key(RANGE)
{
headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("gzip"));
headers.insert(
ACCEPT_ENCODING,
HeaderValue::from_static(accept_encoding.unwrap()),
);
}
let uri = expect_uri(&url);
@@ -834,7 +889,6 @@ impl Client {
.or(self.inner.request_timeout)
.map(|dur| tokio::time::delay_for(dur));
*req.headers_mut() = headers.clone();
let in_flight = self.inner.hyper.request(req);
@@ -913,6 +967,7 @@ impl Config {
}
f.field("gzip", &self.gzip);
f.field("brotli", &self.brotli);
if !self.proxies.is_empty() {
f.field("proxies", &self.proxies);
@@ -977,6 +1032,7 @@ struct ClientRef {
#[cfg(feature = "cookies")]
cookie_store: Option<RwLock<cookie::CookieStore>>,
gzip: bool,
brotli: bool,
headers: HeaderMap,
hyper: HyperClient,
redirect_policy: redirect::Policy,
@@ -999,6 +1055,7 @@ impl ClientRef {
}
f.field("gzip", &self.gzip);
f.field("brotli", &self.brotli);
if !self.proxies.is_empty() {
f.field("proxies", &self.proxies);
@@ -1014,15 +1071,12 @@ impl ClientRef {
f.field("default_headers", &self.headers);
if let Some(ref d) = self.request_timeout {
f.field("timeout", d);
}
}
}
pub(super) struct Pending {
inner: PendingInner,
}
@@ -1227,17 +1281,20 @@ impl Future for PendingRequest {
debug!("redirect policy disallowed redirection to '{}'", loc);
}
redirect::ActionKind::Error(err) => {
return Poll::Ready(Err(crate::error::redirect(
err,
self.url.clone(),
)));
return Poll::Ready(Err(crate::error::redirect(err, self.url.clone())));
}
}
}
}
debug!("response '{}' for {}", res.status(), self.url);
let res = Response::new(res, self.url.clone(), self.client.gzip, self.timeout.take());
let res = Response::new(
res,
self.url.clone(),
self.client.gzip,
self.client.brotli,
self.timeout.take(),
);
return Poll::Ready(Ok(res));
}
}

207
src/async_impl/decoder.rs
View File

@@ -1,62 +1,21 @@
pub(crate) use self::imp::Decoder;
#[cfg(not(feature = "gzip"))]
mod imp {
use std::pin::Pin;
use std::task::{Context, Poll};
use bytes::Bytes;
use futures_core::Stream;
use http::HeaderMap;
use super::super::Body;
pub(crate) struct Decoder {
inner: super::super::body::ImplStream,
}
impl Decoder {
#[cfg(feature = "blocking")]
pub(crate) fn empty() -> Decoder {
Decoder::plain_text(Body::empty())
}
/// A plain text decoder.
///
/// This decoder will emit the underlying chunks as-is.
fn plain_text(body: Body) -> Decoder {
Decoder {
inner: body.into_stream(),
}
}
pub(crate) fn detect(_: &mut HeaderMap, body: Body, _: bool) -> Decoder {
Decoder::plain_text(body)
}
}
impl Stream for Decoder {
type Item = crate::Result<Bytes>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
Pin::new(&mut self.inner).poll_next(cx)
}
}
}
#[cfg(feature = "gzip")]
mod imp {
use std::fmt;
use std::future::Future;
use std::pin::Pin;
use std::task::{Context, Poll};
use std::{fmt, mem};
#[cfg(feature = "gzip")]
use async_compression::stream::GzipDecoder;
#[cfg(feature = "brotli")]
use async_compression::stream::BrotliDecoder;
use bytes::Bytes;
use futures_core::Stream;
use futures_util::stream::Peekable;
use http::header::{CONTENT_ENCODING, CONTENT_LENGTH, TRANSFER_ENCODING};
use http::HeaderMap;
use log::warn;
use super::super::Body;
use crate::error;
@@ -68,17 +27,32 @@ mod imp {
inner: Inner,
}
enum DecoderType {
#[cfg(feature = "gzip")]
Gzip,
#[cfg(feature = "brotli")]
Brotli,
}
enum Inner {
/// A `PlainText` decoder just returns the response content as is.
PlainText(super::super::body::ImplStream),
/// A `Gzip` decoder will uncompress the gzipped response content before returning it.
#[cfg(feature = "gzip")]
Gzip(GzipDecoder<Peekable<IoStream>>),
/// A `Brotli` decoder will uncompress the brotlied response content before returning it.
#[cfg(feature = "brotli")]
Brotli(BrotliDecoder<Peekable<IoStream>>),
/// A decoder that doesn't have a value yet.
#[cfg(any(feature = "brotli", feature = "gzip"))]
Pending(Pending),
}
/// A future attempt to poll the response body for EOF so we know whether to use gzip or not.
struct Pending(Peekable<IoStream>);
struct Pending(Peekable<IoStream>, DecoderType);
struct IoStream(super::super::body::ImplStream);
@@ -108,24 +82,38 @@ mod imp {
/// A gzip decoder.
///
/// This decoder will buffer and decompress chunks that are gzipped.
#[cfg(feature = "gzip")]
fn gzip(body: Body) -> Decoder {
use futures_util::StreamExt;
Decoder {
inner: Inner::Pending(Pending(IoStream(body.into_stream()).peekable())),
inner: Inner::Pending(Pending(
IoStream(body.into_stream()).peekable(),
DecoderType::Gzip,
)),
}
}
/// Constructs a Decoder from a hyper request.
/// A brotli decoder.
///
/// A decoder is just a wrapper around the hyper request that knows
/// how to decode the content body of the request.
///
/// Uses the correct variant by inspecting the Content-Encoding header.
pub(crate) fn detect(headers: &mut HeaderMap, body: Body, check_gzip: bool) -> Decoder {
if !check_gzip {
return Decoder::plain_text(body);
/// This decoder will buffer and decompress chunks that are brotlied.
#[cfg(feature = "brotli")]
fn brotli(body: Body) -> Decoder {
use futures_util::StreamExt;
Decoder {
inner: Inner::Pending(Pending(
IoStream(body.into_stream()).peekable(),
DecoderType::Brotli,
)),
}
}
#[cfg(feature = "gzip")]
fn detect_gzip(headers: &mut HeaderMap) -> bool {
use http::header::{CONTENT_ENCODING, CONTENT_LENGTH, TRANSFER_ENCODING};
use log::warn;
let content_encoding_gzip: bool;
let mut is_gzip = {
content_encoding_gzip = headers
@@ -146,15 +134,76 @@ mod imp {
}
}
}
if content_encoding_gzip {
if is_gzip {
headers.remove(CONTENT_ENCODING);
headers.remove(CONTENT_LENGTH);
}
if is_gzip {
Decoder::gzip(body)
} else {
Decoder::plain_text(body)
is_gzip
}
#[cfg(feature = "brotli")]
fn detect_brotli(headers: &mut HeaderMap) -> bool {
use http::header::{CONTENT_ENCODING, CONTENT_LENGTH, TRANSFER_ENCODING};
use log::warn;
let content_encoding_gzip: bool;
let mut is_brotli = {
content_encoding_gzip = headers
.get_all(CONTENT_ENCODING)
.iter()
.any(|enc| enc == "br");
content_encoding_gzip
|| headers
.get_all(TRANSFER_ENCODING)
.iter()
.any(|enc| enc == "br")
};
if is_brotli {
if let Some(content_length) = headers.get(CONTENT_LENGTH) {
if content_length == "0" {
warn!("brotli response with content-length of 0");
is_brotli = false;
}
}
}
if is_brotli {
headers.remove(CONTENT_ENCODING);
headers.remove(CONTENT_LENGTH);
}
is_brotli
}
/// Constructs a Decoder from a hyper request.
///
/// A decoder is just a wrapper around the hyper request that knows
/// how to decode the content body of the request.
///
/// Uses the correct variant by inspecting the Content-Encoding header.
pub(crate) fn detect(
_headers: &mut HeaderMap,
body: Body,
check_gzip: bool,
check_brotli: bool,
) -> Decoder {
if !check_gzip && !check_brotli {
return Decoder::plain_text(body);
}
#[cfg(feature = "gzip")]
{
if Decoder::detect_gzip(_headers) {
return Decoder::gzip(body);
}
}
#[cfg(feature = "brotli")]
{
if Decoder::detect_brotli(_headers) {
return Decoder::brotli(body);
}
}
Decoder::plain_text(body)
}
}
@@ -163,26 +212,36 @@ mod imp {
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
// Do a read or poll for a pending decoder value.
let new_value = match self.inner {
match self.inner {
#[cfg(any(feature = "brotli", feature = "gzip"))]
Inner::Pending(ref mut future) => match Pin::new(future).poll(cx) {
Poll::Ready(Ok(inner)) => inner,
Poll::Ready(Ok(inner)) => {
self.inner = inner;
return self.poll_next(cx);
}
Poll::Ready(Err(e)) => {
return Poll::Ready(Some(Err(crate::error::decode_io(e))))
return Poll::Ready(Some(Err(crate::error::decode_io(e))));
}
Poll::Pending => return Poll::Pending,
},
Inner::PlainText(ref mut body) => return Pin::new(body).poll_next(cx),
#[cfg(feature = "gzip")]
Inner::Gzip(ref mut decoder) => {
return match futures_core::ready!(Pin::new(decoder).poll_next(cx)) {
Some(Ok(bytes)) => Poll::Ready(Some(Ok(bytes))),
Some(Err(err)) => Poll::Ready(Some(Err(crate::error::decode_io(err)))),
None => Poll::Ready(None),
}
};
}
#[cfg(feature = "brotli")]
Inner::Brotli(ref mut decoder) => {
return match futures_core::ready!(Pin::new(decoder).poll_next(cx)) {
Some(Ok(bytes)) => Poll::Ready(Some(Ok(bytes))),
Some(Err(err)) => Poll::Ready(Some(Err(crate::error::decode_io(err)))),
None => Poll::Ready(None),
};
}
};
self.inner = new_value;
self.poll_next(cx)
}
}
@@ -207,11 +266,17 @@ mod imp {
None => return Poll::Ready(Ok(Inner::PlainText(Body::empty().into_stream()))),
};
let body = mem::replace(
let _body = std::mem::replace(
&mut self.0,
IoStream(Body::empty().into_stream()).peekable(),
);
Poll::Ready(Ok(Inner::Gzip(GzipDecoder::new(body))))
match self.1 {
#[cfg(feature = "brotli")]
DecoderType::Brotli => Poll::Ready(Ok(Inner::Brotli(BrotliDecoder::new(_body)))),
#[cfg(feature = "gzip")]
DecoderType::Gzip => Poll::Ready(Ok(Inner::Gzip(GzipDecoder::new(_body)))),
}
}
}

6
src/async_impl/response.rs
View File

@@ -39,6 +39,7 @@ impl Response {
res: hyper::Response<hyper::Body>,
url: Url,
gzip: bool,
brotli: bool,
timeout: Option<Delay>,
) -> Response {
let (parts, body) = res.into_parts();
@@ -47,7 +48,7 @@ impl Response {
let extensions = parts.extensions;
let mut headers = parts.headers;
let decoder = Decoder::detect(&mut headers, Body::response(body, timeout), gzip);
let decoder = Decoder::detect(&mut headers, Body::response(body, timeout), gzip, brotli);
Response {
status,
@@ -404,7 +405,7 @@ impl<T: Into<Body>> From<http::Response<T>> for Response {
fn from(r: http::Response<T>) -> Response {
let (mut parts, body) = r.into_parts();
let body = body.into();
let body = Decoder::detect(&mut parts.headers, body, false);
let body = Decoder::detect(&mut parts.headers, body, false, false);
let url = parts
.extensions
.remove::<ResponseUrl>()
@@ -440,7 +441,6 @@ pub trait ResponseBuilderExt {
fn url(self, url: Url) -> Self;
}
impl ResponseBuilderExt for http::response::Builder {
fn url(self, url: Url) -> Self {
self.extension(ResponseUrl(url))