forked/reqwest-impersonate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add impersonation capabilites

Browse Source 
Update lib.rs
This commit is contained in:
4JX
2022-08-10 18:18:22 +02:00
parent c2a1870a3e
commit b22267618c
12 changed files with 710 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -1,3 +1,5 @@
target
Cargo.lock
*.swp
.history
.vscode

85
Cargo.toml
View File

@@ -1,7 +1,7 @@
[package]
name = "reqwest"
version = "0.11.13" # remember to update html_root_url
description = "higher level HTTP client library"
name = "reqwest-impersonate"
version = "0.11.13" # remember to update html_root_url
description = "A reqwest fork that impersonates the Chrome browser"
keywords = ["http", "request", "client"]
categories = ["web-programming::http-client", "wasm"]
repository = "https://github.com/seanmonstar/reqwest"
@@ -18,12 +18,7 @@ rustdoc-args = ["--cfg", "docsrs"]
targets = ["x86_64-unknown-linux-gnu", "wasm32-unknown-unknown"]
[package.metadata.playground]
features = [
"blocking",
"cookies",
"json",
"multipart",
]
features = ["blocking", "cookies", "json", "multipart"]
[features]
default = ["default-tls"]
@@ -42,7 +37,12 @@ rustls-tls-manual-roots = ["__rustls"]
rustls-tls-webpki-roots = ["webpki-roots", "__rustls"]
rustls-tls-native-roots = ["rustls-native-certs", "__rustls"]
boring-tls = ["__boring"]
blocking = ["futures-util/io", "tokio/rt-multi-thread", "tokio/sync"]
chrome = ["__chrome"]
cookies = ["cookie_crate", "cookie_store"]
@@ -72,6 +72,19 @@ __tls = []
# Equivalent to rustls-tls-manual-roots but shorter :)
__rustls = ["hyper-rustls", "tokio-rustls", "rustls", "__tls", "rustls-pemfile"]
__boring = [
"boring",
"tokio-boring",
"hyper-boring",
"__tls",
"boring-sys",
"foreign-types",
]
__chrome = ["__boring", "__browser_common"]
__browser_common = ["brotli", "gzip"]
# When enabled, disable using the cached SYS_PROXIES.
__internal_proxy_sys_no_cache = []
@@ -96,13 +109,22 @@ encoding_rs = "0.8"
futures-core = { version = "0.3.0", default-features = false }
futures-util = { version = "0.3.0", default-features = false }
http-body = "0.4.0"
hyper = { version = "0.14.18", default-features = false, features = ["tcp", "http1", "http2", "client", "runtime"] }
hyper = { version = "0.14.18", default-features = false, features = [
"tcp",
"http1",
"http2",
"client",
"runtime",
] }
h2 = "0.3.10"
once_cell = "1"
log = "0.4"
mime = "0.3.16"
percent-encoding = "2.1"
tokio = { version = "1.0", default-features = false, features = ["net", "time"] }
tokio = { version = "1.0", default-features = false, features = [
"net",
"time",
] }
pin-project-lite = "0.2.0"
ipnet = "2.3"
@@ -115,19 +137,33 @@ tokio-native-tls = { version = "0.3.0", optional = true }
# rustls-tls
hyper-rustls = { version = "0.23", default-features = false, optional = true }
rustls = { version = "0.20", features = ["dangerous_configuration"], optional = true }
rustls = { version = "0.20", features = [
"dangerous_configuration",
], optional = true }
tokio-rustls = { version = "0.23", optional = true }
webpki-roots = { version = "0.22", optional = true }
rustls-native-certs = { version = "0.6", optional = true }
rustls-pemfile = { version = "1.0", optional = true }
## boring-tls
hyper-boring = { git = "https://github.com/4JX/boring", rev = "2a7463a", optional = true }
boring = { git = "https://github.com/4JX/boring", rev = "2a7463a", optional = true }
tokio-boring = { git = "https://github.com/4JX/boring", rev = "2a7463a", optional = true }
boring-sys = { git = "https://github.com/4JX/boring", rev = "2a7463a", optional = true }
foreign-types = { version = "0.5.0", optional = true }
## cookies
cookie_crate = { version = "0.16", package = "cookie", optional = true }
cookie_store = { version = "0.16", optional = true }
## compression
async-compression = { version = "0.3.13", default-features = false, features = ["tokio"], optional = true }
tokio-util = { version = "0.7.1", default-features = false, features = ["codec", "io"], optional = true }
async-compression = { version = "0.3.13", default-features = false, features = [
"tokio",
], optional = true }
tokio-util = { version = "0.7.1", default-features = false, features = [
"codec",
"io",
], optional = true }
## socks
tokio-socks = { version = "0.5.1", optional = true }
@@ -137,12 +173,23 @@ trust-dns-resolver = { version = "0.22", optional = true }
[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
env_logger = "0.8"
hyper = { version = "0.14", default-features = false, features = ["tcp", "stream", "http1", "http2", "client", "server", "runtime"] }
hyper = { version = "0.14", default-features = false, features = [
"tcp",
"stream",
"http1",
"http2",
"client",
"server",
"runtime",
] }
serde = { version = "1.0", features = ["derive"] }
libflate = "1.0"
brotli_crate = { package = "brotli", version = "3.3.0" }
doc-comment = "0.3"
tokio = { version = "1.0", default-features = false, features = ["macros", "rt-multi-thread"] }
tokio = { version = "1.0", default-features = false, features = [
"macros",
"rt-multi-thread",
] }
[target.'cfg(windows)'.dependencies]
winreg = "0.10"
@@ -169,7 +216,7 @@ features = [
"BlobPropertyBag",
"ServiceWorkerGlobalScope",
"RequestCredentials",
"File"
"File",
]
[target.'cfg(target_arch = "wasm32")'.dev-dependencies]
@@ -233,3 +280,7 @@ required-features = ["deflate"]
name = "multipart"
path = "tests/multipart.rs"
required-features = ["multipart"]
[patch.crates-io]
hyper = { git = "https://github.com/4JX/hyper.git", branch = "0.14.x-patched", ref = "bd25359" }
h2 = { git = "https://github.com/4JX/h2.git", ref = "b088466" }

63
README.md
View File

@@ -1,4 +1,57 @@
# reqwest
# reqwest-impersonate
A fork of reqwest used to impersonate the Chrome browser. Inspired by [curl-impersonate](https://github.com/lwthiker/curl-impersonate).
This crate was intended to be an experiment to learn more about TLS and HTTP2 fingerprinting. Some parts of reqwest may not have the code needed to work when used to copy Chrome.
It is currently missing HTTP/2 `PRIORITY` support. (PRs to [h2](https://github.com/hyperium/h2) are welcome)
**Notice:** This crate depends on patched dependencies. To use it, please add the following to your `Cargo.toml`.
```toml
[patch.crates-io]
hyper = { git = "https://github.com/4JX/hyper.git", branch = "0.14.x-patched" }
h2 = { git = "https://github.com/4JX/h2.git", branch = "imp" }
```
These patches were made specifically for `reqwest-impersonate` to work, but I would appreciate if someone took the time to PR more "proper" versions to the parent projects.
## Example
`Cargo.toml`
```toml
reqwest-impersonate = { git = "https://github.com/4JX/reqwest-impersonate.git", default-features = false, features = [
"chrome",
"blocking",
] }
```
`main.rs`
```rs
use reqwest_impersonate::browser::ChromeVersion;
fn main() {
// Build a client to mimic Chrome 104
let client = reqwest_impersonate::blocking::Client::builder()
.chrome_builder(ChromeVersion::V104)
.build()
.unwrap();
// Use the API you're already familiar with
match client.get("https://yoururl.com").send() {
Ok(res) => {
println!("{:?}", res.text().unwrap());
}
Err(err) => {
dbg!(err);
}
};
}
```
## Original readme
[![crates.io](https://img.shields.io/crates/v/reqwest.svg)](https://crates.io/crates/reqwest)
[![Documentation](https://docs.rs/reqwest/badge.svg)](https://docs.rs/reqwest)
@@ -15,7 +68,6 @@ An ergonomic, batteries-included HTTP Client for Rust.
- WASM
- [Changelog](CHANGELOG.md)
## Example
This asynchronous example uses [Tokio](https://tokio.rs) and enables some
@@ -67,7 +119,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
On Linux:
- OpenSSL 1.0.1, 1.0.2, 1.1.0, or 1.1.1 with headers (see https://github.com/sfackler/rust-openssl)
- OpenSSL 1.0.1, 1.0.2, 1.1.0, or 1.1.1 with headers (see <https://github.com/sfackler/rust-openssl>)
On Windows and macOS:
@@ -77,13 +129,12 @@ Reqwest uses [rust-native-tls](https://github.com/sfackler/rust-native-tls),
which will use the operating system TLS framework if available, meaning Windows
and macOS. On Linux, it will use OpenSSL 1.1.
## License
Licensed under either of
- Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://apache.org/licenses/LICENSE-2.0)
- MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
- Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or <http://apache.org/licenses/LICENSE-2.0>)
- MIT license ([LICENSE-MIT](LICENSE-MIT) or <http://opensource.org/licenses/MIT>)
### Contribution

97
src/async_impl/client.rs
View File

@@ -28,7 +28,9 @@ use super::decoder::Accepts;
use super::request::{Request, RequestBuilder};
use super::response::Response;
use super::Body;
use crate::connect::Connector;
#[cfg(feature = "__chrome")]
use crate::browser::{configure_chrome, ChromeVersion};
use crate::connect::{Connector};
#[cfg(feature = "cookies")]
use crate::cookie;
#[cfg(feature = "trust-dns")]
@@ -115,6 +117,10 @@ struct Config {
http2_initial_connection_window_size: Option<u32>,
http2_adaptive_window: bool,
http2_max_frame_size: Option<u32>,
http2_max_concurrent_streams: Option<u32>,
http2_max_header_list_size: Option<u32>,
http2_enable_push: Option<bool>,
http2_header_table_size: Option<u32>,
http2_keep_alive_interval: Option<Duration>,
http2_keep_alive_timeout: Option<Duration>,
http2_keep_alive_while_idle: bool,
@@ -186,6 +192,10 @@ impl ClientBuilder {
http2_initial_connection_window_size: None,
http2_adaptive_window: false,
http2_max_frame_size: None,
http2_max_concurrent_streams: None,
http2_max_header_list_size: None,
http2_enable_push: None,
http2_header_table_size: None,
http2_keep_alive_interval: None,
http2_keep_alive_timeout: None,
http2_keep_alive_while_idle: false,
@@ -201,6 +211,11 @@ impl ClientBuilder {
}
}
/// Sets the necessary values to mimic the specified Chrome version.
#[cfg(feature = "__chrome")]
pub fn chrome_builder(self, ver: ChromeVersion) -> ClientBuilder {
configure_chrome(ver, self)
}
/// Returns a `Client` that uses this `ClientBuilder` configuration.
///
/// # Errors
@@ -246,6 +261,15 @@ impl ClientBuilder {
#[cfg(feature = "__tls")]
match config.tls {
#[cfg(feature = "__boring")]
TlsBackend::BoringTls(tls) => Connector::new_boring_tls(
http,
tls,
proxies.clone(),
user_agent(&config.headers),
config.local_address,
config.nodelay,
),
#[cfg(feature = "default-tls")]
TlsBackend::Default => {
let mut tls = TlsConnector::builder();
@@ -459,7 +483,7 @@ impl ClientBuilder {
config.nodelay,
)
}
#[cfg(any(feature = "native-tls", feature = "__rustls",))]
#[cfg(any(feature = "native-tls", feature = "__rustls"))]
TlsBackend::UnknownPreconfigured => {
return Err(crate::error::builder(
"Unknown TLS backend passed to `use_preconfigured_tls`",
@@ -493,6 +517,18 @@ impl ClientBuilder {
if let Some(http2_max_frame_size) = config.http2_max_frame_size {
builder.http2_max_frame_size(http2_max_frame_size);
}
if let Some(max) = config.http2_max_concurrent_streams {
builder.http2_max_concurrent_streams(max);
}
if let Some(max) = config.http2_max_header_list_size {
builder.http2_max_header_list_size(max);
}
if let Some(opt) = config.http2_enable_push {
builder.http2_enable_push(opt);
}
if let Some(max) = config.http2_header_table_size {
builder.http2_header_table_size(max);
}
if let Some(http2_keep_alive_interval) = config.http2_keep_alive_interval {
builder.http2_keep_alive_interval(http2_keep_alive_interval);
}
@@ -628,6 +664,12 @@ impl ClientBuilder {
self
}
#[cfg(feature = "__browser_common")]
pub(crate) fn replace_default_headers(mut self, headers: HeaderMap) -> ClientBuilder {
self.config.headers = headers;
self
}
/// Enable a persistent cookie store for the client.
///
/// Cookies received in responses will be preserved and included in
@@ -965,6 +1007,39 @@ impl ClientBuilder {
self
}
/// Sets the maximum concurrent streams to use for HTTP2.
///
/// Passing `None` will do nothing.
pub fn http2_max_concurrent_streams(mut self, sz: impl Into<Option<u32>>) -> ClientBuilder {
self.config.http2_max_concurrent_streams = sz.into();
self
}
/// Sets the max header list size to use for HTTP2.
///
/// Passing `None` will do nothing.
pub fn http2_max_header_list_size(mut self, sz: impl Into<Option<u32>>) -> ClientBuilder {
self.config.http2_max_header_list_size = sz.into();
self
}
/// Enables and disables the push feature for HTTP2.
///
/// Passing `None` will do nothing.
pub fn http2_enable_push(mut self, sz: impl Into<Option<bool>>) -> ClientBuilder {
self.config.http2_enable_push = sz.into();
self
}
/// Sets the header table size to use for HTTP2.
///
/// Passing `None` will do nothing.
pub fn http2_header_table_size(mut self, sz: impl Into<Option<u32>>) -> ClientBuilder {
self.config.http2_header_table_size = sz.into();
self
}
/// Sets an interval for HTTP2 Ping frames should be sent to keep a connection alive.
///
/// Pass `None` to disable HTTP2 keep-alive.
@@ -1262,6 +1337,24 @@ impl ClientBuilder {
self
}
/// Force using the Boring TLS backend.
///
/// Since multiple TLS backends can be optionally enabled, this option will
/// force the `boring` backend to be used for this `Client`.
///
/// # Optional
///
/// This requires the optional `boring-tls(-...)` feature to be enabled.
#[cfg(feature = "__boring")]
#[cfg_attr(docsrs, doc(cfg(feature = "boring-tls")))]
pub fn use_boring_tls(
mut self,
builder_func: Arc<dyn Fn() -> boring::ssl::SslConnectorBuilder + Send + Sync>,
) -> ClientBuilder {
self.config.tls = TlsBackend::BoringTls(builder_func);
self
}
/// Use a preconfigured TLS backend.
///
/// If the passed `Any` argument is not a TLS backend that reqwest

54
src/blocking/client.rs
View File

@@ -16,6 +16,8 @@ use tokio::sync::{mpsc, oneshot};
use super::request::{Request, RequestBuilder};
use super::response::Response;
use super::wait;
#[cfg(feature = "__chrome")]
use crate::browser::ChromeVersion;
#[cfg(feature = "__tls")]
use crate::tls;
#[cfg(feature = "__tls")]
@@ -88,6 +90,12 @@ impl ClientBuilder {
}
}
/// Sets the necessary values to mimic the specified Chrome version.
#[cfg(feature = "__chrome")]
pub fn chrome_builder(self, ver: ChromeVersion) -> ClientBuilder {
self.with_inner(move |inner| inner.chrome_builder(ver))
}
/// Returns a `Client` that uses this `ClientBuilder` configuration.
///
/// # Errors
@@ -464,6 +472,35 @@ impl ClientBuilder {
self.with_inner(|inner| inner.http2_max_frame_size(sz))
}
/// Sets the maximum concurrent streams to use for HTTP2.
///
/// Passing `None` will do nothing.
pub fn http2_max_concurrent_streams(self, sz: impl Into<Option<u32>>) -> ClientBuilder {
self.with_inner(|inner| inner.http2_max_concurrent_streams(sz))
}
/// Sets the max header list size to use for HTTP2.
///
/// Passing `None` will do nothing.
pub fn http2_max_header_list_size(self, sz: impl Into<Option<u32>>) -> ClientBuilder {
self.with_inner(|inner| inner.http2_max_header_list_size(sz))
}
/// Enables and disables the push feature for HTTP2.
///
/// Passing `None` will do nothing.
pub fn http2_enable_push(self, sz: impl Into<Option<bool>>) -> ClientBuilder {
self.with_inner(|inner| inner.http2_enable_push(sz))
}
/// Sets the header table size to use for HTTP2.
///
/// Passing `None` will do nothing.
pub fn http2_header_table_size(self, sz: impl Into<Option<u32>>) -> ClientBuilder {
self.with_inner(|inner| inner.http2_header_table_size(sz))
}
// TCP options
/// Set whether sockets have `TCP_NODELAY` enabled.
@@ -724,6 +761,23 @@ impl ClientBuilder {
self.with_inner(move |inner| inner.use_rustls_tls())
}
/// Force using the Boring TLS backend.
///
/// Since multiple TLS backends can be optionally enabled, this option will
/// force the `boring` backend to be used for this `Client`.
///
/// # Optional
///
/// This requires the optional `boring-tls(-...)` feature to be enabled.
#[cfg(feature = "__boring")]
#[cfg_attr(docsrs, doc(cfg(feature = "boring-tls")))]
pub fn use_boring_tls(
self,
builder_func: Arc<dyn Fn() -> boring::ssl::SslConnectorBuilder + Send + Sync>,
) -> ClientBuilder {
self.with_inner(move |inner| inner.use_boring_tls(builder_func))
}
/// Use a preconfigured TLS backend.
///
/// If the passed `Any` argument is not a TLS backend that reqwest

27
src/browser/chrome/mod.rs Normal file
View File

@@ -0,0 +1,27 @@
//! Settings for impersonating the Chrome browser
use crate::ClientBuilder;
mod ver;
pub(crate) fn configure_chrome(ver: ChromeVersion, builder: ClientBuilder) -> ClientBuilder {
let settings = ver::get_config_from_ver(ver);
builder
.use_boring_tls(settings.tls_builder_func)
.http2_initial_stream_window_size(settings.http2.initial_stream_window_size)
.http2_initial_connection_window_size(settings.http2.initial_connection_window_size)
.http2_max_concurrent_streams(settings.http2.max_concurrent_streams)
.http2_max_header_list_size(settings.http2.max_header_list_size)
.http2_header_table_size(settings.http2.header_table_size)
.replace_default_headers(settings.headers)
.brotli(settings.brotli)
.gzip(settings.gzip)
}
/// Defines the Chrome version to mimic when setting up a builder
#[derive(Debug)]
#[allow(missing_docs)]
pub enum ChromeVersion {
V104,
}

11
src/browser/chrome/ver/mod.rs Normal file
View File

@@ -0,0 +1,11 @@
use crate::browser::BrowserSettings;
use super::ChromeVersion;
mod v104;
pub(super) fn get_config_from_ver(ver: ChromeVersion) -> BrowserSettings {
match ver {
ChromeVersion::V104 => v104::get_settings(),
}
}

110
src/browser/chrome/ver/v104.rs Normal file
View File

@@ -0,0 +1,110 @@
use std::sync::Arc;
use boring::ssl::{
CertCompressionAlgorithm, SslConnector, SslConnectorBuilder, SslMethod, SslVersion,
};
use http::{
header::{ACCEPT, ACCEPT_ENCODING, ACCEPT_LANGUAGE, UPGRADE_INSECURE_REQUESTS, USER_AGENT},
HeaderMap,
};
use crate::browser::{BrowserSettings, Http2Data};
pub(super) fn get_settings() -> BrowserSettings {
BrowserSettings {
tls_builder_func: Arc::new(create_ssl_connector),
http2: Http2Data {
initial_stream_window_size: 6291456,
initial_connection_window_size: 15728640,
max_concurrent_streams: 1000,
max_header_list_size: 262144,
header_table_size: 65536,
},
headers: create_headers(),
gzip: true,
brotli: true,
}
}
fn create_ssl_connector() -> SslConnectorBuilder {
let mut builder = SslConnector::builder(SslMethod::tls()).unwrap();
builder.set_grease_enabled(true);
builder.enable_ocsp_stapling();
let cipher_list = [
"TLS_AES_128_GCM_SHA256",
"TLS_AES_256_GCM_SHA384",
"TLS_CHACHA20_POLY1305_SHA256",
"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256",
"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256",
"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA",
"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",
"TLS_RSA_WITH_AES_128_GCM_SHA256",
"TLS_RSA_WITH_AES_256_GCM_SHA384",
"TLS_RSA_WITH_AES_128_CBC_SHA",
"TLS_RSA_WITH_AES_256_CBC_SHA",
];
builder.set_cipher_list(&cipher_list.join(":")).unwrap();
let sigalgs_list = [
"ecdsa_secp256r1_sha256",
"rsa_pss_rsae_sha256",
"rsa_pkcs1_sha256",
"ecdsa_secp384r1_sha384",
"rsa_pss_rsae_sha384",
"rsa_pkcs1_sha384",
"rsa_pss_rsae_sha512",
"rsa_pkcs1_sha512",
];
builder.set_sigalgs_list(&sigalgs_list.join(":")).unwrap();
builder.enable_signed_cert_timestamps();
builder.set_alpn_protos(b"\x02h2\x08http/1.1").unwrap();
builder
.add_cert_compression_alg(CertCompressionAlgorithm::Brotli)
.unwrap();
builder
.set_min_proto_version(Some(SslVersion::TLS1_2))
.unwrap();
builder
.set_max_proto_version(Some(SslVersion::TLS1_3))
.unwrap();
builder
}
fn create_headers() -> HeaderMap {
let mut headers = HeaderMap::new();
headers.insert(
"sec-ch-ua",
"\"Chromium\";v=\"104\", \" Not A;Brand\";v=\"99\", \"Google Chrome\";v=\"104\""
.parse()
.unwrap(),
);
headers.insert("sec-ch-ua-mobile", "?0".parse().unwrap());
headers.insert("sec-ch-ua-platform", "\"Windows\"".parse().unwrap());
headers.insert(UPGRADE_INSECURE_REQUESTS, "1".parse().unwrap());
headers.insert(USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36".parse().unwrap());
headers.insert(ACCEPT, "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9".parse().unwrap());
headers.insert("sec-fetch-site", "none".parse().unwrap());
headers.insert("sec-fetch-mode", "navigate".parse().unwrap());
headers.insert("sec-fetch-user", "?1".parse().unwrap());
headers.insert("sec-fetch-dest", "document".parse().unwrap());
headers.insert(ACCEPT_ENCODING, "gzip, deflate, br".parse().unwrap());
headers.insert(ACCEPT_LANGUAGE, "en-US,en;q=0.9".parse().unwrap());
headers
}

31
src/browser/mod.rs Normal file
View File

@@ -0,0 +1,31 @@
//! Holds structs and information to aid in impersonating a set of browsers
use std::sync::Arc;
use boring::ssl::SslConnectorBuilder;
use http::HeaderMap;
#[cfg(feature = "__chrome")]
pub use chrome::ChromeVersion;
#[cfg(feature = "__chrome")]
mod chrome;
#[cfg(feature = "__chrome")]
pub(crate) use chrome::configure_chrome;
struct BrowserSettings {
pub tls_builder_func: Arc<dyn Fn() -> SslConnectorBuilder + Send + Sync>,
pub http2: Http2Data,
pub headers: HeaderMap,
pub gzip: bool,
pub brotli: bool,
}
struct Http2Data {
pub initial_stream_window_size: u32,
pub initial_connection_window_size: u32,
pub max_concurrent_streams: u32,
pub max_header_list_size: u32,
pub header_table_size: u32,
}

228
src/connect.rs
View File

@@ -1,3 +1,8 @@
#[cfg(feature = "__boring")]
use boring::ssl::{ConnectConfiguration, SslConnectorBuilder};
#[cfg(feature = "__boring")]
use foreign_types::ForeignTypeRef;
use futures_util::future::Either;
#[cfg(feature = "__tls")]
use http::header::HeaderValue;
use http::uri::{Authority, Scheme};
@@ -17,6 +22,8 @@ use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::Duration;
#[cfg(feature = "__boring")]
use self::boring_tls_conn::BoringTlsConn;
#[cfg(feature = "default-tls")]
use self::native_tls_conn::NativeTlsConn;
#[cfg(feature = "__rustls")]
@@ -51,6 +58,28 @@ enum Inner {
tls: Arc<rustls::ClientConfig>,
tls_proxy: Arc<rustls::ClientConfig>,
},
#[cfg(feature = "__boring")]
BoringTls {
http: HttpConnector,
tls: Arc<dyn Fn() -> SslConnectorBuilder + Send + Sync>,
},
}
#[cfg(feature = "__boring")]
fn tls_add_application_settings(conf: &mut ConnectConfiguration) {
// curl-impersonate does not know how to set this up, neither do I. Hopefully nothing breaks with these values.
const ALPN_H2: &str = "h2";
const ALPN_H2_LENGTH: usize = 2;
unsafe {
boring_sys::SSL_add_application_settings(
conf.as_ptr(),
ALPN_H2.as_ptr(),
ALPN_H2_LENGTH,
std::ptr::null(),
0,
)
};
}
impl Connector {
@@ -117,6 +146,31 @@ impl Connector {
}
}
#[cfg(feature = "__boring")]
pub(crate) fn new_boring_tls<T>(
mut http: HttpConnector,
tls: Arc<dyn Fn() -> SslConnectorBuilder + Send + Sync>,
proxies: Arc<Vec<Proxy>>,
user_agent: Option<HeaderValue>,
local_addr: T,
nodelay: bool,
) -> Connector
where
T: Into<Option<IpAddr>>,
{
http.set_local_address(local_addr.into());
http.enforce_http(false);
Connector {
inner: Inner::BoringTls { http, tls },
proxies,
verbose: verbose::OFF,
timeout: None,
nodelay,
user_agent,
}
}
#[cfg(feature = "__rustls")]
pub(crate) fn new_rustls_tls<T>(
mut http: HttpConnector,
@@ -211,6 +265,23 @@ impl Connector {
});
}
}
#[cfg(feature = "__boring")]
Inner::BoringTls { tls, .. } => {
if dst.scheme() == Some(&Scheme::HTTPS) {
let host = dst.host().ok_or("no host in url")?.to_string();
let conn = socks::connect(proxy, dst, dns).await?;
let tls_connector = tls().build();
let mut conf = tls_connector.configure()?;
tls_add_application_settings(&mut conf);
let io = tokio_boring::connect(conf, &host, conn).await?;
return Ok(Conn {
inner: self.verbose.wrap(BoringTlsConn { inner: io }),
is_proxy: false,
});
}
}
#[cfg(not(feature = "__tls"))]
Inner::Http(_) => (),
}
@@ -291,6 +362,42 @@ impl Connector {
})
}
}
#[cfg(feature = "__boring")]
Inner::BoringTls { http, tls } => {
let mut http = http.clone();
// Disable Nagle's algorithm for TLS handshake
//
// https://www.openssl.org/docs/man1.1.1/man3/SSL_connect.html#NOTES
if !self.nodelay && (dst.scheme() == Some(&Scheme::HTTPS)) {
http.set_nodelay(true);
}
let mut http = hyper_boring::HttpsConnector::with_connector(http, tls())?;
http.set_callback(|conf, _| {
tls_add_application_settings(conf);
Ok(())
});
let io = http.call(dst).await?;
if let hyper_boring::MaybeHttpsStream::Https(stream) = io {
if !self.nodelay {
let stream_ref = stream.get_ref();
stream_ref.set_nodelay(false)?;
}
Ok(Conn {
inner: self.verbose.wrap(BoringTlsConn { inner: stream }),
is_proxy,
})
} else {
Ok(Conn {
inner: self.verbose.wrap(io),
is_proxy,
})
}
}
}
}
@@ -372,6 +479,45 @@ impl Connector {
});
}
}
#[cfg(feature = "__boring")]
Inner::BoringTls { http, tls } => {
if dst.scheme() == Some(&Scheme::HTTPS) {
let host = dst.host().to_owned();
let port = dst.port().map(|p| p.as_u16()).unwrap_or(443);
let http = http.clone();
let tls_connector = tls();
let mut http =
hyper_boring::HttpsConnector::with_connector(http, tls_connector)?;
http.set_callback(|conf, _| {
tls_add_application_settings(conf);
Ok(())
});
let conn = http.call(proxy_dst).await?;
log::trace!("tunneling HTTPS over proxy");
let tunneled = tunnel(
conn,
host.ok_or("no host in url")?.to_string(),
port,
self.user_agent.clone(),
auth,
)
.await?;
let tls_connector = tls().build();
let mut conf = tls_connector.configure()?;
tls_add_application_settings(&mut conf);
let io = tokio_boring::connect(conf, host.ok_or("no host in url")?, tunneled)
.await?;
return Ok(Conn {
inner: self.verbose.wrap(BoringTlsConn { inner: io }),
is_proxy: false,
});
}
}
#[cfg(not(feature = "__tls"))]
Inner::Http(_) => (),
}
@@ -387,6 +533,8 @@ impl Connector {
Inner::RustlsTls { http, .. } => http.set_keepalive(dur),
#[cfg(not(feature = "__tls"))]
Inner::Http(http) => http.set_keepalive(dur),
#[cfg(feature = "__boring")]
Inner::BoringTls { http, .. } => http.set_keepalive(dur),
}
}
}
@@ -764,6 +912,86 @@ mod rustls_tls_conn {
}
}
#[cfg(feature = "__boring")]
mod boring_tls_conn {
use hyper::client::connect::{Connected, Connection};
use pin_project_lite::pin_project;
use std::{
io::{self, IoSlice},
pin::Pin,
task::{Context, Poll},
};
use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
use tokio_boring::SslStream;
pin_project! {
pub(super) struct BoringTlsConn<T> {
#[pin] pub(super) inner: SslStream<T>,
}
}
impl<T: Connection + AsyncRead + AsyncWrite + Unpin> Connection for BoringTlsConn<T> {
fn connected(&self) -> Connected {
if self.inner.ssl().selected_alpn_protocol() == Some(b"h2") {
self.inner.get_ref().connected().negotiated_h2()
} else {
self.inner.get_ref().connected()
}
}
}
impl<T: AsyncRead + AsyncWrite + Unpin> AsyncRead for BoringTlsConn<T> {
fn poll_read(
self: Pin<&mut Self>,
cx: &mut Context,
buf: &mut ReadBuf<'_>,
) -> Poll<tokio::io::Result<()>> {
let this = self.project();
AsyncRead::poll_read(this.inner, cx, buf)
}
}
impl<T: AsyncRead + AsyncWrite + Unpin> AsyncWrite for BoringTlsConn<T> {
fn poll_write(
self: Pin<&mut Self>,
cx: &mut Context,
buf: &[u8],
) -> Poll<Result<usize, tokio::io::Error>> {
let this = self.project();
AsyncWrite::poll_write(this.inner, cx, buf)
}
fn poll_write_vectored(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
bufs: &[IoSlice<'_>],
) -> Poll<Result<usize, io::Error>> {
let this = self.project();
AsyncWrite::poll_write_vectored(this.inner, cx, bufs)
}
fn is_write_vectored(&self) -> bool {
self.inner.is_write_vectored()
}
fn poll_flush(
self: Pin<&mut Self>,
cx: &mut Context,
) -> Poll<Result<(), tokio::io::Error>> {
let this = self.project();
AsyncWrite::poll_flush(this.inner, cx)
}
fn poll_shutdown(
self: Pin<&mut Self>,
cx: &mut Context,
) -> Poll<Result<(), tokio::io::Error>> {
let this = self.project();
AsyncWrite::poll_shutdown(this.inner, cx)
}
}
}
#[cfg(feature = "socks")]
mod socks {
use std::io;

7
src/lib.rs
View File

@@ -217,6 +217,11 @@ macro_rules! if_hyper {
)*}
}
/// Re-export of boring to keep versions in check
#[cfg(feature = "__boring")]
pub use boring;
#[cfg(feature = "__boring")]
pub use boring_sys;
pub use http::header;
pub use http::Method;
pub use http::{StatusCode, Version};
@@ -225,6 +230,8 @@ pub use url::Url;
// universal mods
#[macro_use]
mod error;
#[cfg(feature = "__browser_common")]
pub mod browser;
mod into_url;
mod response;

23
src/tls.rs
View File

@@ -10,13 +10,14 @@
//! [`Identity`][Identity] type.
//! - Various parts of TLS can also be configured or even disabled on the
//! `ClientBuilder`.
#[cfg(feature = "__rustls")]
use rustls::{
client::HandshakeSignatureValid, client::ServerCertVerified, client::ServerCertVerifier,
internal::msgs::handshake::DigitallySignedStruct, Error as TLSError, ServerName,
};
use std::fmt;
#[cfg(feature = "__boring")]
use std::sync::Arc;
/// Represents a server X509 certificate.
#[derive(Clone)]
@@ -71,6 +72,7 @@ impl Certificate {
/// # Ok(())
/// # }
/// ```
#[cfg(any(not(feature = "__boring"), feature = "native-tls-crate", feature = "__rustls"))]
pub fn from_der(der: &[u8]) -> crate::Result<Certificate> {
Ok(Certificate {
#[cfg(feature = "native-tls-crate")]
@@ -96,6 +98,7 @@ impl Certificate {
/// # Ok(())
/// # }
/// ```
#[cfg(any(not(feature = "__boring"), feature = "native-tls-crate", feature = "__rustls"))]
pub fn from_pem(pem: &[u8]) -> crate::Result<Certificate> {
Ok(Certificate {
#[cfg(feature = "native-tls-crate")]
@@ -387,13 +390,17 @@ pub(crate) enum TlsBackend {
Rustls,
#[cfg(feature = "__rustls")]
BuiltRustls(rustls::ClientConfig),
#[cfg(any(feature = "native-tls", feature = "__rustls",))]
#[cfg(feature = "__boring")]
BoringTls(Arc<dyn Fn() -> boring::ssl::SslConnectorBuilder + Send + Sync>),
#[cfg(any(feature = "native-tls", feature = "__rustls"))]
UnknownPreconfigured,
}
impl fmt::Debug for TlsBackend {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
#[cfg(feature = "__boring")]
TlsBackend::BoringTls(_) => write!(f, "BoringTls"),
#[cfg(feature = "default-tls")]
TlsBackend::Default => write!(f, "Default"),
#[cfg(feature = "native-tls")]
@@ -402,7 +409,7 @@ impl fmt::Debug for TlsBackend {
TlsBackend::Rustls => write!(f, "Rustls"),
#[cfg(feature = "__rustls")]
TlsBackend::BuiltRustls(_) => write!(f, "BuiltRustls"),
#[cfg(any(feature = "native-tls", feature = "__rustls",))]
#[cfg(any(feature = "native-tls", feature = "__rustls"))]
TlsBackend::UnknownPreconfigured => write!(f, "UnknownPreconfigured"),
}
}
@@ -419,6 +426,16 @@ impl Default for TlsBackend {
{
TlsBackend::Rustls
}
#[cfg(all(feature = "__boring", not(feature = "default-tls")))]
{
use boring::ssl::{SslConnector, SslConnectorBuilder, SslMethod};
fn create_builder() -> SslConnectorBuilder {
SslConnector::builder(SslMethod::tls()).unwrap()
}
TlsBackend::BoringTls(Arc::new(create_builder))
}
}
}