diff --git a/.gitignore b/.gitignore index d4f917d..ff679e8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ target Cargo.lock *.swp +.history +.vscode \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 528b539..93462f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "reqwest" -version = "0.11.13" # remember to update html_root_url -description = "higher level HTTP client library" +name = "reqwest-impersonate" +version = "0.11.13" # remember to update html_root_url +description = "A reqwest fork that impersonates the Chrome browser" keywords = ["http", "request", "client"] categories = ["web-programming::http-client", "wasm"] repository = "https://github.com/seanmonstar/reqwest" @@ -18,12 +18,7 @@ rustdoc-args = ["--cfg", "docsrs"] targets = ["x86_64-unknown-linux-gnu", "wasm32-unknown-unknown"] [package.metadata.playground] -features = [ - "blocking", - "cookies", - "json", - "multipart", -] +features = ["blocking", "cookies", "json", "multipart"] [features] default = ["default-tls"] @@ -42,7 +37,12 @@ rustls-tls-manual-roots = ["__rustls"] rustls-tls-webpki-roots = ["webpki-roots", "__rustls"] rustls-tls-native-roots = ["rustls-native-certs", "__rustls"] +boring-tls = ["__boring"] + + blocking = ["futures-util/io", "tokio/rt-multi-thread", "tokio/sync"] +chrome = ["__chrome"] + cookies = ["cookie_crate", "cookie_store"] @@ -72,6 +72,19 @@ __tls = [] # Equivalent to rustls-tls-manual-roots but shorter :) __rustls = ["hyper-rustls", "tokio-rustls", "rustls", "__tls", "rustls-pemfile"] +__boring = [ + "boring", + "tokio-boring", + "hyper-boring", + "__tls", + "boring-sys", + "foreign-types", +] + +__chrome = ["__boring", "__browser_common"] + +__browser_common = ["brotli", "gzip"] + # When enabled, disable using the cached SYS_PROXIES. __internal_proxy_sys_no_cache = [] @@ -96,13 +109,22 @@ encoding_rs = "0.8" futures-core = { version = "0.3.0", default-features = false } futures-util = { version = "0.3.0", default-features = false } http-body = "0.4.0" -hyper = { version = "0.14.18", default-features = false, features = ["tcp", "http1", "http2", "client", "runtime"] } +hyper = { version = "0.14.18", default-features = false, features = [ + "tcp", + "http1", + "http2", + "client", + "runtime", +] } h2 = "0.3.10" once_cell = "1" log = "0.4" mime = "0.3.16" percent-encoding = "2.1" -tokio = { version = "1.0", default-features = false, features = ["net", "time"] } +tokio = { version = "1.0", default-features = false, features = [ + "net", + "time", +] } pin-project-lite = "0.2.0" ipnet = "2.3" @@ -115,19 +137,33 @@ tokio-native-tls = { version = "0.3.0", optional = true } # rustls-tls hyper-rustls = { version = "0.23", default-features = false, optional = true } -rustls = { version = "0.20", features = ["dangerous_configuration"], optional = true } +rustls = { version = "0.20", features = [ + "dangerous_configuration", +], optional = true } tokio-rustls = { version = "0.23", optional = true } webpki-roots = { version = "0.22", optional = true } rustls-native-certs = { version = "0.6", optional = true } rustls-pemfile = { version = "1.0", optional = true } +## boring-tls +hyper-boring = { git = "https://github.com/4JX/boring", rev = "2a7463a", optional = true } +boring = { git = "https://github.com/4JX/boring", rev = "2a7463a", optional = true } +tokio-boring = { git = "https://github.com/4JX/boring", rev = "2a7463a", optional = true } +boring-sys = { git = "https://github.com/4JX/boring", rev = "2a7463a", optional = true } +foreign-types = { version = "0.5.0", optional = true } + ## cookies cookie_crate = { version = "0.16", package = "cookie", optional = true } cookie_store = { version = "0.16", optional = true } ## compression -async-compression = { version = "0.3.13", default-features = false, features = ["tokio"], optional = true } -tokio-util = { version = "0.7.1", default-features = false, features = ["codec", "io"], optional = true } +async-compression = { version = "0.3.13", default-features = false, features = [ + "tokio", +], optional = true } +tokio-util = { version = "0.7.1", default-features = false, features = [ + "codec", + "io", +], optional = true } ## socks tokio-socks = { version = "0.5.1", optional = true } @@ -137,12 +173,23 @@ trust-dns-resolver = { version = "0.22", optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] env_logger = "0.8" -hyper = { version = "0.14", default-features = false, features = ["tcp", "stream", "http1", "http2", "client", "server", "runtime"] } +hyper = { version = "0.14", default-features = false, features = [ + "tcp", + "stream", + "http1", + "http2", + "client", + "server", + "runtime", +] } serde = { version = "1.0", features = ["derive"] } libflate = "1.0" brotli_crate = { package = "brotli", version = "3.3.0" } doc-comment = "0.3" -tokio = { version = "1.0", default-features = false, features = ["macros", "rt-multi-thread"] } +tokio = { version = "1.0", default-features = false, features = [ + "macros", + "rt-multi-thread", +] } [target.'cfg(windows)'.dependencies] winreg = "0.10" @@ -169,7 +216,7 @@ features = [ "BlobPropertyBag", "ServiceWorkerGlobalScope", "RequestCredentials", - "File" + "File", ] [target.'cfg(target_arch = "wasm32")'.dev-dependencies] @@ -233,3 +280,7 @@ required-features = ["deflate"] name = "multipart" path = "tests/multipart.rs" required-features = ["multipart"] + +[patch.crates-io] +hyper = { git = "https://github.com/4JX/hyper.git", branch = "0.14.x-patched", ref = "bd25359" } +h2 = { git = "https://github.com/4JX/h2.git", ref = "b088466" } diff --git a/README.md b/README.md index d2e5d04..3848dcb 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,57 @@ -# reqwest +# reqwest-impersonate + +A fork of reqwest used to impersonate the Chrome browser. Inspired by [curl-impersonate](https://github.com/lwthiker/curl-impersonate). + +This crate was intended to be an experiment to learn more about TLS and HTTP2 fingerprinting. Some parts of reqwest may not have the code needed to work when used to copy Chrome. + +It is currently missing HTTP/2 `PRIORITY` support. (PRs to [h2](https://github.com/hyperium/h2) are welcome) + +**Notice:** This crate depends on patched dependencies. To use it, please add the following to your `Cargo.toml`. + +```toml +[patch.crates-io] +hyper = { git = "https://github.com/4JX/hyper.git", branch = "0.14.x-patched" } +h2 = { git = "https://github.com/4JX/h2.git", branch = "imp" } +``` + +These patches were made specifically for `reqwest-impersonate` to work, but I would appreciate if someone took the time to PR more "proper" versions to the parent projects. + +## Example + +`Cargo.toml` + +```toml +reqwest-impersonate = { git = "https://github.com/4JX/reqwest-impersonate.git", default-features = false, features = [ + "chrome", + "blocking", +] } +``` + +`main.rs` + +```rs +use reqwest_impersonate::browser::ChromeVersion; + +fn main() { + // Build a client to mimic Chrome 104 + let client = reqwest_impersonate::blocking::Client::builder() + .chrome_builder(ChromeVersion::V104) + .build() + .unwrap(); + + // Use the API you're already familiar with + match client.get("https://yoururl.com").send() { + Ok(res) => { + println!("{:?}", res.text().unwrap()); + } + Err(err) => { + dbg!(err); + } + }; +} +``` + +## Original readme [![crates.io](https://img.shields.io/crates/v/reqwest.svg)](https://crates.io/crates/reqwest) [![Documentation](https://docs.rs/reqwest/badge.svg)](https://docs.rs/reqwest) @@ -15,7 +68,6 @@ An ergonomic, batteries-included HTTP Client for Rust. - WASM - [Changelog](CHANGELOG.md) - ## Example This asynchronous example uses [Tokio](https://tokio.rs) and enables some @@ -67,7 +119,7 @@ fn main() -> Result<(), Box> { On Linux: -- OpenSSL 1.0.1, 1.0.2, 1.1.0, or 1.1.1 with headers (see https://github.com/sfackler/rust-openssl) +- OpenSSL 1.0.1, 1.0.2, 1.1.0, or 1.1.1 with headers (see ) On Windows and macOS: @@ -77,13 +129,12 @@ Reqwest uses [rust-native-tls](https://github.com/sfackler/rust-native-tls), which will use the operating system TLS framework if available, meaning Windows and macOS. On Linux, it will use OpenSSL 1.1. - ## License Licensed under either of -- Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://apache.org/licenses/LICENSE-2.0) -- MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or ) +- MIT license ([LICENSE-MIT](LICENSE-MIT) or ) ### Contribution diff --git a/src/async_impl/client.rs b/src/async_impl/client.rs index b56c7fe..01048e5 100644 --- a/src/async_impl/client.rs +++ b/src/async_impl/client.rs @@ -28,7 +28,9 @@ use super::decoder::Accepts; use super::request::{Request, RequestBuilder}; use super::response::Response; use super::Body; -use crate::connect::Connector; +#[cfg(feature = "__chrome")] +use crate::browser::{configure_chrome, ChromeVersion}; +use crate::connect::{Connector}; #[cfg(feature = "cookies")] use crate::cookie; #[cfg(feature = "trust-dns")] @@ -115,6 +117,10 @@ struct Config { http2_initial_connection_window_size: Option, http2_adaptive_window: bool, http2_max_frame_size: Option, + http2_max_concurrent_streams: Option, + http2_max_header_list_size: Option, + http2_enable_push: Option, + http2_header_table_size: Option, http2_keep_alive_interval: Option, http2_keep_alive_timeout: Option, http2_keep_alive_while_idle: bool, @@ -186,6 +192,10 @@ impl ClientBuilder { http2_initial_connection_window_size: None, http2_adaptive_window: false, http2_max_frame_size: None, + http2_max_concurrent_streams: None, + http2_max_header_list_size: None, + http2_enable_push: None, + http2_header_table_size: None, http2_keep_alive_interval: None, http2_keep_alive_timeout: None, http2_keep_alive_while_idle: false, @@ -201,6 +211,11 @@ impl ClientBuilder { } } + /// Sets the necessary values to mimic the specified Chrome version. + #[cfg(feature = "__chrome")] + pub fn chrome_builder(self, ver: ChromeVersion) -> ClientBuilder { + configure_chrome(ver, self) + } /// Returns a `Client` that uses this `ClientBuilder` configuration. /// /// # Errors @@ -246,6 +261,15 @@ impl ClientBuilder { #[cfg(feature = "__tls")] match config.tls { + #[cfg(feature = "__boring")] + TlsBackend::BoringTls(tls) => Connector::new_boring_tls( + http, + tls, + proxies.clone(), + user_agent(&config.headers), + config.local_address, + config.nodelay, + ), #[cfg(feature = "default-tls")] TlsBackend::Default => { let mut tls = TlsConnector::builder(); @@ -459,7 +483,7 @@ impl ClientBuilder { config.nodelay, ) } - #[cfg(any(feature = "native-tls", feature = "__rustls",))] + #[cfg(any(feature = "native-tls", feature = "__rustls"))] TlsBackend::UnknownPreconfigured => { return Err(crate::error::builder( "Unknown TLS backend passed to `use_preconfigured_tls`", @@ -493,6 +517,18 @@ impl ClientBuilder { if let Some(http2_max_frame_size) = config.http2_max_frame_size { builder.http2_max_frame_size(http2_max_frame_size); } + if let Some(max) = config.http2_max_concurrent_streams { + builder.http2_max_concurrent_streams(max); + } + if let Some(max) = config.http2_max_header_list_size { + builder.http2_max_header_list_size(max); + } + if let Some(opt) = config.http2_enable_push { + builder.http2_enable_push(opt); + } + if let Some(max) = config.http2_header_table_size { + builder.http2_header_table_size(max); + } if let Some(http2_keep_alive_interval) = config.http2_keep_alive_interval { builder.http2_keep_alive_interval(http2_keep_alive_interval); } @@ -628,6 +664,12 @@ impl ClientBuilder { self } + #[cfg(feature = "__browser_common")] + pub(crate) fn replace_default_headers(mut self, headers: HeaderMap) -> ClientBuilder { + self.config.headers = headers; + self + } + /// Enable a persistent cookie store for the client. /// /// Cookies received in responses will be preserved and included in @@ -965,6 +1007,39 @@ impl ClientBuilder { self } + /// Sets the maximum concurrent streams to use for HTTP2. + /// + /// Passing `None` will do nothing. + pub fn http2_max_concurrent_streams(mut self, sz: impl Into>) -> ClientBuilder { + self.config.http2_max_concurrent_streams = sz.into(); + self + } + + /// Sets the max header list size to use for HTTP2. + /// + /// Passing `None` will do nothing. + pub fn http2_max_header_list_size(mut self, sz: impl Into>) -> ClientBuilder { + self.config.http2_max_header_list_size = sz.into(); + self + } + + /// Enables and disables the push feature for HTTP2. + /// + /// Passing `None` will do nothing. + pub fn http2_enable_push(mut self, sz: impl Into>) -> ClientBuilder { + self.config.http2_enable_push = sz.into(); + self + } + + /// Sets the header table size to use for HTTP2. + /// + /// Passing `None` will do nothing. + + pub fn http2_header_table_size(mut self, sz: impl Into>) -> ClientBuilder { + self.config.http2_header_table_size = sz.into(); + self + } + /// Sets an interval for HTTP2 Ping frames should be sent to keep a connection alive. /// /// Pass `None` to disable HTTP2 keep-alive. @@ -1262,6 +1337,24 @@ impl ClientBuilder { self } + /// Force using the Boring TLS backend. + /// + /// Since multiple TLS backends can be optionally enabled, this option will + /// force the `boring` backend to be used for this `Client`. + /// + /// # Optional + /// + /// This requires the optional `boring-tls(-...)` feature to be enabled. + #[cfg(feature = "__boring")] + #[cfg_attr(docsrs, doc(cfg(feature = "boring-tls")))] + pub fn use_boring_tls( + mut self, + builder_func: Arc boring::ssl::SslConnectorBuilder + Send + Sync>, + ) -> ClientBuilder { + self.config.tls = TlsBackend::BoringTls(builder_func); + self + } + /// Use a preconfigured TLS backend. /// /// If the passed `Any` argument is not a TLS backend that reqwest diff --git a/src/blocking/client.rs b/src/blocking/client.rs index e6ec673..41d1316 100644 --- a/src/blocking/client.rs +++ b/src/blocking/client.rs @@ -16,6 +16,8 @@ use tokio::sync::{mpsc, oneshot}; use super::request::{Request, RequestBuilder}; use super::response::Response; use super::wait; +#[cfg(feature = "__chrome")] +use crate::browser::ChromeVersion; #[cfg(feature = "__tls")] use crate::tls; #[cfg(feature = "__tls")] @@ -88,6 +90,12 @@ impl ClientBuilder { } } + /// Sets the necessary values to mimic the specified Chrome version. + #[cfg(feature = "__chrome")] + pub fn chrome_builder(self, ver: ChromeVersion) -> ClientBuilder { + self.with_inner(move |inner| inner.chrome_builder(ver)) + } + /// Returns a `Client` that uses this `ClientBuilder` configuration. /// /// # Errors @@ -464,6 +472,35 @@ impl ClientBuilder { self.with_inner(|inner| inner.http2_max_frame_size(sz)) } + /// Sets the maximum concurrent streams to use for HTTP2. + /// + /// Passing `None` will do nothing. + pub fn http2_max_concurrent_streams(self, sz: impl Into>) -> ClientBuilder { + self.with_inner(|inner| inner.http2_max_concurrent_streams(sz)) + } + + /// Sets the max header list size to use for HTTP2. + /// + /// Passing `None` will do nothing. + pub fn http2_max_header_list_size(self, sz: impl Into>) -> ClientBuilder { + self.with_inner(|inner| inner.http2_max_header_list_size(sz)) + } + + /// Enables and disables the push feature for HTTP2. + /// + /// Passing `None` will do nothing. + pub fn http2_enable_push(self, sz: impl Into>) -> ClientBuilder { + self.with_inner(|inner| inner.http2_enable_push(sz)) + } + + /// Sets the header table size to use for HTTP2. + /// + /// Passing `None` will do nothing. + + pub fn http2_header_table_size(self, sz: impl Into>) -> ClientBuilder { + self.with_inner(|inner| inner.http2_header_table_size(sz)) + } + // TCP options /// Set whether sockets have `TCP_NODELAY` enabled. @@ -724,6 +761,23 @@ impl ClientBuilder { self.with_inner(move |inner| inner.use_rustls_tls()) } + /// Force using the Boring TLS backend. + /// + /// Since multiple TLS backends can be optionally enabled, this option will + /// force the `boring` backend to be used for this `Client`. + /// + /// # Optional + /// + /// This requires the optional `boring-tls(-...)` feature to be enabled. + #[cfg(feature = "__boring")] + #[cfg_attr(docsrs, doc(cfg(feature = "boring-tls")))] + pub fn use_boring_tls( + self, + builder_func: Arc boring::ssl::SslConnectorBuilder + Send + Sync>, + ) -> ClientBuilder { + self.with_inner(move |inner| inner.use_boring_tls(builder_func)) + } + /// Use a preconfigured TLS backend. /// /// If the passed `Any` argument is not a TLS backend that reqwest diff --git a/src/browser/chrome/mod.rs b/src/browser/chrome/mod.rs new file mode 100644 index 0000000..caa47b7 --- /dev/null +++ b/src/browser/chrome/mod.rs @@ -0,0 +1,27 @@ +//! Settings for impersonating the Chrome browser + +use crate::ClientBuilder; + +mod ver; + +pub(crate) fn configure_chrome(ver: ChromeVersion, builder: ClientBuilder) -> ClientBuilder { + let settings = ver::get_config_from_ver(ver); + + builder + .use_boring_tls(settings.tls_builder_func) + .http2_initial_stream_window_size(settings.http2.initial_stream_window_size) + .http2_initial_connection_window_size(settings.http2.initial_connection_window_size) + .http2_max_concurrent_streams(settings.http2.max_concurrent_streams) + .http2_max_header_list_size(settings.http2.max_header_list_size) + .http2_header_table_size(settings.http2.header_table_size) + .replace_default_headers(settings.headers) + .brotli(settings.brotli) + .gzip(settings.gzip) +} + +/// Defines the Chrome version to mimic when setting up a builder +#[derive(Debug)] +#[allow(missing_docs)] +pub enum ChromeVersion { + V104, +} diff --git a/src/browser/chrome/ver/mod.rs b/src/browser/chrome/ver/mod.rs new file mode 100644 index 0000000..458be98 --- /dev/null +++ b/src/browser/chrome/ver/mod.rs @@ -0,0 +1,11 @@ +use crate::browser::BrowserSettings; + +use super::ChromeVersion; + +mod v104; + +pub(super) fn get_config_from_ver(ver: ChromeVersion) -> BrowserSettings { + match ver { + ChromeVersion::V104 => v104::get_settings(), + } +} diff --git a/src/browser/chrome/ver/v104.rs b/src/browser/chrome/ver/v104.rs new file mode 100644 index 0000000..5d4538f --- /dev/null +++ b/src/browser/chrome/ver/v104.rs @@ -0,0 +1,110 @@ +use std::sync::Arc; + +use boring::ssl::{ + CertCompressionAlgorithm, SslConnector, SslConnectorBuilder, SslMethod, SslVersion, +}; +use http::{ + header::{ACCEPT, ACCEPT_ENCODING, ACCEPT_LANGUAGE, UPGRADE_INSECURE_REQUESTS, USER_AGENT}, + HeaderMap, +}; + +use crate::browser::{BrowserSettings, Http2Data}; + +pub(super) fn get_settings() -> BrowserSettings { + BrowserSettings { + tls_builder_func: Arc::new(create_ssl_connector), + http2: Http2Data { + initial_stream_window_size: 6291456, + initial_connection_window_size: 15728640, + max_concurrent_streams: 1000, + max_header_list_size: 262144, + header_table_size: 65536, + }, + headers: create_headers(), + gzip: true, + brotli: true, + } +} + +fn create_ssl_connector() -> SslConnectorBuilder { + let mut builder = SslConnector::builder(SslMethod::tls()).unwrap(); + + builder.set_grease_enabled(true); + + builder.enable_ocsp_stapling(); + + let cipher_list = [ + "TLS_AES_128_GCM_SHA256", + "TLS_AES_256_GCM_SHA384", + "TLS_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA", + "TLS_RSA_WITH_AES_128_GCM_SHA256", + "TLS_RSA_WITH_AES_256_GCM_SHA384", + "TLS_RSA_WITH_AES_128_CBC_SHA", + "TLS_RSA_WITH_AES_256_CBC_SHA", + ]; + + builder.set_cipher_list(&cipher_list.join(":")).unwrap(); + + let sigalgs_list = [ + "ecdsa_secp256r1_sha256", + "rsa_pss_rsae_sha256", + "rsa_pkcs1_sha256", + "ecdsa_secp384r1_sha384", + "rsa_pss_rsae_sha384", + "rsa_pkcs1_sha384", + "rsa_pss_rsae_sha512", + "rsa_pkcs1_sha512", + ]; + + builder.set_sigalgs_list(&sigalgs_list.join(":")).unwrap(); + + builder.enable_signed_cert_timestamps(); + + builder.set_alpn_protos(b"\x02h2\x08http/1.1").unwrap(); + + builder + .add_cert_compression_alg(CertCompressionAlgorithm::Brotli) + .unwrap(); + + builder + .set_min_proto_version(Some(SslVersion::TLS1_2)) + .unwrap(); + + builder + .set_max_proto_version(Some(SslVersion::TLS1_3)) + .unwrap(); + + builder +} + +fn create_headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + + headers.insert( + "sec-ch-ua", + "\"Chromium\";v=\"104\", \" Not A;Brand\";v=\"99\", \"Google Chrome\";v=\"104\"" + .parse() + .unwrap(), + ); + headers.insert("sec-ch-ua-mobile", "?0".parse().unwrap()); + headers.insert("sec-ch-ua-platform", "\"Windows\"".parse().unwrap()); + headers.insert(UPGRADE_INSECURE_REQUESTS, "1".parse().unwrap()); + headers.insert(USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36".parse().unwrap()); + headers.insert(ACCEPT, "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9".parse().unwrap()); + headers.insert("sec-fetch-site", "none".parse().unwrap()); + headers.insert("sec-fetch-mode", "navigate".parse().unwrap()); + headers.insert("sec-fetch-user", "?1".parse().unwrap()); + headers.insert("sec-fetch-dest", "document".parse().unwrap()); + headers.insert(ACCEPT_ENCODING, "gzip, deflate, br".parse().unwrap()); + headers.insert(ACCEPT_LANGUAGE, "en-US,en;q=0.9".parse().unwrap()); + + headers +} diff --git a/src/browser/mod.rs b/src/browser/mod.rs new file mode 100644 index 0000000..7fa9a9d --- /dev/null +++ b/src/browser/mod.rs @@ -0,0 +1,31 @@ +//! Holds structs and information to aid in impersonating a set of browsers + +use std::sync::Arc; + +use boring::ssl::SslConnectorBuilder; +use http::HeaderMap; + +#[cfg(feature = "__chrome")] +pub use chrome::ChromeVersion; + +#[cfg(feature = "__chrome")] +mod chrome; + +#[cfg(feature = "__chrome")] +pub(crate) use chrome::configure_chrome; + +struct BrowserSettings { + pub tls_builder_func: Arc SslConnectorBuilder + Send + Sync>, + pub http2: Http2Data, + pub headers: HeaderMap, + pub gzip: bool, + pub brotli: bool, +} + +struct Http2Data { + pub initial_stream_window_size: u32, + pub initial_connection_window_size: u32, + pub max_concurrent_streams: u32, + pub max_header_list_size: u32, + pub header_table_size: u32, +} diff --git a/src/connect.rs b/src/connect.rs index 388a39b..959c8d9 100644 --- a/src/connect.rs +++ b/src/connect.rs @@ -1,3 +1,8 @@ +#[cfg(feature = "__boring")] +use boring::ssl::{ConnectConfiguration, SslConnectorBuilder}; +#[cfg(feature = "__boring")] +use foreign_types::ForeignTypeRef; +use futures_util::future::Either; #[cfg(feature = "__tls")] use http::header::HeaderValue; use http::uri::{Authority, Scheme}; @@ -17,6 +22,8 @@ use std::sync::Arc; use std::task::{Context, Poll}; use std::time::Duration; +#[cfg(feature = "__boring")] +use self::boring_tls_conn::BoringTlsConn; #[cfg(feature = "default-tls")] use self::native_tls_conn::NativeTlsConn; #[cfg(feature = "__rustls")] @@ -51,6 +58,28 @@ enum Inner { tls: Arc, tls_proxy: Arc, }, + #[cfg(feature = "__boring")] + BoringTls { + http: HttpConnector, + tls: Arc SslConnectorBuilder + Send + Sync>, + }, +} + +#[cfg(feature = "__boring")] +fn tls_add_application_settings(conf: &mut ConnectConfiguration) { + // curl-impersonate does not know how to set this up, neither do I. Hopefully nothing breaks with these values. + + const ALPN_H2: &str = "h2"; + const ALPN_H2_LENGTH: usize = 2; + unsafe { + boring_sys::SSL_add_application_settings( + conf.as_ptr(), + ALPN_H2.as_ptr(), + ALPN_H2_LENGTH, + std::ptr::null(), + 0, + ) + }; } impl Connector { @@ -117,6 +146,31 @@ impl Connector { } } + #[cfg(feature = "__boring")] + pub(crate) fn new_boring_tls( + mut http: HttpConnector, + tls: Arc SslConnectorBuilder + Send + Sync>, + proxies: Arc>, + user_agent: Option, + local_addr: T, + nodelay: bool, + ) -> Connector + where + T: Into>, + { + http.set_local_address(local_addr.into()); + http.enforce_http(false); + + Connector { + inner: Inner::BoringTls { http, tls }, + proxies, + verbose: verbose::OFF, + timeout: None, + nodelay, + user_agent, + } + } + #[cfg(feature = "__rustls")] pub(crate) fn new_rustls_tls( mut http: HttpConnector, @@ -211,6 +265,23 @@ impl Connector { }); } } + #[cfg(feature = "__boring")] + Inner::BoringTls { tls, .. } => { + if dst.scheme() == Some(&Scheme::HTTPS) { + let host = dst.host().ok_or("no host in url")?.to_string(); + let conn = socks::connect(proxy, dst, dns).await?; + let tls_connector = tls().build(); + let mut conf = tls_connector.configure()?; + + tls_add_application_settings(&mut conf); + + let io = tokio_boring::connect(conf, &host, conn).await?; + return Ok(Conn { + inner: self.verbose.wrap(BoringTlsConn { inner: io }), + is_proxy: false, + }); + } + } #[cfg(not(feature = "__tls"))] Inner::Http(_) => (), } @@ -291,6 +362,42 @@ impl Connector { }) } } + #[cfg(feature = "__boring")] + Inner::BoringTls { http, tls } => { + let mut http = http.clone(); + + // Disable Nagle's algorithm for TLS handshake + // + // https://www.openssl.org/docs/man1.1.1/man3/SSL_connect.html#NOTES + if !self.nodelay && (dst.scheme() == Some(&Scheme::HTTPS)) { + http.set_nodelay(true); + } + + let mut http = hyper_boring::HttpsConnector::with_connector(http, tls())?; + + http.set_callback(|conf, _| { + tls_add_application_settings(conf); + Ok(()) + }); + + let io = http.call(dst).await?; + + if let hyper_boring::MaybeHttpsStream::Https(stream) = io { + if !self.nodelay { + let stream_ref = stream.get_ref(); + stream_ref.set_nodelay(false)?; + } + Ok(Conn { + inner: self.verbose.wrap(BoringTlsConn { inner: stream }), + is_proxy, + }) + } else { + Ok(Conn { + inner: self.verbose.wrap(io), + is_proxy, + }) + } + } } } @@ -372,6 +479,45 @@ impl Connector { }); } } + #[cfg(feature = "__boring")] + Inner::BoringTls { http, tls } => { + if dst.scheme() == Some(&Scheme::HTTPS) { + let host = dst.host().to_owned(); + let port = dst.port().map(|p| p.as_u16()).unwrap_or(443); + let http = http.clone(); + let tls_connector = tls(); + let mut http = + hyper_boring::HttpsConnector::with_connector(http, tls_connector)?; + + http.set_callback(|conf, _| { + tls_add_application_settings(conf); + + Ok(()) + }); + + let conn = http.call(proxy_dst).await?; + log::trace!("tunneling HTTPS over proxy"); + let tunneled = tunnel( + conn, + host.ok_or("no host in url")?.to_string(), + port, + self.user_agent.clone(), + auth, + ) + .await?; + let tls_connector = tls().build(); + let mut conf = tls_connector.configure()?; + + tls_add_application_settings(&mut conf); + + let io = tokio_boring::connect(conf, host.ok_or("no host in url")?, tunneled) + .await?; + return Ok(Conn { + inner: self.verbose.wrap(BoringTlsConn { inner: io }), + is_proxy: false, + }); + } + } #[cfg(not(feature = "__tls"))] Inner::Http(_) => (), } @@ -387,6 +533,8 @@ impl Connector { Inner::RustlsTls { http, .. } => http.set_keepalive(dur), #[cfg(not(feature = "__tls"))] Inner::Http(http) => http.set_keepalive(dur), + #[cfg(feature = "__boring")] + Inner::BoringTls { http, .. } => http.set_keepalive(dur), } } } @@ -764,6 +912,86 @@ mod rustls_tls_conn { } } +#[cfg(feature = "__boring")] +mod boring_tls_conn { + use hyper::client::connect::{Connected, Connection}; + use pin_project_lite::pin_project; + use std::{ + io::{self, IoSlice}, + pin::Pin, + task::{Context, Poll}, + }; + use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + use tokio_boring::SslStream; + + pin_project! { + pub(super) struct BoringTlsConn { + #[pin] pub(super) inner: SslStream, + } + } + + impl Connection for BoringTlsConn { + fn connected(&self) -> Connected { + if self.inner.ssl().selected_alpn_protocol() == Some(b"h2") { + self.inner.get_ref().connected().negotiated_h2() + } else { + self.inner.get_ref().connected() + } + } + } + + impl AsyncRead for BoringTlsConn { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let this = self.project(); + AsyncRead::poll_read(this.inner, cx, buf) + } + } + + impl AsyncWrite for BoringTlsConn { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context, + buf: &[u8], + ) -> Poll> { + let this = self.project(); + AsyncWrite::poll_write(this.inner, cx, buf) + } + + fn poll_write_vectored( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[IoSlice<'_>], + ) -> Poll> { + let this = self.project(); + AsyncWrite::poll_write_vectored(this.inner, cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + self.inner.is_write_vectored() + } + + fn poll_flush( + self: Pin<&mut Self>, + cx: &mut Context, + ) -> Poll> { + let this = self.project(); + AsyncWrite::poll_flush(this.inner, cx) + } + + fn poll_shutdown( + self: Pin<&mut Self>, + cx: &mut Context, + ) -> Poll> { + let this = self.project(); + AsyncWrite::poll_shutdown(this.inner, cx) + } + } +} + #[cfg(feature = "socks")] mod socks { use std::io; diff --git a/src/lib.rs b/src/lib.rs index 4228c84..ae1bf41 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -217,6 +217,11 @@ macro_rules! if_hyper { )*} } +/// Re-export of boring to keep versions in check +#[cfg(feature = "__boring")] +pub use boring; +#[cfg(feature = "__boring")] +pub use boring_sys; pub use http::header; pub use http::Method; pub use http::{StatusCode, Version}; @@ -225,6 +230,8 @@ pub use url::Url; // universal mods #[macro_use] mod error; +#[cfg(feature = "__browser_common")] +pub mod browser; mod into_url; mod response; diff --git a/src/tls.rs b/src/tls.rs index db898e8..fcda5f4 100644 --- a/src/tls.rs +++ b/src/tls.rs @@ -10,13 +10,14 @@ //! [`Identity`][Identity] type. //! - Various parts of TLS can also be configured or even disabled on the //! `ClientBuilder`. - #[cfg(feature = "__rustls")] use rustls::{ client::HandshakeSignatureValid, client::ServerCertVerified, client::ServerCertVerifier, internal::msgs::handshake::DigitallySignedStruct, Error as TLSError, ServerName, }; use std::fmt; +#[cfg(feature = "__boring")] +use std::sync::Arc; /// Represents a server X509 certificate. #[derive(Clone)] @@ -71,6 +72,7 @@ impl Certificate { /// # Ok(()) /// # } /// ``` + #[cfg(any(not(feature = "__boring"), feature = "native-tls-crate", feature = "__rustls"))] pub fn from_der(der: &[u8]) -> crate::Result { Ok(Certificate { #[cfg(feature = "native-tls-crate")] @@ -96,6 +98,7 @@ impl Certificate { /// # Ok(()) /// # } /// ``` + #[cfg(any(not(feature = "__boring"), feature = "native-tls-crate", feature = "__rustls"))] pub fn from_pem(pem: &[u8]) -> crate::Result { Ok(Certificate { #[cfg(feature = "native-tls-crate")] @@ -387,13 +390,17 @@ pub(crate) enum TlsBackend { Rustls, #[cfg(feature = "__rustls")] BuiltRustls(rustls::ClientConfig), - #[cfg(any(feature = "native-tls", feature = "__rustls",))] + #[cfg(feature = "__boring")] + BoringTls(Arc boring::ssl::SslConnectorBuilder + Send + Sync>), + #[cfg(any(feature = "native-tls", feature = "__rustls"))] UnknownPreconfigured, } impl fmt::Debug for TlsBackend { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { + #[cfg(feature = "__boring")] + TlsBackend::BoringTls(_) => write!(f, "BoringTls"), #[cfg(feature = "default-tls")] TlsBackend::Default => write!(f, "Default"), #[cfg(feature = "native-tls")] @@ -402,7 +409,7 @@ impl fmt::Debug for TlsBackend { TlsBackend::Rustls => write!(f, "Rustls"), #[cfg(feature = "__rustls")] TlsBackend::BuiltRustls(_) => write!(f, "BuiltRustls"), - #[cfg(any(feature = "native-tls", feature = "__rustls",))] + #[cfg(any(feature = "native-tls", feature = "__rustls"))] TlsBackend::UnknownPreconfigured => write!(f, "UnknownPreconfigured"), } } @@ -419,6 +426,16 @@ impl Default for TlsBackend { { TlsBackend::Rustls } + + #[cfg(all(feature = "__boring", not(feature = "default-tls")))] + { + use boring::ssl::{SslConnector, SslConnectorBuilder, SslMethod}; + + fn create_builder() -> SslConnectorBuilder { + SslConnector::builder(SslMethod::tls()).unwrap() + } + TlsBackend::BoringTls(Arc::new(create_builder)) + } } }