use std::{fmt, str}; use std::sync::Arc; use std::time::Duration; use bytes::Bytes; use futures::{Async, Future, Poll}; use header::{ HeaderMap, HeaderValue, ACCEPT, ACCEPT_ENCODING, CONTENT_LENGTH, CONTENT_ENCODING, CONTENT_TYPE, LOCATION, PROXY_AUTHORIZATION, RANGE, REFERER, TRANSFER_ENCODING, USER_AGENT, }; use http::Uri; use hyper::client::ResponseFuture; use mime; #[cfg(feature = "default-tls")] use native_tls::TlsConnector; use super::request::{Request, RequestBuilder}; use super::response::Response; use connect::Connector; use into_url::to_uri; use redirect::{self, RedirectPolicy, remove_sensitive_headers}; use {IntoUrl, Method, Proxy, StatusCode, Url}; #[cfg(feature = "tls")] use {Certificate, Identity}; #[cfg(feature = "tls")] use ::tls::{TlsBackend, inner as tls_inner}; static DEFAULT_USER_AGENT: &'static str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); /// An asynchronous `Client` to make Requests with. /// /// The Client has various configuration values to tweak, but the defaults /// are set to what is usually the most commonly desired value. To configure a /// `Client`, use `Client::builder()`. /// /// The `Client` holds a connection pool internally, so it is advised that /// you create one and **reuse** it. #[derive(Clone)] pub struct Client { inner: Arc, } /// A `ClientBuilder` can be used to create a `Client` with custom configuration. pub struct ClientBuilder { config: Config, } struct Config { gzip: bool, headers: HeaderMap, #[cfg(feature = "default-tls")] hostname_verification: bool, #[cfg(feature = "tls")] certs_verification: bool, proxies: Vec, redirect_policy: RedirectPolicy, referer: bool, timeout: Option, #[cfg(feature = "tls")] root_certs: Vec, #[cfg(feature = "tls")] identity: Option, #[cfg(feature = "tls")] tls: TlsBackend, } impl ClientBuilder { /// Constructs a new `ClientBuilder`. /// /// This is the same as `Client::builder()`. pub fn new() -> ClientBuilder { let mut headers: HeaderMap = HeaderMap::with_capacity(2); headers.insert(USER_AGENT, HeaderValue::from_static(DEFAULT_USER_AGENT)); headers.insert(ACCEPT, HeaderValue::from_str(mime::STAR_STAR.as_ref()).expect("unable to parse mime")); ClientBuilder { config: Config { gzip: true, headers: headers, #[cfg(feature = "default-tls")] hostname_verification: true, #[cfg(feature = "tls")] certs_verification: true, proxies: Vec::new(), redirect_policy: RedirectPolicy::default(), referer: true, timeout: None, #[cfg(feature = "tls")] root_certs: Vec::new(), #[cfg(feature = "tls")] identity: None, #[cfg(feature = "tls")] tls: TlsBackend::default(), }, } } /// Returns a `Client` that uses this `ClientBuilder` configuration. /// /// # Errors /// /// This method fails if TLS backend cannot be initialized. pub fn build(self) -> ::Result { let config = self.config; let proxies = Arc::new(config.proxies); let connector = { #[cfg(feature = "tls")] match config.tls { #[cfg(feature = "default-tls")] TlsBackend::Default => { let mut tls = TlsConnector::builder(); tls.danger_accept_invalid_hostnames(!config.hostname_verification); tls.danger_accept_invalid_certs(!config.certs_verification); for cert in config.root_certs { let cert = match cert.inner { tls_inner::Certificate::Der(buf) => try_!(::native_tls::Certificate::from_der(&buf)), tls_inner::Certificate::Pem(buf) => try_!(::native_tls::Certificate::from_pem(&buf)) }; tls.add_root_certificate(cert); } if let Some(id) = config.identity { let id = match id.inner { tls_inner::Identity::Pkcs12(buf, passwd) => try_!(::native_tls::Identity::from_pkcs12(&buf, &passwd)), #[cfg(feature = "rustls-tls")] _ => return Err(::error::from(::error::Kind::TlsIncompatible)) }; tls.identity(id); } Connector::new_default_tls(tls, proxies.clone())? }, #[cfg(feature = "rustls-tls")] TlsBackend::Rustls => { use std::io::Cursor; use rustls::TLSError; use rustls::internal::pemfile; use ::tls::NoVerifier; let mut tls = ::rustls::ClientConfig::new(); tls.root_store.add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS); if !config.certs_verification { tls.dangerous().set_certificate_verifier(Arc::new(NoVerifier)); } for cert in config.root_certs { match cert.inner { tls_inner::Certificate::Der(buf) => try_!(tls.root_store.add(&::rustls::Certificate(buf)) .map_err(TLSError::WebPKIError)), tls_inner::Certificate::Pem(buf) => { let mut pem = Cursor::new(buf); let mut certs = try_!(pemfile::certs(&mut pem) .map_err(|_| TLSError::General(String::from("No valid certificate was found")))); for c in certs { try_!(tls.root_store.add(&c) .map_err(TLSError::WebPKIError)); } } } } if let Some(id) = config.identity { let (key, certs) = match id.inner { tls_inner::Identity::Pem(buf) => { let mut pem = Cursor::new(buf); let mut certs = try_!(pemfile::certs(&mut pem) .map_err(|_| TLSError::General(String::from("No valid certificate was found")))); pem.set_position(0); let mut sk = try_!(pemfile::pkcs8_private_keys(&mut pem) .or_else(|_| { pem.set_position(0); pemfile::rsa_private_keys(&mut pem) }) .map_err(|_| TLSError::General(String::from("No valid private key was found")))); if let (Some(sk), false) = (sk.pop(), certs.is_empty()) { (sk, certs) } else { return Err(::error::from(TLSError::General(String::from("private key or certificate not found")))); } }, #[cfg(feature = "default-tls")] _ => return Err(::error::from(::error::Kind::TlsIncompatible)) }; tls.set_single_client_cert(certs, key); } Connector::new_rustls_tls(tls, proxies.clone())? } } #[cfg(not(feature = "tls"))] Connector::new(proxies.clone()) }; let hyper_client = ::hyper::Client::builder() .build(connector); let proxies_maybe_http_auth = proxies .iter() .any(|p| p.maybe_has_http_auth()); Ok(Client { inner: Arc::new(ClientRef { gzip: config.gzip, hyper: hyper_client, headers: config.headers, redirect_policy: config.redirect_policy, referer: config.referer, proxies, proxies_maybe_http_auth, }), }) } /// Use native TLS backend. #[cfg(feature = "default-tls")] pub fn use_default_tls(mut self) -> ClientBuilder { self.config.tls = TlsBackend::Default; self } /// Use rustls TLS backend. #[cfg(feature = "rustls-tls")] pub fn use_rustls_tls(mut self) -> ClientBuilder { self.config.tls = TlsBackend::Rustls; self } /// Add a custom root certificate. /// /// This can be used to connect to a server that has a self-signed /// certificate for example. #[cfg(feature = "tls")] pub fn add_root_certificate(mut self, cert: Certificate) -> ClientBuilder { self.config.root_certs.push(cert); self } /// Sets the identity to be used for client certificate authentication. #[cfg(feature = "tls")] pub fn identity(mut self, identity: Identity) -> ClientBuilder { self.config.identity = Some(identity); self } /// Controls the use of hostname verification. /// /// Defaults to `false`. /// /// # Warning /// /// You should think very carefully before you use this method. If /// hostname verification is not used, any valid certificate for any /// site will be trusted for use from any other. This introduces a /// significant vulnerability to man-in-the-middle attacks. #[cfg(feature = "default-tls")] pub fn danger_accept_invalid_hostnames(mut self, accept_invalid_hostname: bool) -> ClientBuilder { self.config.hostname_verification = !accept_invalid_hostname; self } /// Controls the use of certificate validation. /// /// Defaults to `false`. /// /// # Warning /// /// You should think very carefully before using this method. If /// invalid certificates are trusted, *any* certificate for *any* site /// will be trusted for use. This includes expired certificates. This /// introduces significant vulnerabilities, and should only be used /// as a last resort. #[cfg(feature = "tls")] pub fn danger_accept_invalid_certs(mut self, accept_invalid_certs: bool) -> ClientBuilder { self.config.certs_verification = !accept_invalid_certs; self } /// Sets the default headers for every request. pub fn default_headers(mut self, headers: HeaderMap) -> ClientBuilder { for (key, value) in headers.iter() { self.config.headers.insert(key, value.clone()); } self } /// Enable auto gzip decompression by checking the ContentEncoding response header. /// /// If auto gzip decompresson is turned on: /// - When sending a request and if the request's headers do not already contain /// an `Accept-Encoding` **and** `Range` values, the `Accept-Encoding` header is set to `gzip`. /// The body is **not** automatically inflated. /// - When receiving a response, if it's headers contain a `Content-Encoding` value that /// equals to `gzip`, both values `Content-Encoding` and `Content-Length` are removed from the /// headers' set. The body is automatically deinflated. /// /// Default is enabled. pub fn gzip(mut self, enable: bool) -> ClientBuilder { self.config.gzip = enable; self } /// Add a `Proxy` to the list of proxies the `Client` will use. pub fn proxy(mut self, proxy: Proxy) -> ClientBuilder { self.config.proxies.push(proxy); self } /// Set a `RedirectPolicy` for this client. /// /// Default will follow redirects up to a maximum of 10. pub fn redirect(mut self, policy: RedirectPolicy) -> ClientBuilder { self.config.redirect_policy = policy; self } /// Enable or disable automatic setting of the `Referer` header. /// /// Default is `true`. pub fn referer(mut self, enable: bool) -> ClientBuilder { self.config.referer = enable; self } /// Set a timeout for both the read and write operations of a client. pub fn timeout(mut self, timeout: Duration) -> ClientBuilder { self.config.timeout = Some(timeout); self } #[doc(hidden)] #[deprecated(note = "DNS no longer uses blocking threads")] pub fn dns_threads(self, _threads: usize) -> ClientBuilder { self } } type HyperClient = ::hyper::Client; impl Client { /// Constructs a new `Client`. /// /// # Panics /// /// This method panics if TLS backend cannot be created or /// initialized. Use `Client::builder()` if you wish to handle the failure /// as an `Error` instead of panicking. pub fn new() -> Client { ClientBuilder::new() .build() .expect("TLS failed to initialize") } /// Creates a `ClientBuilder` to configure a `Client`. /// /// This is the same as `ClientBuilder::new()`. pub fn builder() -> ClientBuilder { ClientBuilder::new() } /// Convenience method to make a `GET` request to a URL. /// /// # Errors /// /// This method fails whenever supplied `Url` cannot be parsed. pub fn get(&self, url: U) -> RequestBuilder { self.request(Method::GET, url) } /// Convenience method to make a `POST` request to a URL. /// /// # Errors /// /// This method fails whenever supplied `Url` cannot be parsed. pub fn post(&self, url: U) -> RequestBuilder { self.request(Method::POST, url) } /// Convenience method to make a `PUT` request to a URL. /// /// # Errors /// /// This method fails whenever supplied `Url` cannot be parsed. pub fn put(&self, url: U) -> RequestBuilder { self.request(Method::PUT, url) } /// Convenience method to make a `PATCH` request to a URL. /// /// # Errors /// /// This method fails whenever supplied `Url` cannot be parsed. pub fn patch(&self, url: U) -> RequestBuilder { self.request(Method::PATCH, url) } /// Convenience method to make a `DELETE` request to a URL. /// /// # Errors /// /// This method fails whenever supplied `Url` cannot be parsed. pub fn delete(&self, url: U) -> RequestBuilder { self.request(Method::DELETE, url) } /// Convenience method to make a `HEAD` request to a URL. /// /// # Errors /// /// This method fails whenever supplied `Url` cannot be parsed. pub fn head(&self, url: U) -> RequestBuilder { self.request(Method::HEAD, url) } /// Start building a `Request` with the `Method` and `Url`. /// /// Returns a `RequestBuilder`, which will allow setting headers and /// request body before sending. /// /// # Errors /// /// This method fails whenever supplied `Url` cannot be parsed. pub fn request(&self, method: Method, url: U) -> RequestBuilder { let req = url .into_url() .map(move |url| Request::new(method, url)); RequestBuilder::new(self.clone(), req) } /// Executes a `Request`. /// /// A `Request` can be built manually with `Request::new()` or obtained /// from a RequestBuilder with `RequestBuilder::build()`. /// /// You should prefer to use the `RequestBuilder` and /// `RequestBuilder::send()`. /// /// # Errors /// /// This method fails if there was an error while sending request, /// redirect loop was detected or redirect limit was exhausted. pub fn execute(&self, request: Request) -> Pending { self.execute_request(request) } fn execute_request(&self, req: Request) -> Pending { let ( method, url, user_headers, body ) = req.pieces(); let mut headers = self.inner.headers.clone(); // default headers for (key, value) in user_headers.iter() { headers.insert(key, value.clone()); } if self.inner.gzip && !headers.contains_key(ACCEPT_ENCODING) && !headers.contains_key(RANGE) { headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("gzip")); } let uri = to_uri(&url); let (reusable, body) = match body { Some(body) => { let (reusable, body) = body.into_hyper(); (Some(reusable), body) }, None => { (None, ::hyper::Body::empty()) } }; self.proxy_auth(&uri, &mut headers); let mut req = ::hyper::Request::builder() .method(method.clone()) .uri(uri.clone()) .body(body) .expect("valid request parts"); *req.headers_mut() = headers.clone(); let in_flight = self.inner.hyper.request(req); Pending { inner: PendingInner::Request(PendingRequest { method: method, url: url, headers: headers, body: reusable, urls: Vec::new(), client: self.inner.clone(), in_flight: in_flight, }), } } fn proxy_auth(&self, dst: &Uri, headers: &mut HeaderMap) { if !self.inner.proxies_maybe_http_auth { return; } // Only set the header here if the destination scheme is 'http', // since otherwise, the header will be included in the CONNECT tunnel // request instead. if dst.scheme_part() != Some(&::http::uri::Scheme::HTTP) { return; } if headers.contains_key(PROXY_AUTHORIZATION) { return; } for proxy in self.inner.proxies.iter() { if proxy.is_match(dst) { match proxy.auth() { Some(::proxy::Auth::Basic(ref header)) => { headers.insert( PROXY_AUTHORIZATION, header.clone() ); }, None => (), } break; } } } } impl fmt::Debug for Client { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("Client") .field("gzip", &self.inner.gzip) .field("redirect_policy", &self.inner.redirect_policy) .field("referer", &self.inner.referer) .finish() } } impl fmt::Debug for ClientBuilder { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("ClientBuilder") .finish() } } struct ClientRef { gzip: bool, headers: HeaderMap, hyper: HyperClient, redirect_policy: RedirectPolicy, referer: bool, proxies: Arc>, proxies_maybe_http_auth: bool, } pub struct Pending { inner: PendingInner, } enum PendingInner { Request(PendingRequest), Error(Option<::Error>), } struct PendingRequest { method: Method, url: Url, headers: HeaderMap, body: Option>, urls: Vec, client: Arc, in_flight: ResponseFuture, } impl Pending { pub(super) fn new_err(err: ::Error) -> Pending { Pending { inner: PendingInner::Error(Some(err)), } } } impl Future for Pending { type Item = Response; type Error = ::Error; fn poll(&mut self) -> Poll { match self.inner { PendingInner::Request(ref mut req) => req.poll(), PendingInner::Error(ref mut err) => Err(err.take().expect("Pending error polled more than once")), } } } impl Future for PendingRequest { type Item = Response; type Error = ::Error; fn poll(&mut self) -> Poll { loop { let res = match try_!(self.in_flight.poll(), &self.url) { Async::Ready(res) => res, Async::NotReady => return Ok(Async::NotReady), }; let should_redirect = match res.status() { StatusCode::MOVED_PERMANENTLY | StatusCode::FOUND | StatusCode::SEE_OTHER => { self.body = None; for header in &[TRANSFER_ENCODING, CONTENT_ENCODING, CONTENT_TYPE, CONTENT_LENGTH] { self.headers.remove(header); } match self.method { Method::GET | Method::HEAD => {}, _ => { self.method = Method::GET; } } true }, StatusCode::TEMPORARY_REDIRECT | StatusCode::PERMANENT_REDIRECT => match self.body { Some(Some(_)) | None => true, Some(None) => false, }, _ => false, }; if should_redirect { let loc = res.headers() .get(LOCATION) .and_then(|val| { let loc = (|| -> Option { // Some sites may send a utf-8 Location header, // even though we're supposed to treat those bytes // as opaque, we'll check specifically for utf8. self.url.join(str::from_utf8(val.as_bytes()).ok()?).ok() })(); if loc.is_none() { debug!("Location header had invalid URI: {:?}", val); } loc }); if let Some(loc) = loc { if self.client.referer { if let Some(referer) = make_referer(&loc, &self.url) { self.headers.insert(REFERER, referer); } } self.urls.push(self.url.clone()); let action = self.client.redirect_policy.check( res.status(), &loc, &self.urls, ); match action { redirect::Action::Follow => { self.url = loc; remove_sensitive_headers(&mut self.headers, &self.url, &self.urls); debug!("redirecting to {:?} '{}'", self.method, self.url); let uri = to_uri(&self.url); let body = match self.body { Some(Some(ref body)) => ::hyper::Body::from(body.clone()), _ => ::hyper::Body::empty(), }; let mut req = ::hyper::Request::builder() .method(self.method.clone()) .uri(uri.clone()) .body(body) .expect("valid request parts"); *req.headers_mut() = self.headers.clone(); self.in_flight = self.client.hyper.request(req); continue; }, redirect::Action::Stop => { debug!("redirect_policy disallowed redirection to '{}'", loc); }, redirect::Action::LoopDetected => { return Err(::error::loop_detected(self.url.clone())); }, redirect::Action::TooManyRedirects => { return Err(::error::too_many_redirects(self.url.clone())); } } } } let res = Response::new(res, self.url.clone(), self.client.gzip); return Ok(Async::Ready(res)); } } } impl fmt::Debug for Pending { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self.inner { PendingInner::Request(ref req) => { f.debug_struct("Pending") .field("method", &req.method) .field("url", &req.url) .finish() }, PendingInner::Error(ref err) => { f.debug_struct("Pending") .field("error", err) .finish() } } } } fn make_referer(next: &Url, previous: &Url) -> Option { if next.scheme() == "http" && previous.scheme() == "https" { return None; } let mut referer = previous.clone(); let _ = referer.set_username(""); let _ = referer.set_password(None); referer.set_fragment(None); referer.as_str().parse().ok() }