diff --git a/Cargo.toml b/Cargo.toml index d226b7c..05acd49 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,9 @@ readme = "README.md" license = "MIT/Apache-2.0" categories = ["web-programming::http-client"] +[package.metadata.docs.rs] +all-features = true + [dependencies] base64 = "0.10" bytes = "0.4" @@ -43,6 +46,9 @@ socks = { version = "0.3.2", optional = true } tokio-rustls = { version = "0.9", optional = true } trust-dns-resolver = { version = "0.10", optional = true } webpki-roots = { version = "0.16", optional = true } +cookie_store = "0.5.1" +cookie = "0.11.0" +time = "0.1.42" [dev-dependencies] env_logger = "0.6" @@ -63,6 +69,3 @@ rustls-tls = ["hyper-rustls", "tokio-rustls", "webpki-roots", "rustls", "tls"] trust-dns = ["trust-dns-resolver"] hyper-011 = ["hyper-old-types"] - -[package.metadata.docs.rs] -all-features = true diff --git a/src/async_impl/client.rs b/src/async_impl/client.rs index 94ac612..2e494fd 100644 --- a/src/async_impl/client.rs +++ b/src/async_impl/client.rs @@ -1,5 +1,5 @@ use std::{fmt, str}; -use std::sync::Arc; +use std::sync::{Arc, RwLock}; use std::time::Duration; use std::net::IpAddr; @@ -31,6 +31,7 @@ use super::request::{Request, RequestBuilder}; use super::response::Response; use connect::Connector; use into_url::{expect_uri, try_uri}; +use cookie; use redirect::{self, RedirectPolicy, remove_sensitive_headers}; use {IntoUrl, Method, Proxy, StatusCode, Url}; #[cfg(feature = "tls")] @@ -82,6 +83,7 @@ struct Config { http1_title_case_headers: bool, local_address: Option, nodelay: bool, + cookie_store: Option, } impl ClientBuilder { @@ -116,7 +118,8 @@ impl ClientBuilder { http2_only: false, http1_title_case_headers: false, local_address: None, - nodelay: false + nodelay: false, + cookie_store: None, }, } } @@ -204,6 +207,8 @@ impl ClientBuilder { .iter() .any(|p| p.maybe_has_http_auth()); + let cookie_store = config.cookie_store.map(RwLock::new); + Ok(Client { inner: Arc::new(ClientRef { gzip: config.gzip, @@ -213,6 +218,7 @@ impl ClientBuilder { referer: config.referer, proxies, proxies_maybe_http_auth, + cookie_store, }), }) } @@ -388,6 +394,21 @@ impl ClientBuilder { self.config.local_address = addr.into(); self } + + /// Enable a persistent cookie store for the client. + /// + /// Cookies received in responses will be preserved and included in + /// additional requests. + /// + /// By default, no cookie store is used. + pub fn cookie_store(mut self, enable: bool) -> ClientBuilder { + self.config.cookie_store = if enable { + Some(cookie::CookieStore::default()) + } else { + None + }; + self + } } type HyperClient = ::hyper::Client; @@ -514,6 +535,23 @@ impl Client { headers.insert(key, value.clone()); } + // Add cookies from the cookie store. + if let Some(cookie_store_wrapper) = self.inner.cookie_store.as_ref() { + if headers.get(::header::COOKIE).is_none() { + let cookie_store = cookie_store_wrapper.read().unwrap(); + let header = cookie_store + .0 + .get_request_cookies(&url) + .map(|c| c.encoded().to_string()) + .collect::>() + .join("; "); + if !header.is_empty() { + // TODO: is it safe to unwrap here? Investigate if Cookie content is always valid. + headers.insert(::header::COOKIE, HeaderValue::from_bytes(header.as_bytes()).unwrap()); + } + } + } + if self.inner.gzip && !headers.contains_key(ACCEPT_ENCODING) && !headers.contains_key(RANGE) { @@ -620,6 +658,7 @@ struct ClientRef { referer: bool, proxies: Arc>, proxies_maybe_http_auth: bool, + cookie_store: Option>, } pub struct Pending { @@ -674,6 +713,13 @@ impl Future for PendingRequest { Async::Ready(res) => res, Async::NotReady => return Ok(Async::NotReady), }; + if let Some(store_wrapper) = self.client.cookie_store.as_ref() { + let mut store = store_wrapper.write().unwrap(); + let cookies = cookie::extract_response_cookies(&res.headers()) + .filter_map(|res| res.ok()) + .map(|cookie| cookie.into_inner().into_owned()); + store.0.store_response_cookies(cookies, &self.url); + } let should_redirect = match res.status() { StatusCode::MOVED_PERMANENTLY | StatusCode::FOUND | diff --git a/src/async_impl/response.rs b/src/async_impl/response.rs index e727eb8..376712c 100644 --- a/src/async_impl/response.rs +++ b/src/async_impl/response.rs @@ -13,6 +13,7 @@ use serde_json; use url::Url; use http; +use cookie; use super::Decoder; use super::body::Body; @@ -69,6 +70,13 @@ impl Response { &mut self.headers } + /// Retrieve the cookies contained in the response. + pub fn cookies<'a>(&'a self) -> impl Iterator< + Item = Result, cookie::CookieParseError> + > + 'a { + cookie::extract_response_cookies(&self.headers) + } + /// Get the final `Url` of this `Response`. #[inline] pub fn url(&self) -> &Url { diff --git a/src/client.rs b/src/client.rs index 2cfd238..7633194 100644 --- a/src/client.rs +++ b/src/client.rs @@ -366,6 +366,16 @@ impl ClientBuilder { { self.with_inner(move |inner| inner.local_address(addr)) } + + /// Enable a persistent cookie store for the client. + /// + /// Cookies received in responses will be preserved and included in + /// additional requests. + /// + /// By default, no cookie store is used. + pub fn cookie_store(self, enable: bool) -> ClientBuilder { + self.with_inner(|inner| inner.cookie_store(enable)) + } } diff --git a/src/cookie.rs b/src/cookie.rs new file mode 100644 index 0000000..e2245e0 --- /dev/null +++ b/src/cookie.rs @@ -0,0 +1,220 @@ +//! The cookies module contains types for working with request and response cookies. + +use cookie_crate; +use header; +use std::borrow::Cow; +use std::fmt; +use std::time::SystemTime; + +/// Convert a time::Tm time to SystemTime. +fn tm_to_systemtime(tm: ::time::Tm) -> SystemTime { + let seconds = tm.to_timespec().sec; + let duration = std::time::Duration::from_secs(seconds.abs() as u64); + if seconds > 0 { + SystemTime::UNIX_EPOCH + duration + } else { + SystemTime::UNIX_EPOCH - duration + } +} + +/// Convert a SystemTime to time::Tm. +/// Returns None if the conversion failed. +fn systemtime_to_tm(time: SystemTime) -> Option<::time::Tm> { + let seconds = match time.duration_since(SystemTime::UNIX_EPOCH) { + Ok(duration) => duration.as_secs() as i64, + Err(_) => { + if let Ok(duration) = SystemTime::UNIX_EPOCH.duration_since(time) { + (duration.as_secs() as i64) * -1 + } else { + return None; + } + } + }; + Some(::time::at_utc(::time::Timespec::new(seconds, 0))) +} + +/// Represents the 'SameSite' attribute of a cookie. +#[derive(PartialEq, Eq, Clone, Copy, Debug)] +pub enum SameSite { + /// Strict same-site policy. + Strict, + /// Lax same-site policy. + Lax, +} + +impl SameSite { + fn from_inner(value: cookie_crate::SameSite) -> Option { + match value { + cookie_crate::SameSite::Strict => Some(SameSite::Strict), + cookie_crate::SameSite::Lax => Some(SameSite::Lax), + cookie_crate::SameSite::None => None, + } + } + + fn to_inner(value: Option) -> cookie_crate::SameSite { + match value { + Some(SameSite::Strict) => cookie_crate::SameSite::Strict, + Some(SameSite::Lax) => cookie_crate::SameSite::Lax, + None => cookie_crate::SameSite::None, + } + } +} + +/// Error representing a parse failure of a 'Set-Cookie' header. +pub struct CookieParseError(cookie::ParseError); + +impl<'a> fmt::Debug for CookieParseError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +impl<'a> fmt::Display for CookieParseError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +impl std::error::Error for CookieParseError {} + +/// A single HTTP cookie. +pub struct Cookie<'a>(cookie::Cookie<'a>); + +impl<'a> fmt::Debug for Cookie<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +impl Cookie<'static> { + /// Construct a new cookie with the given name and value. + pub fn new(name: N, value: V) -> Self + where + N: Into>, + V: Into>, + { + Self(cookie::Cookie::new(name, value)) + } +} + +impl<'a> Cookie<'a> { + fn parse(value: &'a ::header::HeaderValue) -> Result, CookieParseError> { + std::str::from_utf8(value.as_bytes()) + .map_err(cookie::ParseError::from) + .and_then(cookie::Cookie::parse) + .map_err(CookieParseError) + .map(Cookie) + } + + pub(crate) fn into_inner(self) -> cookie::Cookie<'a> { + self.0 + } + + /// The name of the cookie. + pub fn name(&self) -> &str { + self.0.name() + } + + /// Set the cookie name. + pub fn set_name>>(&mut self, name: P) { + self.0.set_name(name) + } + + /// The value of the cookie. + pub fn value(&self) -> &str { + self.0.value() + } + + /// Set the cookie value. + pub fn set_value>>(&mut self, value: P) { + self.0.set_value(value) + } + + /// Returns true if the 'HttpOnly' directive is enabled. + pub fn http_only(&self) -> bool { + self.0.http_only().unwrap_or(false) + } + + /// Set the 'HttpOnly' directive. + pub fn set_http_only(&mut self, value: bool) { + self.0.set_http_only(value) + } + + /// Returns true if the 'Secure' directive is enabled. + pub fn secure(&self) -> bool { + self.0.secure().unwrap_or(false) + } + + /// Set the 'Secure' directive. + pub fn set_secure(&mut self, value: bool) { + self.0.set_secure(value) + } + + /// Returns the 'SameSite' directive if present. + pub fn same_site(&self) -> Option { + self.0.same_site().and_then(SameSite::from_inner) + } + + /// Set the 'SameSite" directive. + pub fn set_same_site(&mut self, value: Option) { + self.0.set_same_site(SameSite::to_inner(value)) + } + + /// Returns the path directive of the cookie, if set. + pub fn path(&self) -> Option<&str> { + self.0.path() + } + + /// Set the cookie path. + pub fn set_path>>(&mut self, path: P) { + self.0.set_path(path) + } + + /// Returns the domain directive of the cookie, if set. + pub fn domain(&self) -> Option<&str> { + self.0.domain() + } + + /// Set the cookie domain. + pub fn set_domain>>(&mut self, domain: P) { + self.0.set_domain(domain) + } + + /// Get the Max-Age information. + pub fn max_age(&self) -> Option { + self.0.max_age().map(|d| std::time::Duration::new(d.num_seconds() as u64, 0)) + } + + /// The cookie expiration time. + pub fn expires(&self) -> Option { + self.0.expires().map(tm_to_systemtime) + } + + /// Set expiration time. + /// + /// Currently, providing a `None` value will have no effect. + pub fn set_expires(&mut self, value: Option) { + if let Some(tm) = value.and_then(systemtime_to_tm) { + self.0.set_expires(tm); + } + } +} + +pub(crate) fn extract_response_cookies<'a>( + headers: &'a hyper::HeaderMap, +) -> impl Iterator, CookieParseError>> + 'a { + headers + .get_all(header::SET_COOKIE) + .iter() + .map(|value| Cookie::parse(value)) +} + +/// A persistent cookie store that provides session support. +#[derive(Default)] +pub(crate) struct CookieStore(pub(crate) ::cookie_store::CookieStore); + +impl<'a> fmt::Debug for CookieStore { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} diff --git a/src/lib.rs b/src/lib.rs index 6fe7157..979f78b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -173,6 +173,8 @@ extern crate base64; extern crate bytes; +extern crate cookie as cookie_crate; +extern crate cookie_store; extern crate encoding_rs; #[macro_use] extern crate futures; @@ -195,6 +197,7 @@ extern crate serde; extern crate serde_derive; extern crate serde_json; extern crate serde_urlencoded; +extern crate time; extern crate tokio; #[cfg_attr(feature = "default-tls", macro_use)] extern crate tokio_io; @@ -241,6 +244,7 @@ mod async_impl; mod connect; mod body; mod client; +pub mod cookie; #[cfg(feature = "trust-dns")] mod dns; mod into_url; diff --git a/src/response.rs b/src/response.rs index e2d7e79..58595b5 100644 --- a/src/response.rs +++ b/src/response.rs @@ -12,6 +12,7 @@ use mime::Mime; use serde::de::DeserializeOwned; use serde_json; +use cookie; use client::KeepCoreThreadAlive; use hyper::header::HeaderMap; use {async_impl, StatusCode, Url, Version, wait}; @@ -122,6 +123,14 @@ impl Response { self.inner.headers() } + /// Retrieve the cookies contained in the response. + pub fn cookies<'a>(&'a self) -> impl Iterator< + Item = Result, cookie::CookieParseError> + > + 'a { + cookie::extract_response_cookies(self.headers()) + } + + /// Get the HTTP `Version` of this `Response`. #[inline] pub fn version(&self) -> Version { diff --git a/tests/cookie.rs b/tests/cookie.rs new file mode 100644 index 0000000..c5190b2 --- /dev/null +++ b/tests/cookie.rs @@ -0,0 +1,335 @@ +extern crate reqwest; + +#[macro_use] +mod support; + +#[test] +fn cookie_response_accessor() { + let mut rt = tokio::runtime::current_thread::Runtime::new().expect("new rt"); + let client = reqwest::async::Client::new(); + + let server = server! { + request: b"\ + GET / HTTP/1.1\r\n\ + user-agent: $USERAGENT\r\n\ + accept: */*\r\n\ + accept-encoding: gzip\r\n\ + host: $HOST\r\n\ + \r\n\ + ", + response: b"\ + HTTP/1.1 200 OK\r\n\ + Set-Cookie: key=val\r\n\ + Set-Cookie: expires=1; Expires=Wed, 21 Oct 2015 07:28:00 GMT\r\n\ + Set-Cookie: path=1; Path=/the-path\r\n\ + Set-Cookie: maxage=1; Max-Age=100\r\n\ + Set-Cookie: domain=1; Domain=mydomain\r\n\ + Set-Cookie: secure=1; Secure\r\n\ + Set-Cookie: httponly=1; HttpOnly\r\n\ + Set-Cookie: samesitelax=1; SameSite=Lax\r\n\ + Set-Cookie: samesitestrict=1; SameSite=Strict\r\n\ + Content-Length: 0\r\n\ + \r\n\ + " + }; + + let url = format!("http://{}/", server.addr()); + let res = rt.block_on(client.get(&url).send()).unwrap(); + + let cookies = res.cookies().map(|c| c.unwrap()).collect::>(); + + // key=val + assert_eq!(cookies[0].name(), "key"); + assert_eq!(cookies[0].value(), "val"); + + // expires + assert_eq!(cookies[1].name(), "expires"); + assert_eq!( + cookies[1].expires().unwrap(), + std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1445412480) + ); + + // path + assert_eq!(cookies[2].name(), "path"); + assert_eq!(cookies[2].path().unwrap(), "/the-path"); + + // max-age + assert_eq!(cookies[3].name(), "maxage"); + assert_eq!(cookies[3].max_age().unwrap(), std::time::Duration::from_secs(100)); + + // domain + assert_eq!(cookies[4].name(), "domain"); + assert_eq!(cookies[4].domain().unwrap(), "mydomain"); + + // secure + assert_eq!(cookies[5].name(), "secure"); + assert_eq!(cookies[5].secure(), true); + + // httponly + assert_eq!(cookies[6].name(), "httponly"); + assert_eq!(cookies[6].http_only(), true); + + // samesitelax + assert_eq!(cookies[7].name(), "samesitelax"); + assert_eq!(cookies[7].same_site().unwrap(), reqwest::cookie::SameSite::Lax); + + // samesitestrict + assert_eq!(cookies[8].name(), "samesitestrict"); + assert_eq!(cookies[8].same_site().unwrap(), reqwest::cookie::SameSite::Strict); +} + +#[test] +fn cookie_store_simple() { + let mut rt = tokio::runtime::current_thread::Runtime::new().expect("new rt"); + let client = reqwest::async::Client::builder().cookie_store(true).build().unwrap(); + + let server = server! { + request: b"\ + GET / HTTP/1.1\r\n\ + user-agent: $USERAGENT\r\n\ + accept: */*\r\n\ + accept-encoding: gzip\r\n\ + host: $HOST\r\n\ + \r\n\ + ", + response: b"\ + HTTP/1.1 200 OK\r\n\ + Set-Cookie: key=val\r\n\ + Content-Length: 0\r\n\ + \r\n\ + " + }; + let url = format!("http://{}/", server.addr()); + rt.block_on(client.get(&url).send()).unwrap(); + + let server = server! { + request: b"\ + GET / HTTP/1.1\r\n\ + user-agent: $USERAGENT\r\n\ + accept: */*\r\n\ + cookie: key=val\r\n\ + accept-encoding: gzip\r\n\ + host: $HOST\r\n\ + \r\n\ + ", + response: b"\ + HTTP/1.1 200 OK\r\n\ + Content-Length: 0\r\n\ + \r\n\ + " + }; + let url = format!("http://{}/", server.addr()); + rt.block_on(client.get(&url).send()).unwrap(); +} + +#[test] +fn cookie_store_overwrite_existing() { + let mut rt = tokio::runtime::current_thread::Runtime::new().expect("new rt"); + let client = reqwest::async::Client::builder().cookie_store(true).build().unwrap(); + + let server = server! { + request: b"\ + GET / HTTP/1.1\r\n\ + user-agent: $USERAGENT\r\n\ + accept: */*\r\n\ + accept-encoding: gzip\r\n\ + host: $HOST\r\n\ + \r\n\ + ", + response: b"\ + HTTP/1.1 200 OK\r\n\ + Set-Cookie: key=val\r\n\ + Content-Length: 0\r\n\ + \r\n\ + " + }; + let url = format!("http://{}/", server.addr()); + rt.block_on(client.get(&url).send()).unwrap(); + + let server = server! { + request: b"\ + GET / HTTP/1.1\r\n\ + user-agent: $USERAGENT\r\n\ + accept: */*\r\n\ + cookie: key=val\r\n\ + accept-encoding: gzip\r\n\ + host: $HOST\r\n\ + \r\n\ + ", + response: b"\ + HTTP/1.1 200 OK\r\n\ + Set-Cookie: key=val2\r\n\ + Content-Length: 0\r\n\ + \r\n\ + " + }; + let url = format!("http://{}/", server.addr()); + rt.block_on(client.get(&url).send()).unwrap(); + + let server = server! { + request: b"\ + GET / HTTP/1.1\r\n\ + user-agent: $USERAGENT\r\n\ + accept: */*\r\n\ + cookie: key=val2\r\n\ + accept-encoding: gzip\r\n\ + host: $HOST\r\n\ + \r\n\ + ", + response: b"\ + HTTP/1.1 200 OK\r\n\ + Content-Length: 0\r\n\ + \r\n\ + " + }; + let url = format!("http://{}/", server.addr()); + rt.block_on(client.get(&url).send()).unwrap(); +} + +#[test] +fn cookie_store_max_age() { + let mut rt = tokio::runtime::current_thread::Runtime::new().expect("new rt"); + let client = reqwest::async::Client::builder().cookie_store(true).build().unwrap(); + + let server = server! { + request: b"\ + GET / HTTP/1.1\r\n\ + user-agent: $USERAGENT\r\n\ + accept: */*\r\n\ + accept-encoding: gzip\r\n\ + host: $HOST\r\n\ + \r\n\ + ", + response: b"\ + HTTP/1.1 200 OK\r\n\ + Set-Cookie: key=val; Max-Age=0\r\n\ + Content-Length: 0\r\n\ + \r\n\ + " + }; + let url = format!("http://{}/", server.addr()); + rt.block_on(client.get(&url).send()).unwrap(); + + let server = server! { + request: b"\ + GET / HTTP/1.1\r\n\ + user-agent: $USERAGENT\r\n\ + accept: */*\r\n\ + accept-encoding: gzip\r\n\ + host: $HOST\r\n\ + \r\n\ + ", + response: b"\ + HTTP/1.1 200 OK\r\n\ + Content-Length: 0\r\n\ + \r\n\ + " + }; + let url = format!("http://{}/", server.addr()); + rt.block_on(client.get(&url).send()).unwrap(); +} + +#[test] +fn cookie_store_expires() { + let mut rt = tokio::runtime::current_thread::Runtime::new().expect("new rt"); + let client = reqwest::async::Client::builder().cookie_store(true).build().unwrap(); + + let server = server! { + request: b"\ + GET / HTTP/1.1\r\n\ + user-agent: $USERAGENT\r\n\ + accept: */*\r\n\ + accept-encoding: gzip\r\n\ + host: $HOST\r\n\ + \r\n\ + ", + response: b"\ + HTTP/1.1 200 OK\r\n\ + Set-Cookie: key=val; Expires=Wed, 21 Oct 2015 07:28:00 GMT\r\n\ + Content-Length: 0\r\n\ + \r\n\ + " + }; + let url = format!("http://{}/", server.addr()); + rt.block_on(client.get(&url).send()).unwrap(); + + let server = server! { + request: b"\ + GET / HTTP/1.1\r\n\ + user-agent: $USERAGENT\r\n\ + accept: */*\r\n\ + accept-encoding: gzip\r\n\ + host: $HOST\r\n\ + \r\n\ + ", + response: b"\ + HTTP/1.1 200 OK\r\n\ + Content-Length: 0\r\n\ + \r\n\ + " + }; + let url = format!("http://{}/", server.addr()); + rt.block_on(client.get(&url).send()).unwrap(); +} + +#[test] +fn cookie_store_path() { + let mut rt = tokio::runtime::current_thread::Runtime::new().expect("new rt"); + let client = reqwest::async::Client::builder().cookie_store(true).build().unwrap(); + + let server = server! { + request: b"\ + GET / HTTP/1.1\r\n\ + user-agent: $USERAGENT\r\n\ + accept: */*\r\n\ + accept-encoding: gzip\r\n\ + host: $HOST\r\n\ + \r\n\ + ", + response: b"\ + HTTP/1.1 200 OK\r\n\ + Set-Cookie: key=val; Path=/subpath\r\n\ + Content-Length: 0\r\n\ + \r\n\ + " + }; + let url = format!("http://{}/", server.addr()); + rt.block_on(client.get(&url).send()).unwrap(); + + let server = server! { + request: b"\ + GET / HTTP/1.1\r\n\ + user-agent: $USERAGENT\r\n\ + accept: */*\r\n\ + accept-encoding: gzip\r\n\ + host: $HOST\r\n\ + \r\n\ + ", + response: b"\ + HTTP/1.1 200 OK\r\n\ + Content-Length: 0\r\n\ + \r\n\ + " + }; + let url = format!("http://{}/", server.addr()); + rt.block_on(client.get(&url).send()).unwrap(); + + let server = server! { + request: b"\ + GET /subpath HTTP/1.1\r\n\ + user-agent: $USERAGENT\r\n\ + accept: */*\r\n\ + cookie: key=val\r\n\ + accept-encoding: gzip\r\n\ + host: $HOST\r\n\ + \r\n\ + ", + response: b"\ + HTTP/1.1 200 OK\r\n\ + Content-Length: 0\r\n\ + \r\n\ + " + }; + let url = format!("http://{}/subpath", server.addr()); + rt.block_on(client.get(&url).send()).unwrap(); +}