Implement cookie store support

This commit introduces a cookie store / session support
for both the async and the sync client.

Functionality is based on the cookie crate,
which provides a HTTP cookie abstraction,
and the cookie_store crate which provides a
store that handles cookie storage and url/expiration
based cookie resolution for requests.

Changes:
* adds new private dependencies: time, cookie, cookie_store
* a new cookie module which provides wrapper types around
    the dependency crates
* a Response::cookies() accessor for iterating over response cookies
* a ClientBuilder::cookie_store() method that enables session functionality
* addition of a cookie_store member to the async client
* injecting request cookies and persisting response cookies
* cookie tests

NOTE: this commit DOES NOT expose the CookieStore itself,
limiting available functionality.

This is desirable, but omitted for now due to API considerations that should be fleshed out in the future.
This means users do not have direct access to the cookie session for now.
This commit is contained in:
Christoph Herzog
2019-03-22 13:42:48 +01:00
committed by Sean McArthur
parent c45ff29bfb
commit 954fdfae30
8 changed files with 640 additions and 5 deletions

View File

@@ -10,6 +10,9 @@ readme = "README.md"
license = "MIT/Apache-2.0"
categories = ["web-programming::http-client"]
[package.metadata.docs.rs]
all-features = true
[dependencies]
base64 = "0.10"
bytes = "0.4"
@@ -43,6 +46,9 @@ socks = { version = "0.3.2", optional = true }
tokio-rustls = { version = "0.9", optional = true }
trust-dns-resolver = { version = "0.10", optional = true }
webpki-roots = { version = "0.16", optional = true }
cookie_store = "0.5.1"
cookie = "0.11.0"
time = "0.1.42"
[dev-dependencies]
env_logger = "0.6"
@@ -63,6 +69,3 @@ rustls-tls = ["hyper-rustls", "tokio-rustls", "webpki-roots", "rustls", "tls"]
trust-dns = ["trust-dns-resolver"]
hyper-011 = ["hyper-old-types"]
[package.metadata.docs.rs]
all-features = true

View File

@@ -1,5 +1,5 @@
use std::{fmt, str};
use std::sync::Arc;
use std::sync::{Arc, RwLock};
use std::time::Duration;
use std::net::IpAddr;
@@ -31,6 +31,7 @@ use super::request::{Request, RequestBuilder};
use super::response::Response;
use connect::Connector;
use into_url::{expect_uri, try_uri};
use cookie;
use redirect::{self, RedirectPolicy, remove_sensitive_headers};
use {IntoUrl, Method, Proxy, StatusCode, Url};
#[cfg(feature = "tls")]
@@ -82,6 +83,7 @@ struct Config {
http1_title_case_headers: bool,
local_address: Option<IpAddr>,
nodelay: bool,
cookie_store: Option<cookie::CookieStore>,
}
impl ClientBuilder {
@@ -116,7 +118,8 @@ impl ClientBuilder {
http2_only: false,
http1_title_case_headers: false,
local_address: None,
nodelay: false
nodelay: false,
cookie_store: None,
},
}
}
@@ -204,6 +207,8 @@ impl ClientBuilder {
.iter()
.any(|p| p.maybe_has_http_auth());
let cookie_store = config.cookie_store.map(RwLock::new);
Ok(Client {
inner: Arc::new(ClientRef {
gzip: config.gzip,
@@ -213,6 +218,7 @@ impl ClientBuilder {
referer: config.referer,
proxies,
proxies_maybe_http_auth,
cookie_store,
}),
})
}
@@ -388,6 +394,21 @@ impl ClientBuilder {
self.config.local_address = addr.into();
self
}
/// Enable a persistent cookie store for the client.
///
/// Cookies received in responses will be preserved and included in
/// additional requests.
///
/// By default, no cookie store is used.
pub fn cookie_store(mut self, enable: bool) -> ClientBuilder {
self.config.cookie_store = if enable {
Some(cookie::CookieStore::default())
} else {
None
};
self
}
}
type HyperClient = ::hyper::Client<Connector>;
@@ -514,6 +535,23 @@ impl Client {
headers.insert(key, value.clone());
}
// Add cookies from the cookie store.
if let Some(cookie_store_wrapper) = self.inner.cookie_store.as_ref() {
if headers.get(::header::COOKIE).is_none() {
let cookie_store = cookie_store_wrapper.read().unwrap();
let header = cookie_store
.0
.get_request_cookies(&url)
.map(|c| c.encoded().to_string())
.collect::<Vec<_>>()
.join("; ");
if !header.is_empty() {
// TODO: is it safe to unwrap here? Investigate if Cookie content is always valid.
headers.insert(::header::COOKIE, HeaderValue::from_bytes(header.as_bytes()).unwrap());
}
}
}
if self.inner.gzip &&
!headers.contains_key(ACCEPT_ENCODING) &&
!headers.contains_key(RANGE) {
@@ -620,6 +658,7 @@ struct ClientRef {
referer: bool,
proxies: Arc<Vec<Proxy>>,
proxies_maybe_http_auth: bool,
cookie_store: Option<RwLock<cookie::CookieStore>>,
}
pub struct Pending {
@@ -674,6 +713,13 @@ impl Future for PendingRequest {
Async::Ready(res) => res,
Async::NotReady => return Ok(Async::NotReady),
};
if let Some(store_wrapper) = self.client.cookie_store.as_ref() {
let mut store = store_wrapper.write().unwrap();
let cookies = cookie::extract_response_cookies(&res.headers())
.filter_map(|res| res.ok())
.map(|cookie| cookie.into_inner().into_owned());
store.0.store_response_cookies(cookies, &self.url);
}
let should_redirect = match res.status() {
StatusCode::MOVED_PERMANENTLY |
StatusCode::FOUND |

View File

@@ -13,6 +13,7 @@ use serde_json;
use url::Url;
use http;
use cookie;
use super::Decoder;
use super::body::Body;
@@ -69,6 +70,13 @@ impl Response {
&mut self.headers
}
/// Retrieve the cookies contained in the response.
pub fn cookies<'a>(&'a self) -> impl Iterator<
Item = Result<cookie::Cookie<'a>, cookie::CookieParseError>
> + 'a {
cookie::extract_response_cookies(&self.headers)
}
/// Get the final `Url` of this `Response`.
#[inline]
pub fn url(&self) -> &Url {

View File

@@ -366,6 +366,16 @@ impl ClientBuilder {
{
self.with_inner(move |inner| inner.local_address(addr))
}
/// Enable a persistent cookie store for the client.
///
/// Cookies received in responses will be preserved and included in
/// additional requests.
///
/// By default, no cookie store is used.
pub fn cookie_store(self, enable: bool) -> ClientBuilder {
self.with_inner(|inner| inner.cookie_store(enable))
}
}

220
src/cookie.rs Normal file
View File

@@ -0,0 +1,220 @@
//! The cookies module contains types for working with request and response cookies.
use cookie_crate;
use header;
use std::borrow::Cow;
use std::fmt;
use std::time::SystemTime;
/// Convert a time::Tm time to SystemTime.
fn tm_to_systemtime(tm: ::time::Tm) -> SystemTime {
let seconds = tm.to_timespec().sec;
let duration = std::time::Duration::from_secs(seconds.abs() as u64);
if seconds > 0 {
SystemTime::UNIX_EPOCH + duration
} else {
SystemTime::UNIX_EPOCH - duration
}
}
/// Convert a SystemTime to time::Tm.
/// Returns None if the conversion failed.
fn systemtime_to_tm(time: SystemTime) -> Option<::time::Tm> {
let seconds = match time.duration_since(SystemTime::UNIX_EPOCH) {
Ok(duration) => duration.as_secs() as i64,
Err(_) => {
if let Ok(duration) = SystemTime::UNIX_EPOCH.duration_since(time) {
(duration.as_secs() as i64) * -1
} else {
return None;
}
}
};
Some(::time::at_utc(::time::Timespec::new(seconds, 0)))
}
/// Represents the 'SameSite' attribute of a cookie.
#[derive(PartialEq, Eq, Clone, Copy, Debug)]
pub enum SameSite {
/// Strict same-site policy.
Strict,
/// Lax same-site policy.
Lax,
}
impl SameSite {
fn from_inner(value: cookie_crate::SameSite) -> Option<Self> {
match value {
cookie_crate::SameSite::Strict => Some(SameSite::Strict),
cookie_crate::SameSite::Lax => Some(SameSite::Lax),
cookie_crate::SameSite::None => None,
}
}
fn to_inner(value: Option<Self>) -> cookie_crate::SameSite {
match value {
Some(SameSite::Strict) => cookie_crate::SameSite::Strict,
Some(SameSite::Lax) => cookie_crate::SameSite::Lax,
None => cookie_crate::SameSite::None,
}
}
}
/// Error representing a parse failure of a 'Set-Cookie' header.
pub struct CookieParseError(cookie::ParseError);
impl<'a> fmt::Debug for CookieParseError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.0.fmt(f)
}
}
impl<'a> fmt::Display for CookieParseError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.0.fmt(f)
}
}
impl std::error::Error for CookieParseError {}
/// A single HTTP cookie.
pub struct Cookie<'a>(cookie::Cookie<'a>);
impl<'a> fmt::Debug for Cookie<'a> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.0.fmt(f)
}
}
impl Cookie<'static> {
/// Construct a new cookie with the given name and value.
pub fn new<N, V>(name: N, value: V) -> Self
where
N: Into<Cow<'static, str>>,
V: Into<Cow<'static, str>>,
{
Self(cookie::Cookie::new(name, value))
}
}
impl<'a> Cookie<'a> {
fn parse(value: &'a ::header::HeaderValue) -> Result<Cookie<'a>, CookieParseError> {
std::str::from_utf8(value.as_bytes())
.map_err(cookie::ParseError::from)
.and_then(cookie::Cookie::parse)
.map_err(CookieParseError)
.map(Cookie)
}
pub(crate) fn into_inner(self) -> cookie::Cookie<'a> {
self.0
}
/// The name of the cookie.
pub fn name(&self) -> &str {
self.0.name()
}
/// Set the cookie name.
pub fn set_name<P: Into<Cow<'static, str>>>(&mut self, name: P) {
self.0.set_name(name)
}
/// The value of the cookie.
pub fn value(&self) -> &str {
self.0.value()
}
/// Set the cookie value.
pub fn set_value<P: Into<Cow<'static, str>>>(&mut self, value: P) {
self.0.set_value(value)
}
/// Returns true if the 'HttpOnly' directive is enabled.
pub fn http_only(&self) -> bool {
self.0.http_only().unwrap_or(false)
}
/// Set the 'HttpOnly' directive.
pub fn set_http_only(&mut self, value: bool) {
self.0.set_http_only(value)
}
/// Returns true if the 'Secure' directive is enabled.
pub fn secure(&self) -> bool {
self.0.secure().unwrap_or(false)
}
/// Set the 'Secure' directive.
pub fn set_secure(&mut self, value: bool) {
self.0.set_secure(value)
}
/// Returns the 'SameSite' directive if present.
pub fn same_site(&self) -> Option<SameSite> {
self.0.same_site().and_then(SameSite::from_inner)
}
/// Set the 'SameSite" directive.
pub fn set_same_site(&mut self, value: Option<SameSite>) {
self.0.set_same_site(SameSite::to_inner(value))
}
/// Returns the path directive of the cookie, if set.
pub fn path(&self) -> Option<&str> {
self.0.path()
}
/// Set the cookie path.
pub fn set_path<P: Into<Cow<'static, str>>>(&mut self, path: P) {
self.0.set_path(path)
}
/// Returns the domain directive of the cookie, if set.
pub fn domain(&self) -> Option<&str> {
self.0.domain()
}
/// Set the cookie domain.
pub fn set_domain<P: Into<Cow<'static, str>>>(&mut self, domain: P) {
self.0.set_domain(domain)
}
/// Get the Max-Age information.
pub fn max_age(&self) -> Option<std::time::Duration> {
self.0.max_age().map(|d| std::time::Duration::new(d.num_seconds() as u64, 0))
}
/// The cookie expiration time.
pub fn expires(&self) -> Option<SystemTime> {
self.0.expires().map(tm_to_systemtime)
}
/// Set expiration time.
///
/// Currently, providing a `None` value will have no effect.
pub fn set_expires(&mut self, value: Option<SystemTime>) {
if let Some(tm) = value.and_then(systemtime_to_tm) {
self.0.set_expires(tm);
}
}
}
pub(crate) fn extract_response_cookies<'a>(
headers: &'a hyper::HeaderMap,
) -> impl Iterator<Item = Result<Cookie<'a>, CookieParseError>> + 'a {
headers
.get_all(header::SET_COOKIE)
.iter()
.map(|value| Cookie::parse(value))
}
/// A persistent cookie store that provides session support.
#[derive(Default)]
pub(crate) struct CookieStore(pub(crate) ::cookie_store::CookieStore);
impl<'a> fmt::Debug for CookieStore {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.0.fmt(f)
}
}

View File

@@ -173,6 +173,8 @@
extern crate base64;
extern crate bytes;
extern crate cookie as cookie_crate;
extern crate cookie_store;
extern crate encoding_rs;
#[macro_use]
extern crate futures;
@@ -195,6 +197,7 @@ extern crate serde;
extern crate serde_derive;
extern crate serde_json;
extern crate serde_urlencoded;
extern crate time;
extern crate tokio;
#[cfg_attr(feature = "default-tls", macro_use)]
extern crate tokio_io;
@@ -241,6 +244,7 @@ mod async_impl;
mod connect;
mod body;
mod client;
pub mod cookie;
#[cfg(feature = "trust-dns")]
mod dns;
mod into_url;

View File

@@ -12,6 +12,7 @@ use mime::Mime;
use serde::de::DeserializeOwned;
use serde_json;
use cookie;
use client::KeepCoreThreadAlive;
use hyper::header::HeaderMap;
use {async_impl, StatusCode, Url, Version, wait};
@@ -122,6 +123,14 @@ impl Response {
self.inner.headers()
}
/// Retrieve the cookies contained in the response.
pub fn cookies<'a>(&'a self) -> impl Iterator<
Item = Result<cookie::Cookie<'a>, cookie::CookieParseError>
> + 'a {
cookie::extract_response_cookies(self.headers())
}
/// Get the HTTP `Version` of this `Response`.
#[inline]
pub fn version(&self) -> Version {

335
tests/cookie.rs Normal file
View File

@@ -0,0 +1,335 @@
extern crate reqwest;
#[macro_use]
mod support;
#[test]
fn cookie_response_accessor() {
let mut rt = tokio::runtime::current_thread::Runtime::new().expect("new rt");
let client = reqwest::async::Client::new();
let server = server! {
request: b"\
GET / HTTP/1.1\r\n\
user-agent: $USERAGENT\r\n\
accept: */*\r\n\
accept-encoding: gzip\r\n\
host: $HOST\r\n\
\r\n\
",
response: b"\
HTTP/1.1 200 OK\r\n\
Set-Cookie: key=val\r\n\
Set-Cookie: expires=1; Expires=Wed, 21 Oct 2015 07:28:00 GMT\r\n\
Set-Cookie: path=1; Path=/the-path\r\n\
Set-Cookie: maxage=1; Max-Age=100\r\n\
Set-Cookie: domain=1; Domain=mydomain\r\n\
Set-Cookie: secure=1; Secure\r\n\
Set-Cookie: httponly=1; HttpOnly\r\n\
Set-Cookie: samesitelax=1; SameSite=Lax\r\n\
Set-Cookie: samesitestrict=1; SameSite=Strict\r\n\
Content-Length: 0\r\n\
\r\n\
"
};
let url = format!("http://{}/", server.addr());
let res = rt.block_on(client.get(&url).send()).unwrap();
let cookies = res.cookies().map(|c| c.unwrap()).collect::<Vec<_>>();
// key=val
assert_eq!(cookies[0].name(), "key");
assert_eq!(cookies[0].value(), "val");
// expires
assert_eq!(cookies[1].name(), "expires");
assert_eq!(
cookies[1].expires().unwrap(),
std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1445412480)
);
// path
assert_eq!(cookies[2].name(), "path");
assert_eq!(cookies[2].path().unwrap(), "/the-path");
// max-age
assert_eq!(cookies[3].name(), "maxage");
assert_eq!(cookies[3].max_age().unwrap(), std::time::Duration::from_secs(100));
// domain
assert_eq!(cookies[4].name(), "domain");
assert_eq!(cookies[4].domain().unwrap(), "mydomain");
// secure
assert_eq!(cookies[5].name(), "secure");
assert_eq!(cookies[5].secure(), true);
// httponly
assert_eq!(cookies[6].name(), "httponly");
assert_eq!(cookies[6].http_only(), true);
// samesitelax
assert_eq!(cookies[7].name(), "samesitelax");
assert_eq!(cookies[7].same_site().unwrap(), reqwest::cookie::SameSite::Lax);
// samesitestrict
assert_eq!(cookies[8].name(), "samesitestrict");
assert_eq!(cookies[8].same_site().unwrap(), reqwest::cookie::SameSite::Strict);
}
#[test]
fn cookie_store_simple() {
let mut rt = tokio::runtime::current_thread::Runtime::new().expect("new rt");
let client = reqwest::async::Client::builder().cookie_store(true).build().unwrap();
let server = server! {
request: b"\
GET / HTTP/1.1\r\n\
user-agent: $USERAGENT\r\n\
accept: */*\r\n\
accept-encoding: gzip\r\n\
host: $HOST\r\n\
\r\n\
",
response: b"\
HTTP/1.1 200 OK\r\n\
Set-Cookie: key=val\r\n\
Content-Length: 0\r\n\
\r\n\
"
};
let url = format!("http://{}/", server.addr());
rt.block_on(client.get(&url).send()).unwrap();
let server = server! {
request: b"\
GET / HTTP/1.1\r\n\
user-agent: $USERAGENT\r\n\
accept: */*\r\n\
cookie: key=val\r\n\
accept-encoding: gzip\r\n\
host: $HOST\r\n\
\r\n\
",
response: b"\
HTTP/1.1 200 OK\r\n\
Content-Length: 0\r\n\
\r\n\
"
};
let url = format!("http://{}/", server.addr());
rt.block_on(client.get(&url).send()).unwrap();
}
#[test]
fn cookie_store_overwrite_existing() {
let mut rt = tokio::runtime::current_thread::Runtime::new().expect("new rt");
let client = reqwest::async::Client::builder().cookie_store(true).build().unwrap();
let server = server! {
request: b"\
GET / HTTP/1.1\r\n\
user-agent: $USERAGENT\r\n\
accept: */*\r\n\
accept-encoding: gzip\r\n\
host: $HOST\r\n\
\r\n\
",
response: b"\
HTTP/1.1 200 OK\r\n\
Set-Cookie: key=val\r\n\
Content-Length: 0\r\n\
\r\n\
"
};
let url = format!("http://{}/", server.addr());
rt.block_on(client.get(&url).send()).unwrap();
let server = server! {
request: b"\
GET / HTTP/1.1\r\n\
user-agent: $USERAGENT\r\n\
accept: */*\r\n\
cookie: key=val\r\n\
accept-encoding: gzip\r\n\
host: $HOST\r\n\
\r\n\
",
response: b"\
HTTP/1.1 200 OK\r\n\
Set-Cookie: key=val2\r\n\
Content-Length: 0\r\n\
\r\n\
"
};
let url = format!("http://{}/", server.addr());
rt.block_on(client.get(&url).send()).unwrap();
let server = server! {
request: b"\
GET / HTTP/1.1\r\n\
user-agent: $USERAGENT\r\n\
accept: */*\r\n\
cookie: key=val2\r\n\
accept-encoding: gzip\r\n\
host: $HOST\r\n\
\r\n\
",
response: b"\
HTTP/1.1 200 OK\r\n\
Content-Length: 0\r\n\
\r\n\
"
};
let url = format!("http://{}/", server.addr());
rt.block_on(client.get(&url).send()).unwrap();
}
#[test]
fn cookie_store_max_age() {
let mut rt = tokio::runtime::current_thread::Runtime::new().expect("new rt");
let client = reqwest::async::Client::builder().cookie_store(true).build().unwrap();
let server = server! {
request: b"\
GET / HTTP/1.1\r\n\
user-agent: $USERAGENT\r\n\
accept: */*\r\n\
accept-encoding: gzip\r\n\
host: $HOST\r\n\
\r\n\
",
response: b"\
HTTP/1.1 200 OK\r\n\
Set-Cookie: key=val; Max-Age=0\r\n\
Content-Length: 0\r\n\
\r\n\
"
};
let url = format!("http://{}/", server.addr());
rt.block_on(client.get(&url).send()).unwrap();
let server = server! {
request: b"\
GET / HTTP/1.1\r\n\
user-agent: $USERAGENT\r\n\
accept: */*\r\n\
accept-encoding: gzip\r\n\
host: $HOST\r\n\
\r\n\
",
response: b"\
HTTP/1.1 200 OK\r\n\
Content-Length: 0\r\n\
\r\n\
"
};
let url = format!("http://{}/", server.addr());
rt.block_on(client.get(&url).send()).unwrap();
}
#[test]
fn cookie_store_expires() {
let mut rt = tokio::runtime::current_thread::Runtime::new().expect("new rt");
let client = reqwest::async::Client::builder().cookie_store(true).build().unwrap();
let server = server! {
request: b"\
GET / HTTP/1.1\r\n\
user-agent: $USERAGENT\r\n\
accept: */*\r\n\
accept-encoding: gzip\r\n\
host: $HOST\r\n\
\r\n\
",
response: b"\
HTTP/1.1 200 OK\r\n\
Set-Cookie: key=val; Expires=Wed, 21 Oct 2015 07:28:00 GMT\r\n\
Content-Length: 0\r\n\
\r\n\
"
};
let url = format!("http://{}/", server.addr());
rt.block_on(client.get(&url).send()).unwrap();
let server = server! {
request: b"\
GET / HTTP/1.1\r\n\
user-agent: $USERAGENT\r\n\
accept: */*\r\n\
accept-encoding: gzip\r\n\
host: $HOST\r\n\
\r\n\
",
response: b"\
HTTP/1.1 200 OK\r\n\
Content-Length: 0\r\n\
\r\n\
"
};
let url = format!("http://{}/", server.addr());
rt.block_on(client.get(&url).send()).unwrap();
}
#[test]
fn cookie_store_path() {
let mut rt = tokio::runtime::current_thread::Runtime::new().expect("new rt");
let client = reqwest::async::Client::builder().cookie_store(true).build().unwrap();
let server = server! {
request: b"\
GET / HTTP/1.1\r\n\
user-agent: $USERAGENT\r\n\
accept: */*\r\n\
accept-encoding: gzip\r\n\
host: $HOST\r\n\
\r\n\
",
response: b"\
HTTP/1.1 200 OK\r\n\
Set-Cookie: key=val; Path=/subpath\r\n\
Content-Length: 0\r\n\
\r\n\
"
};
let url = format!("http://{}/", server.addr());
rt.block_on(client.get(&url).send()).unwrap();
let server = server! {
request: b"\
GET / HTTP/1.1\r\n\
user-agent: $USERAGENT\r\n\
accept: */*\r\n\
accept-encoding: gzip\r\n\
host: $HOST\r\n\
\r\n\
",
response: b"\
HTTP/1.1 200 OK\r\n\
Content-Length: 0\r\n\
\r\n\
"
};
let url = format!("http://{}/", server.addr());
rt.block_on(client.get(&url).send()).unwrap();
let server = server! {
request: b"\
GET /subpath HTTP/1.1\r\n\
user-agent: $USERAGENT\r\n\
accept: */*\r\n\
cookie: key=val\r\n\
accept-encoding: gzip\r\n\
host: $HOST\r\n\
\r\n\
",
response: b"\
HTTP/1.1 200 OK\r\n\
Content-Length: 0\r\n\
\r\n\
"
};
let url = format!("http://{}/subpath", server.addr());
rt.block_on(client.get(&url).send()).unwrap();
}