forked/h2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SETTINGS_MAX_HEADER_LIST_SIZE (#206)

Browse Source 
This, uh, grew into something far bigger than expected, but it turns out, all of it was needed to eventually support this correctly.

- Adds configuration to client and server to set [SETTINGS_MAX_HEADER_LIST_SIZE](http://httpwg.org/specs/rfc7540.html#SETTINGS_MAX_HEADER_LIST_SIZE)
- If not set, a "sane default" of 16 MB is used (taken from golang's http2)
- Decoding header blocks now happens as they are received, instead of buffering up possibly forever until the last continuation frame is parsed.
- As each field is decoded, it's undecoded size is added to the total. Whenever a header block goes over the maximum size, the `frame` will be marked as such.
- Whenever a header block is deemed over max limit, decoding will still continue, but new fields will not be appended to `HeaderMap`. This is also can save wasted hashing.
- To protect against enormous string literals, such that they span multiple continuation frames, a check is made that the combined encoded bytes is less than the max allowed size. While technically not exactly what the spec suggests (counting decoded size instead), this should hopefully only happen when someone is indeed malicious. If found, a `GOAWAY` of `COMPRESSION_ERROR` is sent, and the connection shut down.
- After an oversize header block frame is finished decoding, the streams state machine will notice it is oversize, and handle that.
  - If the local peer is a server, a 431 response is sent, as suggested by the spec.
  - A `REFUSED_STREAM` reset is sent, since we cannot actually give the stream to the user.
- In order to be able to send both the 431 headers frame, and a reset frame afterwards, the scheduled `Canceled` machinery was made more general to a `Scheduled(Reason)` state instead.

Closes #18 
Closes #191
This commit is contained in:
Sean McArthur
2018-01-05 09:23:48 -08:00
committed by GitHub
parent 6f7b826b0a
commit aa23a9735d
26 changed files with 752 additions and 226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/codec/error.rs
View File

@@ -54,6 +54,15 @@ pub enum UserError {
// ===== impl RecvError =====
impl RecvError {
pub(crate) fn is_stream_error(&self) -> bool {
match *self {
RecvError::Stream { .. } => true,
_ => false,
}
}
}
impl From<io::Error> for RecvError {
fn from(src: io::Error) -> Self {
RecvError::Io(src)

243
src/codec/framed_read.rs
View File

@@ -13,6 +13,9 @@ use std::io;
use tokio_io::AsyncRead;
use tokio_io::codec::length_delimited;
// 16 MB "sane default" taken from golang http2
const DEFAULT_SETTINGS_MAX_HEADER_LIST_SIZE: usize = 16 << 20;
#[derive(Debug)]
pub struct FramedRead<T> {
inner: length_delimited::FramedRead<T>,
@@ -20,6 +23,8 @@ pub struct FramedRead<T> {
// hpack decoder state
hpack: hpack::Decoder,
max_header_list_size: usize,
partial: Option<Partial>,
}
@@ -36,8 +41,6 @@ struct Partial {
#[derive(Debug)]
enum Continuable {
Headers(frame::Headers),
// Decode the Continuation frame but ignore it...
// Ignore(StreamId),
PushPromise(frame::PushPromise),
}
@@ -46,6 +49,7 @@ impl<T> FramedRead<T> {
FramedRead {
inner: inner,
hpack: hpack::Decoder::new(DEFAULT_SETTINGS_HEADER_TABLE_SIZE),
max_header_list_size: DEFAULT_SETTINGS_MAX_HEADER_LIST_SIZE,
partial: None,
}
}
@@ -67,6 +71,66 @@ impl<T> FramedRead<T> {
trace!(" -> kind={:?}", kind);
macro_rules! header_block {
($frame:ident, $head:ident, $bytes:ident) => ({
// Drop the frame header
// TODO: Change to drain: carllerche/bytes#130
let _ = $bytes.split_to(frame::HEADER_LEN);
// Parse the header frame w/o parsing the payload
let (mut frame, mut payload) = match frame::$frame::load($head, $bytes) {
Ok(res) => res,
Err(frame::Error::InvalidDependencyId) => {
debug!("stream error PROTOCOL_ERROR -- invalid HEADERS dependency ID");
// A stream cannot depend on itself. An endpoint MUST
// treat this as a stream error (Section 5.4.2) of type
// `PROTOCOL_ERROR`.
return Err(Stream {
id: $head.stream_id(),
reason: Reason::PROTOCOL_ERROR,
});
},
Err(e) => {
debug!("connection error PROTOCOL_ERROR -- failed to load frame; err={:?}", e);
return Err(Connection(Reason::PROTOCOL_ERROR));
}
};
let is_end_headers = frame.is_end_headers();
// Load the HPACK encoded headers
match frame.load_hpack(&mut payload, self.max_header_list_size, &mut self.hpack) {
Ok(_) => {},
Err(frame::Error::Hpack(hpack::DecoderError::NeedMore(_))) if !is_end_headers => {},
Err(frame::Error::MalformedMessage) => {
debug!("stream error PROTOCOL_ERROR -- malformed header block");
return Err(Stream {
id: $head.stream_id(),
reason: Reason::PROTOCOL_ERROR,
});
},
Err(e) => {
debug!("connection error PROTOCOL_ERROR -- failed HPACK decoding; err={:?}", e);
return Err(Connection(Reason::PROTOCOL_ERROR));
}
}
if is_end_headers {
frame.into()
} else {
trace!("loaded partial header block");
// Defer returning the frame
self.partial = Some(Partial {
frame: Continuable::$frame(frame),
buf: payload,
});
return Ok(None);
}
});
}
let frame = match kind {
Kind::Settings => {
let res = frame::Settings::load(head, &bytes[frame::HEADER_LEN..]);
@@ -103,56 +167,7 @@ impl<T> FramedRead<T> {
})?.into()
},
Kind::Headers => {
// Drop the frame header
// TODO: Change to drain: carllerche/bytes#130
let _ = bytes.split_to(frame::HEADER_LEN);
// Parse the header frame w/o parsing the payload
let (mut headers, payload) = match frame::Headers::load(head, bytes) {
Ok(res) => res,
Err(frame::Error::InvalidDependencyId) => {
// A stream cannot depend on itself. An endpoint MUST
// treat this as a stream error (Section 5.4.2) of type
// `PROTOCOL_ERROR`.
debug!("stream error PROTOCOL_ERROR -- invalid HEADERS dependency ID");
return Err(Stream {
id: head.stream_id(),
reason: Reason::PROTOCOL_ERROR,
});
},
Err(e) => {
debug!("connection error PROTOCOL_ERROR -- failed to load HEADERS frame; err={:?}", e);
return Err(Connection(Reason::PROTOCOL_ERROR));
}
};
if headers.is_end_headers() {
// Load the HPACK encoded headers & return the frame
match headers.load_hpack(payload, &mut self.hpack) {
Ok(_) => {},
Err(frame::Error::MalformedMessage) => {
debug!("stream error PROTOCOL_ERROR -- malformed HEADERS frame");
return Err(Stream {
id: head.stream_id(),
reason: Reason::PROTOCOL_ERROR,
});
},
Err(e) => {
debug!("connection error PROTOCOL_ERROR -- failed HEADERS frame HPACK decoding; err={:?}", e);
return Err(Connection(Reason::PROTOCOL_ERROR));
}
}
headers.into()
} else {
// Defer loading the frame
self.partial = Some(Partial {
frame: Continuable::Headers(headers),
buf: payload,
});
return Ok(None);
}
header_block!(Headers, head, bytes)
},
Kind::Reset => {
let res = frame::Reset::load(head, &bytes[frame::HEADER_LEN..]);
@@ -163,44 +178,7 @@ impl<T> FramedRead<T> {
res.map_err(|_| Connection(Reason::PROTOCOL_ERROR))?.into()
},
Kind::PushPromise => {
// Drop the frame header
// TODO: Change to drain: carllerche/bytes#130
let _ = bytes.split_to(frame::HEADER_LEN);
// Parse the frame w/o parsing the payload
let (mut push, payload) = frame::PushPromise::load(head, bytes)
.map_err(|e| {
debug!("connection error PROTOCOL_ERROR -- failed to load PUSH_PROMISE frame; err={:?}", e);
Connection(Reason::PROTOCOL_ERROR)
})?;
if push.is_end_headers() {
// Load the HPACK encoded headers & return the frame
match push.load_hpack(payload, &mut self.hpack) {
Ok(_) => {},
Err(frame::Error::MalformedMessage) => {
debug!("stream error PROTOCOL_ERROR -- malformed PUSH_PROMISE frame");
return Err(Stream {
id: head.stream_id(),
reason: Reason::PROTOCOL_ERROR,
});
},
Err(e) => {
debug!("connection error PROTOCOL_ERROR -- failed PUSH_PROMISE frame HPACK decoding; err={:?}", e);
return Err(Connection(Reason::PROTOCOL_ERROR));
}
}
push.into()
} else {
// Defer loading the frame
self.partial = Some(Partial {
frame: Continuable::PushPromise(push),
buf: payload,
});
return Ok(None);
}
header_block!(PushPromise, head, bytes)
},
Kind::Priority => {
if head.stream_id() == 0 {
@@ -224,8 +202,7 @@ impl<T> FramedRead<T> {
}
},
Kind::Continuation => {
// TODO: Un-hack this
let end_of_headers = (head.flag() & 0x4) == 0x4;
let is_end_headers = (head.flag() & 0x4) == 0x4;
let mut partial = match self.partial.take() {
Some(partial) => partial,
@@ -235,22 +212,43 @@ impl<T> FramedRead<T> {
}
};
// Extend the buf
partial.buf.extend_from_slice(&bytes[frame::HEADER_LEN..]);
if !end_of_headers {
self.partial = Some(partial);
return Ok(None);
}
// The stream identifiers must match
if partial.frame.stream_id() != head.stream_id() {
debug!("connection error PROTOCOL_ERROR -- CONTINUATION frame stream ID does not match previous frame stream ID");
return Err(Connection(Reason::PROTOCOL_ERROR));
}
match partial.frame.load_hpack(partial.buf, &mut self.hpack) {
// Extend the buf
if partial.buf.is_empty() {
partial.buf = bytes.split_off(frame::HEADER_LEN);
} else {
if partial.frame.is_over_size() {
// If there was left over bytes previously, they may be
// needed to continue decoding, even though we will
// be ignoring this frame. This is done to keep the HPACK
// decoder state up-to-date.
//
// Still, we need to be careful, because if a malicious
// attacker were to try to send a gigantic string, such
// that it fits over multiple header blocks, we could
// grow memory uncontrollably again, and that'd be a shame.
//
// Instead, we use a simple heuristic to determine if
// we should continue to ignore decoding, or to tell
// the attacker to go away.
if partial.buf.len() + bytes.len() > self.max_header_list_size {
debug!("connection error COMPRESSION_ERROR -- CONTINUATION frame header block size over ignorable limit");
return Err(Connection(Reason::COMPRESSION_ERROR));
}
}
partial.buf.extend_from_slice(&bytes[frame::HEADER_LEN..]);
}
match partial.frame.load_hpack(&mut partial.buf, self.max_header_list_size, &mut self.hpack) {
Ok(_) => {},
Err(frame::Error::Hpack(hpack::DecoderError::NeedMore(_))) if !is_end_headers => {},
Err(frame::Error::MalformedMessage) => {
debug!("stream error PROTOCOL_ERROR -- malformed CONTINUATION frame");
return Err(Stream {
@@ -258,10 +256,18 @@ impl<T> FramedRead<T> {
reason: Reason::PROTOCOL_ERROR,
});
},
Err(_) => return Err(Connection(Reason::PROTOCOL_ERROR)),
Err(e) => {
debug!("connection error PROTOCOL_ERROR -- failed HPACK decoding; err={:?}", e);
return Err(Connection(Reason::PROTOCOL_ERROR));
},
}
partial.frame.into()
if is_end_headers {
partial.frame.into()
} else {
self.partial = Some(partial);
return Ok(None);
}
},
Kind::Unknown => {
// Unknown frames are ignored
@@ -295,6 +301,12 @@ impl<T> FramedRead<T> {
assert!(DEFAULT_MAX_FRAME_SIZE as usize <= val && val <= MAX_MAX_FRAME_SIZE as usize);
self.inner.set_max_frame_length(val)
}
/// Update the max header list size setting.
#[inline]
pub fn set_max_header_list_size(&mut self, val: usize) {
self.max_header_list_size = val;
}
}
impl<T> Stream for FramedRead<T>
@@ -322,14 +334,13 @@ where
}
fn map_err(err: io::Error) -> RecvError {
use std::error::Error;
use tokio_io::codec::length_delimited::FrameTooBig;
if let io::ErrorKind::InvalidData = err.kind() {
// woah, brittle...
// TODO: with tokio-io v0.1.4, we can check
// err.get_ref().is::<tokio_io::length_delimited::FrameTooBig>()
if err.description() == "frame size too big" {
return RecvError::Connection(Reason::FRAME_SIZE_ERROR);
if let Some(custom) = err.get_ref() {
if custom.is::<FrameTooBig>() {
return RecvError::Connection(Reason::FRAME_SIZE_ERROR);
}
}
}
err.into()
@@ -345,14 +356,22 @@ impl Continuable {
}
}
fn is_over_size(&self) -> bool {
match *self {
Continuable::Headers(ref h) => h.is_over_size(),
Continuable::PushPromise(ref p) => p.is_over_size(),
}
}
fn load_hpack(
&mut self,
src: BytesMut,
src: &mut BytesMut,
max_header_list_size: usize,
decoder: &mut hpack::Decoder,
) -> Result<(), frame::Error> {
match *self {
Continuable::Headers(ref mut h) => h.load_hpack(src, decoder),
Continuable::PushPromise(ref mut p) => p.load_hpack(src, decoder),
Continuable::Headers(ref mut h) => h.load_hpack(src, max_header_list_size, decoder),
Continuable::PushPromise(ref mut p) => p.load_hpack(src, max_header_list_size, decoder),
}
}
}

5
src/codec/mod.rs
View File

@@ -90,6 +90,11 @@ impl<T, B> Codec<T, B> {
self.framed_write().set_max_frame_size(val)
}
/// Set the max header list size that can be received.
pub fn set_max_recv_header_list_size(&mut self, val: usize) {
self.inner.set_max_header_list_size(val);
}
/// Get a reference to the inner stream.
#[cfg(feature = "unstable")]
pub fn get_ref(&self) -> &T {