forked/h2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SETTINGS_MAX_HEADER_LIST_SIZE (#206)

Browse Source 
This, uh, grew into something far bigger than expected, but it turns out, all of it was needed to eventually support this correctly.

- Adds configuration to client and server to set [SETTINGS_MAX_HEADER_LIST_SIZE](http://httpwg.org/specs/rfc7540.html#SETTINGS_MAX_HEADER_LIST_SIZE)
- If not set, a "sane default" of 16 MB is used (taken from golang's http2)
- Decoding header blocks now happens as they are received, instead of buffering up possibly forever until the last continuation frame is parsed.
- As each field is decoded, it's undecoded size is added to the total. Whenever a header block goes over the maximum size, the `frame` will be marked as such.
- Whenever a header block is deemed over max limit, decoding will still continue, but new fields will not be appended to `HeaderMap`. This is also can save wasted hashing.
- To protect against enormous string literals, such that they span multiple continuation frames, a check is made that the combined encoded bytes is less than the max allowed size. While technically not exactly what the spec suggests (counting decoded size instead), this should hopefully only happen when someone is indeed malicious. If found, a `GOAWAY` of `COMPRESSION_ERROR` is sent, and the connection shut down.
- After an oversize header block frame is finished decoding, the streams state machine will notice it is oversize, and handle that.
  - If the local peer is a server, a 431 response is sent, as suggested by the spec.
  - A `REFUSED_STREAM` reset is sent, since we cannot actually give the stream to the user.
- In order to be able to send both the 431 headers frame, and a reset frame afterwards, the scheduled `Canceled` machinery was made more general to a `Scheduled(Reason)` state instead.

Closes #18 
Closes #191
This commit is contained in:
Sean McArthur
2018-01-05 09:23:48 -08:00
committed by GitHub
parent 6f7b826b0a
commit aa23a9735d
26 changed files with 752 additions and 226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
src/hpack/decoder.rs
View File

@@ -34,10 +34,15 @@ pub enum DecoderError {
InvalidStatusCode,
InvalidPseudoheader,
InvalidMaxDynamicSize,
IntegerUnderflow,
IntegerOverflow,
StringUnderflow,
NeedMore(NeedMore),
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum NeedMore {
UnexpectedEndOfStream,
IntegerUnderflow,
StringUnderflow,
}
enum Representation {
@@ -163,7 +168,7 @@ impl Decoder {
}
/// Decodes the headers found in the given buffer.
pub fn decode<F>(&mut self, src: &mut Cursor<Bytes>, mut f: F) -> Result<(), DecoderError>
pub fn decode<F>(&mut self, src: &mut Cursor<&mut BytesMut>, mut f: F) -> Result<(), DecoderError>
where
F: FnMut(Header),
{
@@ -185,7 +190,9 @@ impl Decoder {
Indexed => {
trace!(" Indexed; rem={:?}", src.remaining());
can_resize = false;
f(self.decode_indexed(src)?);
let entry = self.decode_indexed(src)?;
consume(src);
f(entry);
},
LiteralWithIndexing => {
trace!(" LiteralWithIndexing; rem={:?}", src.remaining());
@@ -194,6 +201,7 @@ impl Decoder {
// Insert the header into the table
self.table.insert(entry.clone());
consume(src);
f(entry);
},
@@ -201,12 +209,14 @@ impl Decoder {
trace!(" LiteralWithoutIndexing; rem={:?}", src.remaining());
can_resize = false;
let entry = self.decode_literal(src, false)?;
consume(src);
f(entry);
},
LiteralNeverIndexed => {
trace!(" LiteralNeverIndexed; rem={:?}", src.remaining());
can_resize = false;
let entry = self.decode_literal(src, false)?;
consume(src);
// TODO: Track that this should never be indexed
@@ -220,6 +230,7 @@ impl Decoder {
// Handle the dynamic table size update
self.process_size_update(src)?;
consume(src);
},
}
}
@@ -227,7 +238,7 @@ impl Decoder {
Ok(())
}
fn process_size_update(&mut self, buf: &mut Cursor<Bytes>) -> Result<(), DecoderError> {
fn process_size_update(&mut self, buf: &mut Cursor<&mut BytesMut>) -> Result<(), DecoderError> {
let new_size = decode_int(buf, 5)?;
if new_size > self.last_max_update {
@@ -245,14 +256,14 @@ impl Decoder {
Ok(())
}
fn decode_indexed(&self, buf: &mut Cursor<Bytes>) -> Result<Header, DecoderError> {
fn decode_indexed(&self, buf: &mut Cursor<&mut BytesMut>) -> Result<Header, DecoderError> {
let index = decode_int(buf, 7)?;
self.table.get(index)
}
fn decode_literal(
&mut self,
buf: &mut Cursor<Bytes>,
buf: &mut Cursor<&mut BytesMut>,
index: bool,
) -> Result<Header, DecoderError> {
let prefix = if index { 6 } else { 4 };
@@ -275,13 +286,13 @@ impl Decoder {
}
}
fn decode_string(&mut self, buf: &mut Cursor<Bytes>) -> Result<Bytes, DecoderError> {
fn decode_string(&mut self, buf: &mut Cursor<&mut BytesMut>) -> Result<Bytes, DecoderError> {
const HUFF_FLAG: u8 = 0b10000000;
// The first bit in the first byte contains the huffman encoded flag.
let huff = match peek_u8(buf) {
Some(hdr) => (hdr & HUFF_FLAG) == HUFF_FLAG,
None => return Err(DecoderError::UnexpectedEndOfStream),
None => return Err(DecoderError::NeedMore(NeedMore::UnexpectedEndOfStream)),
};
// Decode the string length using 7 bit prefix
@@ -293,7 +304,7 @@ impl Decoder {
len,
buf.remaining()
);
return Err(DecoderError::StringUnderflow);
return Err(DecoderError::NeedMore(NeedMore::StringUnderflow));
}
if huff {
@@ -358,7 +369,7 @@ fn decode_int<B: Buf>(buf: &mut B, prefix_size: u8) -> Result<usize, DecoderErro
}
if !buf.has_remaining() {
return Err(DecoderError::IntegerUnderflow);
return Err(DecoderError::NeedMore(NeedMore::IntegerUnderflow));
}
let mask = if prefix_size == 8 {
@@ -401,7 +412,7 @@ fn decode_int<B: Buf>(buf: &mut B, prefix_size: u8) -> Result<usize, DecoderErro
}
}
Err(DecoderError::IntegerUnderflow)
Err(DecoderError::NeedMore(NeedMore::IntegerUnderflow))
}
fn peek_u8<B: Buf>(buf: &mut B) -> Option<u8> {
@@ -412,11 +423,19 @@ fn peek_u8<B: Buf>(buf: &mut B) -> Option<u8> {
}
}
fn take(buf: &mut Cursor<Bytes>, n: usize) -> Bytes {
fn take(buf: &mut Cursor<&mut BytesMut>, n: usize) -> Bytes {
let pos = buf.position() as usize;
let ret = buf.get_ref().slice(pos, pos + n);
buf.set_position((pos + n) as u64);
ret
let mut head = buf.get_mut().split_to(pos + n);
buf.set_position(0);
head.split_to(pos);
head.freeze()
}
fn consume(buf: &mut Cursor<&mut BytesMut>) {
// remove bytes from the internal BytesMut when they have been successfully
// decoded. This is a more permanent cursor position, which will be
// used to resume if decoding was only partial.
take(buf, 0);
}
// ===== impl Table =====
@@ -778,15 +797,15 @@ fn test_peek_u8() {
#[test]
fn test_decode_string_empty() {
let mut de = Decoder::new(0);
let buf = Bytes::new();
let err = de.decode_string(&mut Cursor::new(buf)).unwrap_err();
assert_eq!(err, DecoderError::UnexpectedEndOfStream);
let mut buf = BytesMut::new();
let err = de.decode_string(&mut Cursor::new(&mut buf)).unwrap_err();
assert_eq!(err, DecoderError::NeedMore(NeedMore::UnexpectedEndOfStream));
}
#[test]
fn test_decode_empty() {
let mut de = Decoder::new(0);
let buf = Bytes::new();
let empty = de.decode(&mut Cursor::new(buf), |_| {}).unwrap();
let mut buf = BytesMut::new();
let empty = de.decode(&mut Cursor::new(&mut buf), |_| {}).unwrap();
assert_eq!(empty, ());
}

2
src/hpack/mod.rs
View File

@@ -7,6 +7,6 @@ mod table;
#[cfg(test)]
mod test;
pub use self::decoder::{Decoder, DecoderError};
pub use self::decoder::{Decoder, DecoderError, NeedMore};
pub use self::encoder::{Encode, EncodeState, Encoder, EncoderError};
pub use self::header::Header;

4
src/hpack/test/fixture.rs
View File

@@ -74,7 +74,7 @@ fn test_story(story: Value) {
}
decoder
.decode(&mut Cursor::new(case.wire.clone().into()), |e| {
.decode(&mut Cursor::new(&mut case.wire.clone().into()), |e| {
let (name, value) = expect.remove(0);
assert_eq!(name, key_str(&e));
assert_eq!(value, value_str(&e));
@@ -108,7 +108,7 @@ fn test_story(story: Value) {
encoder.encode(None, &mut input.clone().into_iter(), &mut buf);
decoder
.decode(&mut Cursor::new(buf.into()), |e| {
.decode(&mut Cursor::new(&mut buf), |e| {
assert_eq!(e, input.remove(0).reify().unwrap());
})
.unwrap();

4
src/hpack/test/fuzz.rs
View File

@@ -149,7 +149,7 @@ impl FuzzHpack {
// Decode the chunk!
decoder
.decode(&mut Cursor::new(buf.into()), |e| {
.decode(&mut Cursor::new(&mut buf), |e| {
assert_eq!(e, expect.remove(0).reify().unwrap());
})
.unwrap();
@@ -161,7 +161,7 @@ impl FuzzHpack {
// Decode the chunk!
decoder
.decode(&mut Cursor::new(buf.into()), |e| {
.decode(&mut Cursor::new(&mut buf), |e| {
assert_eq!(e, expect.remove(0).reify().unwrap());
})
.unwrap();