forked/hyper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(http): introduce MemBuf, a shared read buffer

Browse Source
This commit is contained in:
Sean McArthur
2017-01-07 09:08:17 -08:00
parent 2d2d5574a6
commit be461b4663
13 changed files with 473 additions and 456 deletions
Download Patch File Download Diff File Expand all files Collapse all files

210
src/http/h1/decode.rs
View File

@@ -1,5 +1,8 @@
use std::{cmp, usize};
use std::io::{self, Read};
use std::usize;
use std::io;
use http::buf::MemSlice;
use http::io::MemRead;
use self::Kind::{Length, Chunked, Eof};
@@ -76,15 +79,16 @@ impl Decoder {
}
impl Decoder {
pub fn decode<R: Read>(&mut self, body: &mut R, buf: &mut [u8]) -> io::Result<usize> {
pub fn decode<R: MemRead>(&mut self, body: &mut R) -> io::Result<MemSlice> {
match self.kind {
Length(ref mut remaining) => {
trace!("Sized read, remaining={:?}", remaining);
if *remaining == 0 {
Ok(0)
Ok(MemSlice::empty())
} else {
let to_read = cmp::min(*remaining as usize, buf.len());
let num = try!(body.read(&mut buf[..to_read])) as u64;
let to_read = *remaining as usize;
let buf = try!(body.read_mem(to_read));
let num = buf.len() as u64;
trace!("Length read: {}", num);
if num > *remaining {
*remaining = 0;
@@ -93,30 +97,37 @@ impl Decoder {
} else {
*remaining -= num;
}
Ok(num as usize)
Ok(buf)
}
}
Chunked(ref mut state, ref mut size) => {
loop {
let mut read = 0;
let mut buf = None;
// advances the chunked state
*state = try!(state.step(body, size, buf, &mut read));
*state = try!(state.step(body, size, &mut buf));
if *state == ChunkedState::End {
trace!("end of chunked");
return Ok(0);
return Ok(MemSlice::empty());
}
if read > 0 {
return Ok(read);
if let Some(buf) = buf {
return Ok(buf);
}
}
}
Eof(ref mut is_eof) => {
match body.read(buf) {
Ok(0) => {
*is_eof = true;
Ok(0)
if *is_eof {
Ok(MemSlice::empty())
} else {
// 8192 chosen because its about 2 packets, there probably
// won't be that much available, so don't have MemReaders
// allocate buffers to big
match body.read_mem(8192) {
Ok(slice) => {
*is_eof = slice.is_empty();
Ok(slice)
}
other => other,
}
other => other,
}
}
}
@@ -125,29 +136,29 @@ impl Decoder {
macro_rules! byte (
($rdr:ident) => ({
let mut buf = [0];
match try!($rdr.read(&mut buf)) {
1 => buf[0],
_ => return Err(io::Error::new(io::ErrorKind::UnexpectedEof,
"Unexpected eof during chunk size line")),
let buf = try!($rdr.read_mem(1));
if !buf.is_empty() {
buf[0]
} else {
return Err(io::Error::new(io::ErrorKind::UnexpectedEof,
"Unexpected eof during chunk size line"));
}
})
);
impl ChunkedState {
fn step<R: Read>(&self,
body: &mut R,
size: &mut u64,
buf: &mut [u8],
read: &mut usize)
-> io::Result<ChunkedState> {
fn step<R: MemRead>(&self,
body: &mut R,
size: &mut u64,
buf: &mut Option<MemSlice>)
-> io::Result<ChunkedState> {
use self::ChunkedState::*;
Ok(match *self {
Size => try!(ChunkedState::read_size(body, size)),
SizeLws => try!(ChunkedState::read_size_lws(body)),
Extension => try!(ChunkedState::read_extension(body)),
SizeLf => try!(ChunkedState::read_size_lf(body, size)),
Body => try!(ChunkedState::read_body(body, size, buf, read)),
Body => try!(ChunkedState::read_body(body, size, buf)),
BodyCr => try!(ChunkedState::read_body_cr(body)),
BodyLf => try!(ChunkedState::read_body_lf(body)),
EndCr => try!(ChunkedState::read_end_cr(body)),
@@ -155,8 +166,8 @@ impl ChunkedState {
End => ChunkedState::End,
})
}
fn read_size<R: Read>(rdr: &mut R, size: &mut u64) -> io::Result<ChunkedState> {
trace!("Read size");
fn read_size<R: MemRead>(rdr: &mut R, size: &mut u64) -> io::Result<ChunkedState> {
trace!("Read chunk hex size");
let radix = 16;
match byte!(rdr) {
b @ b'0'...b'9' => {
@@ -181,7 +192,7 @@ impl ChunkedState {
}
Ok(ChunkedState::Size)
}
fn read_size_lws<R: Read>(rdr: &mut R) -> io::Result<ChunkedState> {
fn read_size_lws<R: MemRead>(rdr: &mut R) -> io::Result<ChunkedState> {
trace!("read_size_lws");
match byte!(rdr) {
// LWS can follow the chunk size, but no more digits can come
@@ -194,14 +205,14 @@ impl ChunkedState {
}
}
}
fn read_extension<R: Read>(rdr: &mut R) -> io::Result<ChunkedState> {
fn read_extension<R: MemRead>(rdr: &mut R) -> io::Result<ChunkedState> {
trace!("read_extension");
match byte!(rdr) {
b'\r' => return Ok(ChunkedState::SizeLf),
_ => return Ok(ChunkedState::Extension), // no supported extensions
}
}
fn read_size_lf<R: Read>(rdr: &mut R, size: &mut u64) -> io::Result<ChunkedState> {
fn read_size_lf<R: MemRead>(rdr: &mut R, size: &mut u64) -> io::Result<ChunkedState> {
trace!("Chunk size is {:?}", size);
match byte!(rdr) {
b'\n' if *size > 0 => Ok(ChunkedState::Body),
@@ -210,10 +221,9 @@ impl ChunkedState {
}
}
fn read_body<R: Read>(rdr: &mut R,
fn read_body<R: MemRead>(rdr: &mut R,
rem: &mut u64,
buf: &mut [u8],
read: &mut usize)
buf: &mut Option<MemSlice>)
-> io::Result<ChunkedState> {
trace!("Chunked read, remaining={:?}", rem);
@@ -223,19 +233,16 @@ impl ChunkedState {
r => r as usize,
};
let to_read = cmp::min(rem_cap, buf.len());
let count = try!(rdr.read(&mut buf[..to_read]));
trace!("to_read = {}", to_read);
trace!("count = {}", count);
let to_read = rem_cap;
let slice = try!(rdr.read_mem(to_read));
let count = slice.len();
if count == 0 {
*rem = 0;
return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "early eof"));
}
*buf = Some(slice);
*rem -= count as u64;
*read = count;
if *rem > 0 {
Ok(ChunkedState::Body)
@@ -243,26 +250,26 @@ impl ChunkedState {
Ok(ChunkedState::BodyCr)
}
}
fn read_body_cr<R: Read>(rdr: &mut R) -> io::Result<ChunkedState> {
fn read_body_cr<R: MemRead>(rdr: &mut R) -> io::Result<ChunkedState> {
match byte!(rdr) {
b'\r' => Ok(ChunkedState::BodyLf),
_ => Err(io::Error::new(io::ErrorKind::InvalidInput, "Invalid chunk body CR")),
}
}
fn read_body_lf<R: Read>(rdr: &mut R) -> io::Result<ChunkedState> {
fn read_body_lf<R: MemRead>(rdr: &mut R) -> io::Result<ChunkedState> {
match byte!(rdr) {
b'\n' => Ok(ChunkedState::Size),
_ => Err(io::Error::new(io::ErrorKind::InvalidInput, "Invalid chunk body LF")),
}
}
fn read_end_cr<R: Read>(rdr: &mut R) -> io::Result<ChunkedState> {
fn read_end_cr<R: MemRead>(rdr: &mut R) -> io::Result<ChunkedState> {
match byte!(rdr) {
b'\r' => Ok(ChunkedState::EndLf),
_ => Err(io::Error::new(io::ErrorKind::InvalidInput, "Invalid chunk end CR")),
}
}
fn read_end_lf<R: Read>(rdr: &mut R) -> io::Result<ChunkedState> {
fn read_end_lf<R: MemRead>(rdr: &mut R) -> io::Result<ChunkedState> {
match byte!(rdr) {
b'\n' => Ok(ChunkedState::End),
_ => Err(io::Error::new(io::ErrorKind::InvalidInput, "Invalid chunk end LF")),
@@ -277,8 +284,23 @@ mod tests {
use std::io::Write;
use super::Decoder;
use super::ChunkedState;
use http::io::MemRead;
use http::buf::{MemBuf, MemSlice};
use mock::AsyncIo;
impl<'a> MemRead for &'a [u8] {
fn read_mem(&mut self, len: usize) -> io::Result<MemSlice> {
let n = ::std::cmp::min(len, self.len());
if n > 0 {
let mut buf = MemBuf::with_capacity(n);
buf.read_from(self).unwrap();
Ok(buf.slice(n))
} else {
Ok(MemSlice::empty())
}
}
}
#[test]
fn test_read_chunk_size() {
use std::io::ErrorKind::{UnexpectedEof, InvalidInput};
@@ -287,13 +309,10 @@ mod tests {
let mut state = ChunkedState::Size;
let mut rdr = &mut s.as_bytes();
let mut size = 0;
let mut count = 0;
loop {
let mut buf = [0u8; 10];
let result = state.step(&mut rdr, &mut size, &mut buf, &mut count);
let result = state.step(rdr, &mut size, &mut None);
let desc = format!("read_size failed for {:?}", s);
state = result.expect(desc.as_str());
trace!("State {:?}", state);
if state == ChunkedState::Body || state == ChunkedState::EndCr {
break;
}
@@ -305,10 +324,8 @@ mod tests {
let mut state = ChunkedState::Size;
let mut rdr = &mut s.as_bytes();
let mut size = 0;
let mut count = 0;
loop {
let mut buf = [0u8; 10];
let result = state.step(&mut rdr, &mut size, &mut buf, &mut count);
let result = state.step(rdr, &mut size, &mut None);
state = match result {
Ok(s) => s,
Err(e) => {
@@ -317,7 +334,6 @@ mod tests {
return;
}
};
trace!("State {:?}", state);
if state == ChunkedState::Body || state == ChunkedState::End {
panic!(format!("Was Ok. Expected Err for {:?}", s));
}
@@ -359,9 +375,8 @@ mod tests {
fn test_read_sized_early_eof() {
let mut bytes = &b"foo bar"[..];
let mut decoder = Decoder::length(10);
let mut buf = [0u8; 10];
assert_eq!(decoder.decode(&mut bytes, &mut buf).unwrap(), 7);
let e = decoder.decode(&mut bytes, &mut buf).unwrap_err();
assert_eq!(decoder.decode(&mut bytes).unwrap().len(), 7);
let e = decoder.decode(&mut bytes).unwrap_err();
assert_eq!(e.kind(), io::ErrorKind::Other);
assert_eq!(e.description(), "early eof");
}
@@ -373,68 +388,63 @@ mod tests {
foo bar\
"[..];
let mut decoder = Decoder::chunked();
let mut buf = [0u8; 10];
assert_eq!(decoder.decode(&mut bytes, &mut buf).unwrap(), 7);
let e = decoder.decode(&mut bytes, &mut buf).unwrap_err();
assert_eq!(decoder.decode(&mut bytes).unwrap().len(), 7);
let e = decoder.decode(&mut bytes).unwrap_err();
assert_eq!(e.kind(), io::ErrorKind::UnexpectedEof);
assert_eq!(e.description(), "early eof");
}
#[test]
fn test_read_chunked_single_read() {
let content = b"10\r\n1234567890abcdef\r\n0\r\n";
let mut mock_buf = io::Cursor::new(content);
let mut buf = [0u8; 16];
let count = Decoder::chunked().decode(&mut mock_buf, &mut buf).expect("decode");
assert_eq!(16, count);
let mut mock_buf = &b"10\r\n1234567890abcdef\r\n0\r\n"[..];
let buf = Decoder::chunked().decode(&mut mock_buf).expect("decode");
assert_eq!(16, buf.len());
let result = String::from_utf8(buf.to_vec()).expect("decode String");
assert_eq!("1234567890abcdef", &result);
}
#[test]
fn test_read_chunked_after_eof() {
let content = b"10\r\n1234567890abcdef\r\n0\r\n\r\n";
let mut mock_buf = io::Cursor::new(content);
let mut buf = [0u8; 50];
let mut mock_buf = &b"10\r\n1234567890abcdef\r\n0\r\n\r\n"[..];
let mut decoder = Decoder::chunked();
// normal read
let count = decoder.decode(&mut mock_buf, &mut buf).expect("decode");
assert_eq!(16, count);
let result = String::from_utf8(buf[0..count].to_vec()).expect("decode String");
let buf = decoder.decode(&mut mock_buf).expect("decode");
assert_eq!(16, buf.len());
let result = String::from_utf8(buf.to_vec()).expect("decode String");
assert_eq!("1234567890abcdef", &result);
// eof read
let count = decoder.decode(&mut mock_buf, &mut buf).expect("decode");
assert_eq!(0, count);
let buf = decoder.decode(&mut mock_buf).expect("decode");
assert_eq!(0, buf.len());
// ensure read after eof also returns eof
let count = decoder.decode(&mut mock_buf, &mut buf).expect("decode");
assert_eq!(0, count);
let buf = decoder.decode(&mut mock_buf).expect("decode");
assert_eq!(0, buf.len());
}
// perform an async read using a custom buffer size and causing a blocking
// read at the specified byte
fn read_async(mut decoder: Decoder,
content: &[u8],
block_at: usize,
read_buffer_size: usize)
block_at: usize)
-> String {
let content_len = content.len();
let mock_buf = io::Cursor::new(content.clone());
let mut ins = AsyncIo::new(mock_buf, block_at);
let mut outs = vec![];
let mut ins = AsyncIo::new(content, block_at);
let mut outs = Vec::new();
loop {
let mut buf = vec![0; read_buffer_size];
match decoder.decode(&mut ins, buf.as_mut_slice()) {
Ok(0) => break,
Ok(i) => outs.write(&buf[0..i]).expect("write buffer"),
Err(e) => {
if e.kind() != io::ErrorKind::WouldBlock {
break;
match decoder.decode(&mut ins) {
Ok(buf) => {
if buf.is_empty() {
break; // eof
}
ins.block_in(content_len); // we only block once
0 as usize
outs.write(&buf).expect("write buffer");
}
Err(e) => match e.kind() {
io::ErrorKind::WouldBlock => {
ins.block_in(content_len); // we only block once
},
_ => panic!("unexpected decode error: {}", e),
}
};
}
@@ -442,22 +452,12 @@ mod tests {
}
// iterate over the different ways that this async read could go.
// tests every combination of buffer size that is passed in, with a blocking
// read at each byte along the content - The shotgun approach
// tests blocking a read at each byte along the content - The shotgun approach
fn all_async_cases(content: &str, expected: &str, decoder: Decoder) {
let content_len = content.len();
for block_at in 0..content_len {
for read_buffer_size in 1..content_len {
let actual = read_async(decoder.clone(),
content.as_bytes(),
block_at,
read_buffer_size);
assert_eq!(expected,
&actual,
"Failed async. Blocking at {} with read buffer size {}",
block_at,
read_buffer_size);
}
let actual = read_async(decoder.clone(), content.as_bytes(), block_at);
assert_eq!(expected, &actual, "Failed async. Blocking at {}", block_at);
}
}
@@ -469,7 +469,7 @@ mod tests {
#[test]
fn test_read_chunked_async() {
let content = "3\r\nfoo\r\n3\r\nbar\r\n0\r\n";
let content = "3\r\nfoo\r\n3\r\nbar\r\n0\r\n\r\n";
let expected = "foobar";
all_async_cases(content, expected, Decoder::chunked());
}