Skip to content

Commit

Permalink
feat(doit): optimizations and simplification; seek
Browse files Browse the repository at this point in the history 
* MultiPartReader is using match to handle state, reducing unnecessary
  calls to 0 in that regard.
* Fixed seek() calls on readers, assuring they are reset to start each
  time the loop is done.
* both media-parameters now use `ReadSeek` streams.
* Use `seek()` to figure out size, simplifying the interface.

Closes #17
  • Loading branch information
@Byron
Byron committed Mar 19, 2015
1 parent 6b23013 commit 9d401f5
Show file tree
Hide file tree
Showing 4 changed files with 47 additions and 41 deletions.
45 changes: 24 additions & 21 deletions src/mako/lib/mbuild.mako
View file
Original file line number Diff line number Diff line change
Expand Up @@ -375,17 +375,18 @@ match result {
rtype = 'cmn::Result<(hyper::client::Response, %s)>' % (response_schema.id)
reserved_params = ['alt']
mtype_param = 'RS'
mtype_where = 'ReadSeek'
possible_urls = [m.path]
simple_media_param = None
if media_params:
stripped = lambda s: s.strip().strip(',')
qualifier = ''
type_params = mtype_param
where = mtype_param + ': ' + mtype_where
for p in media_params:
type_params += p.type.param + ', '
where += p.type.param + ': ' + p.type.where + ', '
add_args += 'mut ' + p.type.arg_name + ': ' + ('Option<(%s, u64, mime::Mime)>' % p.type.param) + ', '
add_args += 'mut ' + p.type.arg_name + ': ' + ('Option<(%s, mime::Mime)>' % mtype_param) + ', '
possible_urls.append(p.path)
if p.protocol == 'simple':
simple_media_param = p
Expand Down Expand Up @@ -422,6 +423,7 @@ match result {
all_required_param_name = set(p.name for p in params if is_required_property(p))
MULTI_SLASH = 'multi-slash-prefix'
URL_ENCODE = 'url-encode'
READER_SEEK = "let size = reader.seek(io::SeekFrom::End(0)).unwrap();\nreader.seek(io::SeekFrom::Start(0)).unwrap();"
special_cases = set()
for possible_url in possible_urls:
Expand Down Expand Up @@ -595,8 +597,10 @@ else {
}
% if request_value:
let json_encoded_request = json::encode(&self.${property(REQUEST_VALUE_PROPERTY_NAME)}).unwrap();
let mut json_mime_type = mime::Mime(mime::TopLevel::Application, mime::SubLevel::Json, Default::default());
let mut request_value_reader = io::Cursor::new(json::encode(&self.${property(REQUEST_VALUE_PROPERTY_NAME)}).unwrap().into_bytes());
let request_size = request_value_reader.seek(io::SeekFrom::End(0)).unwrap();
request_value_reader.seek(io::SeekFrom::Start(0)).unwrap();
% endif
let mut client = &mut *self.hub.client.borrow_mut();
Expand All @@ -611,15 +615,16 @@ else {
}
let auth_header = hyper::header::Authorization(token.unwrap().access_token);
% endif
% if request_value:
request_value_reader.seek(io::SeekFrom::Start(0)).unwrap();
% endif
% if request_value and simple_media_param:
let mut request_value_reader = io::Cursor::new(json_encoded_request.clone().into_bytes());
let mut mp_reader: cmn::MultiPartReader = Default::default();
let (mut body_reader, content_type) = match ${simple_media_param.type.arg_name}.as_mut() {
Some(&mut (ref mut reader, size, ref mime)) => {
Some(&mut (ref mut reader, ref mime)) => {
mp_reader.reserve_exact(2);
let rsize = request_value_reader.seek(io::SeekFrom::End(0)).unwrap();
request_value_reader.seek(io::SeekFrom::Start(0)).unwrap();
mp_reader.add_part(&mut request_value_reader, rsize, json_mime_type.clone())
${READER_SEEK | indent_all_but_first_by(5)}
mp_reader.add_part(&mut request_value_reader, request_size, json_mime_type.clone())
.add_part(reader, size, mime.clone());
let mime_type = mp_reader.mime_type();
(&mut mp_reader as &mut io::Read, ContentType(mime_type))
Expand All @@ -638,8 +643,8 @@ else {
% if not simple_media_param:
.header(ContentType(json_mime_type.clone()))
.header(ContentLength(json_encoded_request.len() as u64))
.body(json_encoded_request.as_slice())\
.header(ContentLength(request_size as u64))
.body(request_value_reader.into_body())\
% else:
.header(content_type)
Expand All @@ -648,7 +653,8 @@ else {
% endif
;
% if simple_media_param and not request_value:
if let Some(&mut (ref mut reader, size, ref mime)) = ${simple_media_param.type.arg_name}.as_mut() {
if let Some(&mut (ref mut reader, ref mime)) = ${simple_media_param.type.arg_name}.as_mut() {
${READER_SEEK | indent_all_but_first_by(4)}
req = req.header(ContentType(mime.clone()))
.header(ContentLength(size))
.body(reader.into_body());
Expand Down Expand Up @@ -697,23 +703,20 @@ else {
}
% for p in media_params:
<%
none_type = 'None::<(' + p.type.default + ', u64, mime::Mime)>'
%>\
${p.description | rust_doc_comment, indent_all_but_first_by(1)}
///
% for item_name, item in p.info.iteritems():
/// * *${split_camelcase_s(item_name)}*: ${isinstance(item, (list, tuple)) and put_and(enclose_in("'", item)) or str(item)}
% endfor
pub fn ${api.terms.upload_action}${p.type.suffix}<${p.type.param}>(self, ${p.type.arg_name}: ${p.type.param}, size: u64, mime_type: mime::Mime) -> ${rtype}
where ${p.type.param}: ${p.type.where} {
pub fn ${api.terms.upload_action}${p.type.suffix}<${mtype_param}>(self, ${p.type.arg_name}: ${mtype_param}, mime_type: mime::Mime) -> ${rtype}
where ${mtype_param}: ${mtype_where} {
self.${api.terms.action}(\
% for _ in range(0, loop.index):
${none_type}, \
None, \
% endfor
Some((${p.type.arg_name}, size, mime_type)), \
Some((${p.type.arg_name}, mime_type)), \
% for _ in range(loop.index+1, len(media_params)):
${none_type}, \
None, \
% endfor
)
}
Expand Down
8 changes: 2 additions & 6 deletions src/mako/lib/util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,26 +71,22 @@
PROTOCOL_TYPE_INFO = {
'simple' : {
'arg_name': 'stream',
'param': 'R',
'description': """Upload media all at once.
If the upload fails for whichever reason, all progress is lost.""",
'default': 'fs::File',
'where': 'ReadSeek',
'suffix': '',
'example_value': 'fs::File::open("file.ext").unwrap(), 148, "application/octet-stream".parse().unwrap()'
'example_value': 'fs::File::open("file.ext").unwrap(), "application/octet-stream".parse().unwrap()'
},
'resumable' : {
'arg_name': 'resumeable_stream',
'param': 'RS',
'description': """Upload media in a resumeable fashion.
Even if the upload fails or is interrupted, it can be resumed for a
certain amount of time as the server maintains state temporarily.
TODO: Write more about how delegation works in this particular case.""",
'default': 'fs::File',
'where': 'ReadSeek',
'suffix': '_resumable',
'example_value': 'fs::File::open("file.ext").unwrap(), 282, "application/octet-stream".parse().unwrap()'
'example_value': 'fs::File::open("file.ext").unwrap(), "application/octet-stream".parse().unwrap()'
}
}

Expand Down
34 changes: 20 additions & 14 deletions src/rust/cmn.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,22 +183,28 @@ impl<'a> MultiPartReader<'a> {

impl<'a> Read for MultiPartReader<'a> {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
if self.last_part_boundary.is_some() {
let br = self.last_part_boundary.as_mut().unwrap().read(buf).unwrap_or(0);
if br < buf.len() {
self.last_part_boundary = None;
match (self.raw_parts.len(),
self.current_part.is_none(),
self.last_part_boundary.is_none()) {
(_, _, false) => {
let br = self.last_part_boundary.as_mut().unwrap().read(buf).unwrap_or(0);
if br < buf.len() {
self.last_part_boundary = None;
}
return Ok(br)
},
(0, true, true) => return Ok(0),
(n, true, _) if n > 0 => {
let (headers, reader) = self.raw_parts.remove(0);
let mut c = Cursor::new(Vec::<u8>::new());
write!(&mut c, "{}--{}{}{}{}", LINE_ENDING, BOUNDARY, LINE_ENDING,
headers, LINE_ENDING).unwrap();
c.seek(SeekFrom::Start(0)).unwrap();
self.current_part = Some((c, reader));
}
return Ok(br)
} else if self.is_depleted() {
return Ok(0)
} else if self.raw_parts.len() > 0 && self.current_part.is_none() {
let (headers, reader) = self.raw_parts.remove(0);
let mut c = Cursor::new(Vec::<u8>::new());
write!(&mut c, "{}--{}{}{}{}", LINE_ENDING, BOUNDARY, LINE_ENDING,
headers, LINE_ENDING).unwrap();
c.seek(SeekFrom::Start(0)).unwrap();
self.current_part = Some((c, reader));
_ => {},
}

// read headers as long as possible
let (hb, rr) = {
let &mut (ref mut c, ref mut reader) = self.current_part.as_mut().unwrap();
Expand Down
1 change: 1 addition & 0 deletions src/rust/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ mod tests {
use self::hyper_mock::*;
use std::io::Read;
use std::default::Default;
use std::old_path::BytesContainer;

const EXPECTED: &'static str =
"\r\n--MDuXWGyeE33QFXGchb2VFWc4Z7945d\r\n\
Expand Down

0 comments on commit 9d401f5

Please sign in to comment.