Skip to content

Commit

Permalink
refactor lexer to use idiomatic borrowing
Browse files Browse the repository at this point in the history
  • Loading branch information
@matklad
matklad committed Jun 25, 2019
1 parent 40ab9d2 commit b75021b
Showing 1 changed file with 104 additions and 121 deletions.
225 changes: 104 additions & 121 deletions src/libsyntax/parse/lexer/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,33 +321,29 @@ impl<'a> StringReader<'a> {
(pos - self.source_file.start_pos).to_usize()
}

/// Calls `f` with a string slice of the source text spanning from `start`
/// up to but excluding `self.pos`, meaning the slice does not include
/// the character `self.ch`.
fn with_str_from<T, F>(&self, start: BytePos, f: F) -> T
where F: FnOnce(&str) -> T
/// Slice of the source text from `start` up to but excluding `self.pos`,
/// meaning the slice does not include the character `self.ch`.
fn str_from(&self, start: BytePos) -> &str
{
self.with_str_from_to(start, self.pos, f)
self.str_from_to(start, self.pos)
}

/// Creates a Name from a given offset to the current offset.
fn name_from(&self, start: BytePos) -> ast::Name {
debug!("taking an ident from {:?} to {:?}", start, self.pos);
self.with_str_from(start, Symbol::intern)
Symbol::intern(self.str_from(start))
}

/// As name_from, with an explicit endpoint.
fn name_from_to(&self, start: BytePos, end: BytePos) -> ast::Name {
debug!("taking an ident from {:?} to {:?}", start, end);
self.with_str_from_to(start, end, Symbol::intern)
Symbol::intern(self.str_from_to(start, end))
}

/// Calls `f` with a string slice of the source text spanning from `start`
/// up to but excluding `end`.
fn with_str_from_to<T, F>(&self, start: BytePos, end: BytePos, f: F) -> T
where F: FnOnce(&str) -> T
/// Slice of the source text spanning from `start` up to but excluding `end`.
fn str_from_to(&self, start: BytePos, end: BytePos) -> &str
{
f(&self.src[self.src_index(start)..self.src_index(end)])
&self.src[self.src_index(start)..self.src_index(end)]
}

/// Converts CRLF to LF in the given string, raising an error on bare CR.
Expand Down Expand Up @@ -456,8 +452,8 @@ impl<'a> StringReader<'a> {
self.bump();
}

self.with_str_from(start, |string| {
if string == "_" {
match self.str_from(start) {
"_" => {
self.sess.span_diagnostic
.struct_span_warn(self.mk_sp(start, self.pos),
"underscore literal suffix is not allowed")
Expand All @@ -468,10 +464,9 @@ impl<'a> StringReader<'a> {
<https://github.com/rust-lang/rust/issues/42326>")
.emit();
None
} else {
Some(Symbol::intern(string))
}
})
name => Some(Symbol::intern(name))
}
}

/// PRECONDITION: self.ch is not whitespace
Expand Down Expand Up @@ -513,9 +508,7 @@ impl<'a> StringReader<'a> {
}

let kind = if doc_comment {
self.with_str_from(start_bpos, |string| {
token::DocComment(Symbol::intern(string))
})
token::DocComment(self.name_from(start_bpos))
} else {
token::Comment
};
Expand Down Expand Up @@ -615,23 +608,22 @@ impl<'a> StringReader<'a> {
self.bump();
}

self.with_str_from(start_bpos, |string| {
// but comments with only "*"s between two "/"s are not
let kind = if is_block_doc_comment(string) {
let string = if has_cr {
self.translate_crlf(start_bpos,
string,
"bare CR not allowed in block doc-comment")
} else {
string.into()
};
token::DocComment(Symbol::intern(&string[..]))
let string = self.str_from(start_bpos);
// but comments with only "*"s between two "/"s are not
let kind = if is_block_doc_comment(string) {
let string = if has_cr {
self.translate_crlf(start_bpos,
string,
"bare CR not allowed in block doc-comment")
} else {
token::Comment
string.into()
};
token::DocComment(Symbol::intern(&string[..]))
} else {
token::Comment
};

Some(Token::new(kind, self.mk_sp(start_bpos, self.pos)))
})
Some(Token::new(kind, self.mk_sp(start_bpos, self.pos)))
}

/// Scan through any digits (base `scan_radix`) or underscores,
Expand Down Expand Up @@ -838,20 +830,17 @@ impl<'a> StringReader<'a> {
self.bump();
}

return Ok(self.with_str_from(start, |string| {
// FIXME: perform NFKC normalization here. (Issue #2253)
let name = ast::Name::intern(string);

if is_raw_ident {
let span = self.mk_sp(raw_start, self.pos);
if !name.can_be_raw() {
self.err_span(span, &format!("`{}` cannot be a raw identifier", name));
}
self.sess.raw_identifier_spans.borrow_mut().push(span);
// FIXME: perform NFKC normalization here. (Issue #2253)
let name = self.name_from(start);
if is_raw_ident {
let span = self.mk_sp(raw_start, self.pos);
if !name.can_be_raw() {
self.err_span(span, &format!("`{}` cannot be a raw identifier", name));
}
self.sess.raw_identifier_spans.borrow_mut().push(span);
}

token::Ident(name, is_raw_ident)
}));
return Ok(token::Ident(name, is_raw_ident));
}
}

Expand Down Expand Up @@ -1300,101 +1289,95 @@ impl<'a> StringReader<'a> {
}

fn validate_char_escape(&self, start_with_quote: BytePos) {
self.with_str_from_to(start_with_quote + BytePos(1), self.pos - BytePos(1), |lit| {
if let Err((off, err)) = unescape::unescape_char(lit) {
let lit = self.str_from_to(start_with_quote + BytePos(1), self.pos - BytePos(1));
if let Err((off, err)) = unescape::unescape_char(lit) {
emit_unescape_error(
&self.sess.span_diagnostic,
lit,
self.mk_sp(start_with_quote, self.pos),
unescape::Mode::Char,
0..off,
err,
)
}
}

fn validate_byte_escape(&self, start_with_quote: BytePos) {
let lit = self.str_from_to(start_with_quote + BytePos(1), self.pos - BytePos(1));
if let Err((off, err)) = unescape::unescape_byte(lit) {
emit_unescape_error(
&self.sess.span_diagnostic,
lit,
self.mk_sp(start_with_quote, self.pos),
unescape::Mode::Byte,
0..off,
err,
)
}
}

fn validate_str_escape(&self, start_with_quote: BytePos) {
let lit = self.str_from_to(start_with_quote + BytePos(1), self.pos - BytePos(1));
unescape::unescape_str(lit, &mut |range, c| {
if let Err(err) = c {
emit_unescape_error(
&self.sess.span_diagnostic,
lit,
self.mk_sp(start_with_quote, self.pos),
unescape::Mode::Char,
0..off,
unescape::Mode::Str,
range,
err,
)
}
});
})
}

fn validate_byte_escape(&self, start_with_quote: BytePos) {
self.with_str_from_to(start_with_quote + BytePos(1), self.pos - BytePos(1), |lit| {
if let Err((off, err)) = unescape::unescape_byte(lit) {
fn validate_raw_str_escape(&self, content_start: BytePos, content_end: BytePos) {
let lit = self.str_from_to(content_start, content_end);
unescape::unescape_raw_str(lit, &mut |range, c| {
if let Err(err) = c {
emit_unescape_error(
&self.sess.span_diagnostic,
lit,
self.mk_sp(start_with_quote, self.pos),
unescape::Mode::Byte,
0..off,
self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)),
unescape::Mode::Str,
range,
err,
)
}
});
}

fn validate_str_escape(&self, start_with_quote: BytePos) {
self.with_str_from_to(start_with_quote + BytePos(1), self.pos - BytePos(1), |lit| {
unescape::unescape_str(lit, &mut |range, c| {
if let Err(err) = c {
emit_unescape_error(
&self.sess.span_diagnostic,
lit,
self.mk_sp(start_with_quote, self.pos),
unescape::Mode::Str,
range,
err,
)
}
})
});
}

fn validate_raw_str_escape(&self, content_start: BytePos, content_end: BytePos) {
self.with_str_from_to(content_start, content_end, |lit: &str| {
unescape::unescape_raw_str(lit, &mut |range, c| {
if let Err(err) = c {
emit_unescape_error(
&self.sess.span_diagnostic,
lit,
self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)),
unescape::Mode::Str,
range,
err,
)
}
})
});
})
}

fn validate_raw_byte_str_escape(&self, content_start: BytePos, content_end: BytePos) {
self.with_str_from_to(content_start, content_end, |lit: &str| {
unescape::unescape_raw_byte_str(lit, &mut |range, c| {
if let Err(err) = c {
emit_unescape_error(
&self.sess.span_diagnostic,
lit,
self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)),
unescape::Mode::ByteStr,
range,
err,
)
}
})
});
let lit = self.str_from_to(content_start, content_end);
unescape::unescape_raw_byte_str(lit, &mut |range, c| {
if let Err(err) = c {
emit_unescape_error(
&self.sess.span_diagnostic,
lit,
self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)),
unescape::Mode::ByteStr,
range,
err,
)
}
})
}

fn validate_byte_str_escape(&self, start_with_quote: BytePos) {
self.with_str_from_to(start_with_quote + BytePos(1), self.pos - BytePos(1), |lit| {
unescape::unescape_byte_str(lit, &mut |range, c| {
if let Err(err) = c {
emit_unescape_error(
&self.sess.span_diagnostic,
lit,
self.mk_sp(start_with_quote, self.pos),
unescape::Mode::ByteStr,
range,
err,
)
}
})
});
let lit = self.str_from_to(start_with_quote + BytePos(1), self.pos - BytePos(1));
unescape::unescape_byte_str(lit, &mut |range, c| {
if let Err(err) = c {
emit_unescape_error(
&self.sess.span_diagnostic,
lit,
self.mk_sp(start_with_quote, self.pos),
unescape::Mode::ByteStr,
range,
err,
)
}
})
}
}

Expand Down

0 comments on commit b75021b

Please sign in to comment.