Lexer changes

src/libcore/str.rs

@@ -1764,7 +1764,9 @@ impl<'a> StrSlice<'a> for &'a str {
 
     #[inline]
     fn slice(&self, begin: uint, end: uint) -> &'a str {
-        assert!(self.is_char_boundary(begin) && self.is_char_boundary(end));
+        assert!(self.is_char_boundary(begin) && self.is_char_boundary(end),
+                "index {} and/or {} in `{}` do not lie on character boundary", begin,
+                end, *self);
         unsafe { raw::slice_bytes(*self, begin, end) }
     }
 
@@ -1775,7 +1777,8 @@ impl<'a> StrSlice<'a> for &'a str {
 
     #[inline]
     fn slice_to(&self, end: uint) -> &'a str {
-        assert!(self.is_char_boundary(end));
+        assert!(self.is_char_boundary(end), "index {} in `{}` does not lie on \
+                a character boundary", end, *self);
         unsafe { raw::slice_bytes(*self, 0, end) }
     }
 

src/librustc/lint/builtin.rs

@@ -1114,7 +1114,7 @@ impl UnusedMut {
                 match mode {
                     ast::BindByValue(ast::MutMutable) => {
                         if !token::get_ident(ident).get().starts_with("_") {
-                            mutables.insert_or_update_with(ident.name as uint,
+                            mutables.insert_or_update_with(ident.name.uint(),
                                 vec!(id), |_, old| { old.push(id); });
                         }
                     }

src/librustc/metadata/decoder.rs

@@ -323,7 +323,7 @@ fn item_name(intr: &IdentInterner, item: ebml::Doc) -> ast::Ident {
     let string = name.as_str_slice();
     match intr.find_equiv(&string) {
         None => token::str_to_ident(string),
-        Some(val) => ast::Ident::new(val as ast::Name),
+        Some(val) => ast::Ident::new(val),
     }
 }
 

src/librustc/middle/astencode.rs

@@ -1523,14 +1523,15 @@ fn test_basic() {
         fn foo() {}
     ));
 }
-
+/* NOTE: When there's a snapshot, update this (yay quasiquoter!)
 #[test]
 fn test_smalltalk() {
     let cx = mk_ctxt();
     roundtrip(quote_item!(cx,
         fn foo() -> int { 3 + 4 } // first smalltalk program ever executed.
     ));
 }
+*/
 
 #[test]
 fn test_more() {

src/librustc/middle/trans/consts.rs

@@ -42,6 +42,7 @@ use syntax::{ast, ast_util};
 pub fn const_lit(cx: &CrateContext, e: &ast::Expr, lit: ast::Lit)
     -> ValueRef {
     let _icx = push_ctxt("trans_lit");
+    debug!("const_lit: {}", lit);
     match lit.node {
         ast::LitByte(b) => C_integral(Type::uint_from_ty(cx, ast::TyU8), b as u64, false),
         ast::LitChar(i) => C_integral(Type::char(cx), i as u64, false),

src/librustdoc/html/highlight.rs

@@ -18,7 +18,6 @@ use std::io;
 
 use syntax::parse;
 use syntax::parse::lexer;
-use syntax::codemap::{BytePos, Span};
 
 use html::escape::Escape;
 
@@ -59,38 +58,30 @@ fn doit(sess: &parse::ParseSess, mut lexer: lexer::StringReader,
         None => {}
     }
     try!(write!(out, "class='rust {}'>\n", class.unwrap_or("")));
-    let mut last = BytePos(0);
     let mut is_attribute = false;
     let mut is_macro = false;
     let mut is_macro_nonterminal = false;
     loop {
         let next = lexer.next_token();
-        let test = if next.tok == t::EOF {lexer.pos} else {next.sp.lo};
-
-        // The lexer consumes all whitespace and non-doc-comments when iterating
-        // between tokens. If this token isn't directly adjacent to our last
-        // token, then we need to emit the whitespace/comment.
-        //
-        // If the gap has any '/' characters then we consider the whole thing a
-        // comment. This will classify some whitespace as a comment, but that
-        // doesn't matter too much for syntax highlighting purposes.
-        if test > last {
-            let snip = sess.span_diagnostic.cm.span_to_snippet(Span {
-                lo: last,
-                hi: test,
-                expn_info: None,
-            }).unwrap();
-            if snip.as_slice().contains("/") {
-                try!(write!(out, "<span class='comment'>{}</span>",
-                              Escape(snip.as_slice())));
-            } else {
-                try!(write!(out, "{}", Escape(snip.as_slice())));
-            }
-        }
-        last = next.sp.hi;
+
+        let snip = |sp| sess.span_diagnostic.cm.span_to_snippet(sp).unwrap();
+
         if next.tok == t::EOF { break }
 
         let klass = match next.tok {
+            t::WS => {
+                try!(write!(out, "{}", Escape(snip(next.sp).as_slice())));
+                continue
+            },
+            t::COMMENT => {
+                try!(write!(out, "<span class='comment'>{}</span>",
+                            Escape(snip(next.sp).as_slice())));
+                continue
+            },
+            t::SHEBANG(s) => {
+                try!(write!(out, "{}", Escape(s.as_str())));
+                continue
+            },
             // If this '&' token is directly adjacent to another token, assume
             // that it's the address-of operator instead of the and-operator.
             // This allows us to give all pointers their own class (`Box` and
@@ -144,8 +135,7 @@ fn doit(sess: &parse::ParseSess, mut lexer: lexer::StringReader,
                 t::LIT_CHAR(..) | t::LIT_STR(..) | t::LIT_STR_RAW(..) => "string",
 
             // number literals
-            t::LIT_INT(..) | t::LIT_UINT(..) | t::LIT_INT_UNSUFFIXED(..) |
-                t::LIT_FLOAT(..) | t::LIT_FLOAT_UNSUFFIXED(..) => "number",
+            t::LIT_INTEGER(..) | t::LIT_FLOAT(..) => "number",
 
             // keywords are also included in the identifier set
             t::IDENT(ident, _is_mod_sep) => {

src/libsyntax/abi.rs

@@ -60,9 +60,12 @@ pub struct AbiData {
 }
 
 pub enum AbiArchitecture {
-    RustArch,   // Not a real ABI (e.g., intrinsic)
-    AllArch,    // An ABI that specifies cross-platform defaults (e.g., "C")
-    Archs(u32)  // Multiple architectures (bitset)
+    /// Not a real ABI (e.g., intrinsic)
+    RustArch,
+    /// An ABI that specifies cross-platform defaults (e.g., "C")
+    AllArch,
+    /// Multiple architectures (bitset)
+    Archs(u32)
 }
 
 static AbiDatas: &'static [AbiData] = &[
@@ -84,21 +87,13 @@ static AbiDatas: &'static [AbiData] = &[
     AbiData {abi: RustIntrinsic, name: "rust-intrinsic", abi_arch: RustArch},
 ];
 
+/// Iterates through each of the defined ABIs.
 fn each_abi(op: |abi: Abi| -> bool) -> bool {
-    /*!
-     *
-     * Iterates through each of the defined ABIs.
-     */
-
     AbiDatas.iter().advance(|abi_data| op(abi_data.abi))
 }
 
+/// Returns the ABI with the given name (if any).
 pub fn lookup(name: &str) -> Option<Abi> {
-    /*!
-     *
-     * Returns the ABI with the given name (if any).
-     */
-
     let mut res = None;
 
     each_abi(|abi| {