syntax: don't parse numeric literals in the lexer

This removes a bunch of token types. Tokens now store the original, unaltered
numeric literal (that is still checked for correctness), which is parsed into
an actual number later, as needed, when creating the AST.

This can change how syntax extensions work, but otherwise poses no visible
changes.

[breaking-change]

Author: emberian

src/librustc/middle/trans/consts.rs

@@ -42,6 +42,7 @@ use syntax::{ast, ast_util};
 pub fn const_lit(cx: &CrateContext, e: &ast::Expr, lit: ast::Lit)
     -> ValueRef {
     let _icx = push_ctxt("trans_lit");
+    debug!("const_lit: {}", lit);
     match lit.node {
         ast::LitByte(b) => C_integral(Type::uint_from_ty(cx, ast::TyU8), b as u64, false),
         ast::LitChar(i) => C_integral(Type::char(cx), i as u64, false),

src/librustdoc/html/highlight.rs

@@ -144,8 +144,7 @@ fn doit(sess: &parse::ParseSess, mut lexer: lexer::StringReader,
                 t::LIT_CHAR(..) | t::LIT_STR(..) | t::LIT_STR_RAW(..) => "string",
 
             // number literals
-            t::LIT_INT(..) | t::LIT_UINT(..) | t::LIT_INT_UNSUFFIXED(..) |
-                t::LIT_FLOAT(..) | t::LIT_FLOAT_UNSUFFIXED(..) => "number",
+            t::LIT_INTEGER(..) | t::LIT_FLOAT(..) => "number",
 
             // keywords are also included in the identifier set
             t::IDENT(ident, _is_mod_sep) => {

src/libsyntax/ast.rs

@@ -619,15 +619,15 @@ pub enum Mac_ {
     MacInvocTT(Path, Vec<TokenTree> , SyntaxContext),   // new macro-invocation
 }
 
-#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash)]
+#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)]
 pub enum StrStyle {
     CookedStr,
     RawStr(uint)
 }
 
 pub type Lit = Spanned<Lit_>;
 
-#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash)]
+#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)]
 pub enum Lit_ {
     LitStr(InternedString, StrStyle),
     LitBinary(Rc<Vec<u8> >),
@@ -697,6 +697,16 @@ impl fmt::Show for IntTy {
     }
 }
 
+impl IntTy {
+    pub fn suffix_len(&self) -> uint {
+        match *self {
+            TyI => 1,
+            TyI8 => 2,
+            TyI16 | TyI32 | TyI64  => 3,
+        }
+    }
+}
+
 #[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash)]
 pub enum UintTy {
     TyU,
@@ -706,6 +716,16 @@ pub enum UintTy {
     TyU64,
 }
 
+impl UintTy {
+    pub fn suffix_len(&self) -> uint {
+        match *self {
+            TyU => 1,
+            TyU8 => 2,
+            TyU16 | TyU32 | TyU64  => 3,
+        }
+    }
+}
+
 impl fmt::Show for UintTy {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "{}", ast_util::uint_ty_to_string(*self, None))
@@ -724,6 +744,14 @@ impl fmt::Show for FloatTy {
     }
 }
 
+impl FloatTy {
+    pub fn suffix_len(&self) -> uint {
+        match *self {
+            TyF32 | TyF64 => 3, // add F128 handling here
+        }
+    }
+}
+
 // NB PartialEq method appears below.
 #[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash)]
 pub struct Ty {

src/libsyntax/codemap.rs

@@ -96,7 +96,7 @@ pub struct Span {
 
 pub static DUMMY_SP: Span = Span { lo: BytePos(0), hi: BytePos(0), expn_info: None };
 
-#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash)]
+#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)]
 pub struct Spanned<T> {
     pub node: T,
     pub span: Span,

src/libsyntax/ext/quote.rs

@@ -412,45 +412,14 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> Gc<ast::Expr> {
             return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_CHAR"), vec!(e_char));
         }
 
-        LIT_INT(i, ity) => {
-            let s_ity = match ity {
-                ast::TyI => "TyI",
-                ast::TyI8 => "TyI8",
-                ast::TyI16 => "TyI16",
-                ast::TyI32 => "TyI32",
-                ast::TyI64 => "TyI64"
-            };
-            let e_ity = mk_ast_path(cx, sp, s_ity);
-            let e_i64 = cx.expr_lit(sp, ast::LitInt(i, ast::TyI64));
-            return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_INT"), vec!(e_i64, e_ity));
+        LIT_INTEGER(i) => {
+            let e_int = mk_ident(cx, sp, i);
+            return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_INTEGER"), vec!(e_int));
         }
 
-        LIT_UINT(u, uty) => {
-            let s_uty = match uty {
-                ast::TyU => "TyU",
-                ast::TyU8 => "TyU8",
-                ast::TyU16 => "TyU16",
-                ast::TyU32 => "TyU32",
-                ast::TyU64 => "TyU64"
-            };
-            let e_uty = mk_ast_path(cx, sp, s_uty);
-            let e_u64 = cx.expr_lit(sp, ast::LitUint(u, ast::TyU64));
-            return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_UINT"), vec!(e_u64, e_uty));
-        }
-
-        LIT_INT_UNSUFFIXED(i) => {
-            let e_i64 = cx.expr_lit(sp, ast::LitInt(i, ast::TyI64));
-            return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_INT_UNSUFFIXED"), vec!(e_i64));
-        }
-
-        LIT_FLOAT(fident, fty) => {
-            let s_fty = match fty {
-                ast::TyF32 => "TyF32",
-                ast::TyF64 => "TyF64",
-            };
-            let e_fty = mk_ast_path(cx, sp, s_fty);
+        LIT_FLOAT(fident) => {
             let e_fident = mk_ident(cx, sp, fident);
-            return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_FLOAT"), vec!(e_fident, e_fty));
+            return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_FLOAT"), vec!(e_fident));
         }
 
         LIT_STR(ident) => {