Remove ast::Lit::kind.

This shrinks `ast::Lit` kind considerably. The conversion to semantic
literals now happens at AST lowering time, with a few exceptions where
the literal values are needed before that.

Author: nnethercote

compiler/rustc_ast/src/ast.rs

@@ -1691,10 +1691,6 @@ pub enum StrStyle {
 pub struct Lit {
     /// The original literal token as written in source code.
     pub token_lit: token::Lit,
-    /// The "semantic" representation of the literal lowered from the original tokens.
-    /// Strings are unescaped, hexadecimal forms are eliminated, etc.
-    /// FIXME: Remove this and only create the semantic representation during lowering to HIR.
-    pub kind: LitKind,
     pub span: Span,
 }
 
@@ -1717,11 +1713,7 @@ impl StrLit {
             StrStyle::Cooked => token::Str,
             StrStyle::Raw(n) => token::StrRaw(n),
         };
-        Lit {
-            token_lit: token::Lit::new(token_kind, self.symbol, self.suffix),
-            span: self.span,
-            kind: LitKind::Str(self.symbol_unescaped, self.style),
-        }
+        Lit { token_lit: token::Lit::new(token_kind, self.symbol, self.suffix), span: self.span }
     }
 }
 
@@ -1778,22 +1770,11 @@ impl LitKind {
         matches!(self, LitKind::Str(..))
     }
 
-    /// Returns `true` if this literal is byte literal string.
-    pub fn is_bytestr(&self) -> bool {
-        matches!(self, LitKind::ByteStr(_))
-    }
-
     /// Returns `true` if this is a numeric literal.
     pub fn is_numeric(&self) -> bool {
         matches!(self, LitKind::Int(..) | LitKind::Float(..))
     }
 
-    /// Returns `true` if this literal has no suffix.
-    /// Note: this will return true for literals with prefixes such as raw strings and byte strings.
-    pub fn is_unsuffixed(&self) -> bool {
-        !self.is_suffixed()
-    }
-
     /// Returns `true` if this literal has a suffix.
     pub fn is_suffixed(&self) -> bool {
         match *self {
@@ -3056,7 +3037,7 @@ mod size_asserts {
     static_assert_size!(Impl, 200);
     static_assert_size!(Item, 184);
     static_assert_size!(ItemKind, 112);
-    static_assert_size!(Lit, 48);
+    static_assert_size!(Lit, 20);
     static_assert_size!(LitKind, 24);
     static_assert_size!(Local, 72);
     static_assert_size!(Param, 40);

compiler/rustc_ast/src/attr/mod.rs

@@ -335,7 +335,7 @@ pub fn mk_name_value_item_str(ident: Ident, str: Symbol, str_span: Span) -> Meta
 }
 
 pub fn mk_name_value_item(ident: Ident, lit_kind: LitKind, lit_span: Span) -> MetaItem {
-    let lit = Lit::from_lit_kind(lit_kind, lit_span);
+    let lit = Lit { token_lit: lit_kind.to_token_lit(), span: lit_span };
     let span = ident.span.to(lit_span);
     MetaItem { path: Path::from_ident(ident), span, kind: MetaItemKind::NameValue(lit) }
 }
@@ -519,8 +519,8 @@ impl MetaItem {
 impl MetaItemKind {
     pub fn value_str(&self) -> Option<Symbol> {
         match self {
-            MetaItemKind::NameValue(ref v) => match v.kind {
-                LitKind::Str(ref s, _) => Some(*s),
+            MetaItemKind::NameValue(ref v) => match LitKind::from_token_lit(v.token_lit) {
+                Ok(LitKind::Str(ref s, _)) => Some(*s),
                 _ => None,
             },
             _ => None,
@@ -605,7 +605,7 @@ impl MetaItemKind {
                 MetaItemKind::name_value_from_tokens(&mut inner_tokens.into_trees())
             }
             Some(TokenTree::Token(token, _)) => {
-                Lit::from_token(&token).ok().map(MetaItemKind::NameValue)
+                Lit::from_token(&token).map(MetaItemKind::NameValue)
             }
             _ => None,
         }
@@ -667,9 +667,7 @@ impl NestedMetaItem {
         I: Iterator<Item = TokenTree>,
     {
         match tokens.peek() {
-            Some(TokenTree::Token(token, _))
-                if let Ok(lit) = Lit::from_token(token) =>
-            {
+            Some(TokenTree::Token(token, _)) if let Some(lit) = Lit::from_token(token) => {
                 tokens.next();
                 return Some(NestedMetaItem::Literal(lit));
             }

compiler/rustc_ast/src/token.rs

@@ -80,6 +80,28 @@ pub struct Lit {
     pub suffix: Option<Symbol>,
 }
 
+impl Lit {
+    pub fn is_bool_true(&self) -> bool {
+        matches!(self.kind, LitKind::Bool) && self.symbol == kw::True
+    }
+
+    /// Returns `true` if this literal is a string.
+    pub fn is_str(&self) -> bool {
+        matches!(self.kind, LitKind::Str | LitKind::StrRaw(_))
+    }
+
+    /// Returns `true` if this literal is byte literal string.
+    pub fn is_bytestr(&self) -> bool {
+        matches!(self.kind, LitKind::ByteStr | LitKind::ByteStrRaw(_))
+    }
+
+    /// Returns `true` if this literal has no suffix.
+    /// Note: this will return true for literals with prefixes such as raw strings and byte strings.
+    pub fn is_unsuffixed(&self) -> bool {
+        self.suffix.is_none()
+    }
+}
+
 impl fmt::Display for Lit {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let Lit { kind, symbol, suffix } = *self;

compiler/rustc_ast/src/util/literal.rs

@@ -6,12 +6,10 @@ use crate::token::{self, Token};
 use rustc_lexer::unescape::{unescape_byte, unescape_char};
 use rustc_lexer::unescape::{unescape_byte_literal, unescape_literal, Mode};
 use rustc_span::symbol::{kw, sym, Symbol};
-use rustc_span::Span;
 
 use std::ascii;
 
 pub enum LitError {
-    NotLiteral,
     LexerError,
     InvalidSuffix,
     InvalidIntSuffix,
@@ -149,7 +147,7 @@ impl LitKind {
         })
     }
 
-    /// Attempts to recover a token from semantic literal.
+    /// Creates a token literal from a semantic literal kind.
     /// This function is used when the original token doesn't exist (e.g. the literal is created
     /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
     pub fn to_token_lit(&self) -> token::Lit {
@@ -203,39 +201,30 @@ impl LitKind {
 }
 
 impl Lit {
-    /// Converts literal token into an AST literal.
-    pub fn from_token_lit(token_lit: token::Lit, span: Span) -> Result<Lit, LitError> {
-        Ok(Lit { token_lit, kind: LitKind::from_token_lit(token_lit)?, span })
-    }
-
-    /// Converts arbitrary token into an AST literal.
+    /// Converts an arbitrary token into an AST literal. Returns `None` if the
+    /// token is not a literal. The literal may be invalid in some fashion
+    /// (e.g. invalid suffix, too-large int, etc.); this will be caught during
+    /// lowering to HIR.
     ///
     /// Keep this in sync with `Token::can_begin_literal_or_bool` excluding unary negation.
-    pub fn from_token(token: &Token) -> Result<Lit, LitError> {
-        let lit = match token.uninterpolate().kind {
+    pub fn from_token(token: &Token) -> Option<Lit> {
+        let token_lit = match token.uninterpolate().kind {
             token::Ident(name, false) if name.is_bool_lit() => {
                 token::Lit::new(token::Bool, name, None)
             }
-            token::Literal(lit) => lit,
+            token::Literal(token_lit) => token_lit,
             token::Interpolated(ref nt) => {
                 if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt
                     && let ast::ExprKind::Lit(lit) = &expr.kind
                 {
-                    return Ok(lit.clone());
+                    return Some(lit.clone());
                 }
-                return Err(LitError::NotLiteral);
+                return None;
             }
-            _ => return Err(LitError::NotLiteral),
+            _ => return None,
         };
 
-        Lit::from_token_lit(lit, token.span)
-    }
-
-    /// Attempts to recover an AST literal from semantic literal.
-    /// This function is used when the original token doesn't exist (e.g. the literal is created
-    /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
-    pub fn from_lit_kind(kind: LitKind, span: Span) -> Lit {
-        Lit { token_lit: kind.to_token_lit(), kind, span }
+        Some(Lit { token_lit, span: token.span })
     }
 
     /// Losslessly convert an AST literal into a token.

compiler/rustc_ast_lowering/src/errors.rs

@@ -2,6 +2,61 @@ use rustc_errors::{fluent, AddSubdiagnostic, Applicability, Diagnostic, Diagnost
 use rustc_macros::{SessionDiagnostic, SessionSubdiagnostic};
 use rustc_span::{symbol::Ident, Span, Symbol};
 
+#[derive(SessionDiagnostic)]
+#[diag(parser::invalid_int_literal_width)]
+#[help]
+pub(crate) struct InvalidIntLiteralWidth {
+    #[primary_span]
+    pub span: Span,
+    pub width: String,
+}
+
+#[derive(SessionDiagnostic)]
+#[diag(parser::invalid_num_literal_base_prefix)]
+#[note]
+pub(crate) struct InvalidNumLiteralBasePrefix {
+    #[primary_span]
+    #[suggestion(applicability = "maybe-incorrect", code = "{fixed}")]
+    pub span: Span,
+    pub fixed: String,
+}
+
+#[derive(SessionDiagnostic)]
+#[diag(parser::invalid_num_literal_suffix)]
+#[help]
+pub(crate) struct InvalidNumLiteralSuffix {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    pub suffix: String,
+}
+
+#[derive(SessionDiagnostic)]
+#[diag(parser::invalid_float_literal_width)]
+#[help]
+pub(crate) struct InvalidFloatLiteralWidth {
+    #[primary_span]
+    pub span: Span,
+    pub width: String,
+}
+
+#[derive(SessionDiagnostic)]
+#[diag(parser::invalid_float_literal_suffix)]
+#[help]
+pub(crate) struct InvalidFloatLiteralSuffix {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    pub suffix: String,
+}
+
+#[derive(SessionDiagnostic)]
+#[diag(parser::int_literal_too_large)]
+pub(crate) struct IntLiteralTooLarge {
+    #[primary_span]
+    pub span: Span,
+}
+
 #[derive(SessionDiagnostic, Clone, Copy)]
 #[diag(ast_lowering::generic_type_with_parentheses, code = "E0214")]
 pub struct GenericTypeWithParentheses {

compiler/rustc_ast_lowering/src/expr.rs

@@ -1,14 +1,17 @@
 use super::errors::{
     AsyncGeneratorsNotSupported, AsyncNonMoveClosureNotSupported, AwaitOnlyInAsyncFnAndBlocks,
     BaseExpressionDoubleDot, ClosureCannotBeStatic, FunctionalRecordUpdateDestructuringAssignemnt,
-    GeneratorTooManyParameters, InclusiveRangeWithNoEnd, NotSupportedForLifetimeBinderAsyncClosure,
-    RustcBoxAttributeError, UnderscoreExprLhsAssign,
+    GeneratorTooManyParameters, InclusiveRangeWithNoEnd, IntLiteralTooLarge,
+    InvalidFloatLiteralSuffix, InvalidFloatLiteralWidth, InvalidIntLiteralWidth,
+    InvalidNumLiteralBasePrefix, InvalidNumLiteralSuffix,
+    NotSupportedForLifetimeBinderAsyncClosure, RustcBoxAttributeError, UnderscoreExprLhsAssign,
 };
 use super::ResolverAstLoweringExt;
 use super::{ImplTraitContext, LoweringContext, ParamMode, ParenthesizedGenericArgs};
 use crate::{FnDeclKind, ImplTraitPosition};
 use rustc_ast::attr;
 use rustc_ast::ptr::P as AstP;
+use rustc_ast::util::literal::LitError;
 use rustc_ast::*;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir as hir;
@@ -84,8 +87,17 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     let ohs = self.lower_expr(ohs);
                     hir::ExprKind::Unary(op, ohs)
                 }
-                ExprKind::Lit(ref l) => {
-                    hir::ExprKind::Lit(respan(self.lower_span(l.span), l.kind.clone()))
+                ExprKind::Lit(ref lit) => {
+                    let token_lit = lit.token_lit;
+                    let lit_kind = match LitKind::from_token_lit(token_lit) {
+                        Ok(lit_kind) => lit_kind,
+                        Err(err) => {
+                            let span = lit.span;
+                            self.report_lit_error(err, token_lit, span);
+                            LitKind::Err
+                        }
+                    };
+                    hir::ExprKind::Lit(respan(self.lower_span(lit.span), lit_kind))
                 }
                 ExprKind::Cast(ref expr, ref ty) => {
                     let expr = self.lower_expr(expr);
@@ -306,6 +318,86 @@ impl<'hir> LoweringContext<'_, 'hir> {
         })
     }
 
+    #[allow(rustc::diagnostic_outside_of_impl)]
+    #[allow(rustc::untranslatable_diagnostic)]
+    fn report_lit_error(&self, err: LitError, lit: token::Lit, span: Span) {
+        // Checks if `s` looks like i32 or u1234 etc.
+        fn looks_like_width_suffix(first_chars: &[char], s: &str) -> bool {
+            s.len() > 1 && s.starts_with(first_chars) && s[1..].chars().all(|c| c.is_ascii_digit())
+        }
+
+        // Try to lowercase the prefix if it's a valid base prefix.
+        fn fix_base_capitalisation(s: &str) -> Option<String> {
+            if let Some(stripped) = s.strip_prefix('B') {
+                Some(format!("0b{stripped}"))
+            } else if let Some(stripped) = s.strip_prefix('O') {
+                Some(format!("0o{stripped}"))
+            } else if let Some(stripped) = s.strip_prefix('X') {
+                Some(format!("0x{stripped}"))
+            } else {
+                None
+            }
+        }
+
+        let token::Lit { kind, suffix, .. } = lit;
+        match err {
+            // `LexerError` *is* an error, but it was already reported
+            // by lexer, so here we don't report it the second time.
+            LitError::LexerError => {}
+            LitError::InvalidSuffix => {
+                self.tcx.sess.parse_sess.expect_no_suffix(
+                    span,
+                    &format!("{} {} literal", kind.article(), kind.descr()),
+                    suffix,
+                );
+            }
+            LitError::InvalidIntSuffix => {
+                let suf = suffix.expect("suffix error with no suffix");
+                let suf = suf.as_str();
+                if looks_like_width_suffix(&['i', 'u'], &suf) {
+                    // If it looks like a width, try to be helpful.
+                    self.tcx.sess.emit_err(InvalidIntLiteralWidth { span, width: suf[1..].into() });
+                } else if let Some(fixed) = fix_base_capitalisation(suf) {
+                    self.tcx.sess.emit_err(InvalidNumLiteralBasePrefix { span, fixed });
+                } else {
+                    self.tcx
+                        .sess
+                        .emit_err(InvalidNumLiteralSuffix { span, suffix: suf.to_string() });
+                }
+            }
+            LitError::InvalidFloatSuffix => {
+                let suf = suffix.expect("suffix error with no suffix");
+                let suf = suf.as_str();
+                if looks_like_width_suffix(&['f'], suf) {
+                    // If it looks like a width, try to be helpful.
+                    self.tcx
+                        .sess
+                        .emit_err(InvalidFloatLiteralWidth { span, width: suf[1..].to_string() });
+                } else {
+                    self.tcx
+                        .sess
+                        .emit_err(InvalidFloatLiteralSuffix { span, suffix: suf.to_string() });
+                }
+            }
+            LitError::NonDecimalFloat(base) => {
+                let descr = match base {
+                    16 => "hexadecimal",
+                    8 => "octal",
+                    2 => "binary",
+                    _ => unreachable!(),
+                };
+                self.tcx
+                    .sess
+                    .struct_span_err(span, &format!("{descr} float literal is not supported"))
+                    .span_label(span, "not supported")
+                    .emit();
+            }
+            LitError::IntTooLarge => {
+                self.tcx.sess.emit_err(IntLiteralTooLarge { span });
+            }
+        }
+    }
+
     fn lower_unop(&mut self, u: UnOp) -> hir::UnOp {
         match u {
             UnOp::Deref => hir::UnOp::Deref,

compiler/rustc_ast_lowering/src/lib.rs

@@ -952,7 +952,6 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 } else {
                     Lit {
                         token_lit: token::Lit::new(token::LitKind::Err, kw::Empty, None),
-                        kind: LitKind::Err,
                         span: DUMMY_SP,
                     }
                 };