reduce the amount of panics in {TokenStream, Literal}::from_str calls

Author: cyrgani

compiler/rustc_errors/src/diagnostic.rs

@@ -1424,6 +1424,13 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
         drop(self);
     }
 
+    /// Cancels this diagnostic and returns its first message, if it exists.
+    pub fn cancel_into_message(self) -> Option<String> {
+        let s = self.diag.as_ref()?.messages.get(0)?.0.as_str().map(ToString::to_string);
+        self.cancel();
+        s
+    }
+
     /// See `DiagCtxt::stash_diagnostic` for details.
     pub fn stash(mut self, span: Span, key: StashKey) -> Option<ErrorGuaranteed> {
         let diag = self.take_diag();

compiler/rustc_expand/src/proc_macro_server.rs

@@ -10,7 +10,7 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{Diag, ErrorGuaranteed, MultiSpan, PResult};
 use rustc_parse::lexer::{StripTokens, nfc_normalize};
 use rustc_parse::parser::Parser;
-use rustc_parse::{exp, new_parser_from_source_str, source_str_to_stream, unwrap_or_emit_fatal};
+use rustc_parse::{exp, new_parser_from_source_str, source_str_to_stream};
 use rustc_proc_macro::bridge::{
     DelimSpan, Diagnostic, ExpnGlobals, Group, Ident, LitKind, Literal, Punct, TokenTree, server,
 };
@@ -431,6 +431,13 @@ impl ToInternal<rustc_errors::Level> for Level {
     }
 }
 
+fn cancel_diags_into_string(diags: Vec<Diag<'_>>) -> String {
+    let mut messages = diags.into_iter().map(Diag::cancel_into_message).flatten();
+    let msg = messages.next().expect("no diagnostic has a message");
+    messages.for_each(drop);
+    msg
+}
+
 pub(crate) struct FreeFunctions;
 
 pub(crate) struct Rustc<'a, 'b> {
@@ -483,35 +490,32 @@ impl server::FreeFunctions for Rustc<'_, '_> {
         self.psess().file_depinfo.borrow_mut().insert(Symbol::intern(path));
     }
 
-    fn literal_from_str(&mut self, s: &str) -> Result<Literal<Self::Span, Self::Symbol>, ()> {
+    fn literal_from_str(&mut self, s: &str) -> Result<Literal<Self::Span, Self::Symbol>, String> {
         let name = FileName::proc_macro_source_code(s);
 
-        let mut parser = unwrap_or_emit_fatal(new_parser_from_source_str(
-            self.psess(),
-            name,
-            s.to_owned(),
-            StripTokens::Nothing,
-        ));
+        let mut parser =
+            new_parser_from_source_str(self.psess(), name, s.to_owned(), StripTokens::Nothing)
+                .map_err(cancel_diags_into_string)?;
 
         let first_span = parser.token.span.data();
         let minus_present = parser.eat(exp!(Minus));
 
         let lit_span = parser.token.span.data();
         let token::Literal(mut lit) = parser.token.kind else {
-            return Err(());
+            return Err("not a literal".to_string());
         };
 
         // Check no comment or whitespace surrounding the (possibly negative)
         // literal, or more tokens after it.
         if (lit_span.hi.0 - first_span.lo.0) as usize != s.len() {
-            return Err(());
+            return Err("comment or whitespace around literal".to_string());
         }
 
         if minus_present {
             // If minus is present, check no comment or whitespace in between it
             // and the literal token.
             if first_span.hi.0 != lit_span.lo.0 {
-                return Err(());
+                return Err("comment or whitespace after minus".to_string());
             }
 
             // Check literal is a kind we allow to be negated in a proc macro token.
@@ -525,7 +529,9 @@ impl server::FreeFunctions for Rustc<'_, '_> {
                 | token::LitKind::ByteStrRaw(_)
                 | token::LitKind::CStr
                 | token::LitKind::CStrRaw(_)
-                | token::LitKind::Err(_) => return Err(()),
+                | token::LitKind::Err(_) => {
+                    return Err("non-numeric literal may not be negated".to_string());
+                }
                 token::LitKind::Integer | token::LitKind::Float => {}
             }
 
@@ -562,13 +568,14 @@ impl server::TokenStream for Rustc<'_, '_> {
         stream.is_empty()
     }
 
-    fn from_str(&mut self, src: &str) -> Self::TokenStream {
-        unwrap_or_emit_fatal(source_str_to_stream(
+    fn from_str(&mut self, src: &str) -> Result<Self::TokenStream, String> {
+        source_str_to_stream(
             self.psess(),
             FileName::proc_macro_source_code(src),
             src.to_string(),
             Some(self.call_site),
-        ))
+        )
+        .map_err(cancel_diags_into_string)
     }
 
     fn to_string(&mut self, stream: &Self::TokenStream) -> String {

library/proc_macro/src/bridge/mod.rs

@@ -53,15 +53,15 @@ macro_rules! with_api {
                 fn injected_env_var(var: &str) -> Option<String>;
                 fn track_env_var(var: &str, value: Option<&str>);
                 fn track_path(path: &str);
-                fn literal_from_str(s: &str) -> Result<Literal<$S::Span, $S::Symbol>, ()>;
+                fn literal_from_str(s: &str) -> Result<Literal<$S::Span, $S::Symbol>, String>;
                 fn emit_diagnostic(diagnostic: Diagnostic<$S::Span>);
             },
             TokenStream {
                 fn drop($self: $S::TokenStream);
                 fn clone($self: &$S::TokenStream) -> $S::TokenStream;
                 fn is_empty($self: &$S::TokenStream) -> bool;
                 fn expand_expr($self: &$S::TokenStream) -> Result<$S::TokenStream, ()>;
-                fn from_str(src: &str) -> $S::TokenStream;
+                fn from_str(src: &str) -> Result<$S::TokenStream, String>;
                 fn to_string($self: &$S::TokenStream) -> String;
                 fn from_token_tree(
                     tree: TokenTree<$S::TokenStream, $S::Span, $S::Symbol>,

library/proc_macro/src/lib.rs

@@ -110,12 +110,12 @@ impl !Sync for TokenStream {}
 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
 #[non_exhaustive]
 #[derive(Debug)]
-pub struct LexError;
+pub struct LexError(String);
 
 #[stable(feature = "proc_macro_lexerror_impls", since = "1.44.0")]
 impl fmt::Display for LexError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.write_str("cannot parse string into token stream")
+        f.write_str(&self.0)
     }
 }
 
@@ -194,7 +194,7 @@ impl FromStr for TokenStream {
     type Err = LexError;
 
     fn from_str(src: &str) -> Result<TokenStream, LexError> {
-        Ok(TokenStream(Some(bridge::client::TokenStream::from_str(src))))
+        Ok(TokenStream(Some(bridge::client::TokenStream::from_str(src).map_err(LexError)?)))
     }
 }
 
@@ -1559,10 +1559,7 @@ impl FromStr for Literal {
     type Err = LexError;
 
     fn from_str(src: &str) -> Result<Self, LexError> {
-        match bridge::client::FreeFunctions::literal_from_str(src) {
-            Ok(literal) => Ok(Literal(literal)),
-            Err(()) => Err(LexError),
-        }
+        bridge::client::FreeFunctions::literal_from_str(src).map(Literal).map_err(LexError)
     }
 }
 

src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs

@@ -49,10 +49,17 @@ impl server::FreeFunctions for RaSpanServer {
         self.tracked_paths.insert(path.into());
     }
 
+    #[cfg(bootstrap)]
     fn literal_from_str(&mut self, s: &str) -> Result<Literal<Self::Span>, ()> {
         literal_from_str(s, self.call_site)
     }
 
+    #[cfg(not(bootstrap))]
+    fn literal_from_str(&mut self, s: &str) -> Result<Literal<Self::Span>, String> {
+        literal_from_str(s, self.call_site)
+            .map_err(|()| "cannot parse string into literal".to_string())
+    }
+
     fn emit_diagnostic(&mut self, _: Diagnostic<Self::Span>) {
         // FIXME handle diagnostic
     }
@@ -62,6 +69,12 @@ impl server::TokenStream for RaSpanServer {
     fn is_empty(&mut self, stream: &Self::TokenStream) -> bool {
         stream.is_empty()
     }
+    #[cfg(not(bootstrap))]
+    fn from_str(&mut self, src: &str) -> Result<Self::TokenStream, String> {
+        Self::TokenStream::from_str(src, self.call_site)
+            .map_err(|e| format!("failed to parse str to token stream: {e}"))
+    }
+    #[cfg(bootstrap)]
     fn from_str(&mut self, src: &str) -> Self::TokenStream {
         Self::TokenStream::from_str(src, self.call_site).unwrap_or_else(|e| {
             Self::TokenStream::from_str(

src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/token_id.rs

@@ -47,24 +47,38 @@ impl server::FreeFunctions for SpanIdServer {
     fn injected_env_var(&mut self, _: &str) -> Option<std::string::String> {
         None
     }
+
     fn track_env_var(&mut self, var: &str, value: Option<&str>) {
         self.tracked_env_vars.insert(var.into(), value.map(Into::into));
     }
     fn track_path(&mut self, path: &str) {
         self.tracked_paths.insert(path.into());
     }
 
+    #[cfg(bootstrap)]
     fn literal_from_str(&mut self, s: &str) -> Result<Literal<Self::Span>, ()> {
         literal_from_str(s, self.call_site)
     }
 
+    #[cfg(not(bootstrap))]
+    fn literal_from_str(&mut self, s: &str) -> Result<Literal<Self::Span>, String> {
+        literal_from_str(s, self.call_site)
+            .map_err(|()| "cannot parse string into literal".to_string())
+    }
+
     fn emit_diagnostic(&mut self, _: Diagnostic<Self::Span>) {}
 }
 
 impl server::TokenStream for SpanIdServer {
     fn is_empty(&mut self, stream: &Self::TokenStream) -> bool {
         stream.is_empty()
     }
+    #[cfg(not(bootstrap))]
+    fn from_str(&mut self, src: &str) -> Result<Self::TokenStream, String> {
+        Self::TokenStream::from_str(src, self.call_site)
+            .map_err(|e| format!("failed to parse str to token stream: {e}"))
+    }
+    #[cfg(bootstrap)]
     fn from_str(&mut self, src: &str) -> Self::TokenStream {
         Self::TokenStream::from_str(src, self.call_site).unwrap_or_else(|e| {
             Self::TokenStream::from_str(

tests/ui/proc-macro/auxiliary/invalid-punct-ident.rs

@@ -20,5 +20,9 @@ pub fn invalid_raw_ident(_: TokenStream) -> TokenStream {
 
 #[proc_macro]
 pub fn lexer_failure(_: TokenStream) -> TokenStream {
-    "a b ) c".parse().expect("parsing failed without panic")
+    assert_eq!(
+        "a b ) c".parse::<TokenStream>().unwrap_err().to_string(),
+        "unexpected closing delimiter: `)`"
+    );
+    TokenStream::new()
 }

tests/ui/proc-macro/auxiliary/nonfatal-parsing-body.rs

@@ -110,6 +110,10 @@ pub fn run() {
     lit("3//\n4", NormalErr);
     lit("18.u8E", NormalErr);
     lit("/*a*/ //", NormalErr);
+    stream("1 ) 2", NormalErr);
+    stream("( x  [ ) ]", NormalErr);
+    lit("1 ) 2", NormalErr);
+    lit("( x  [ ) ]", NormalErr);
     // FIXME: all of the cases below should return an Err and emit no diagnostics, but don't yet.
 
     // emits diagnostics and returns LexError
@@ -122,8 +126,6 @@ pub fn run() {
 
     for parse in [stream as fn(&str, Mode), lit] {
         // emits diagnostic(s), then panics
-        parse("1 ) 2", OtherWithPanic);
-        parse("( x  [ ) ]", OtherWithPanic);
         parse("r#", OtherWithPanic);
 
         // emits diagnostic(s), then returns Ok(Literal { kind: ErrWithGuar, .. })

tests/ui/proc-macro/invalid-punct-ident-4.rs

@@ -1,13 +1,9 @@
 //@ proc-macro: invalid-punct-ident.rs
-//@ needs-unwind proc macro panics to report errors
+//@ check-pass
 
 #[macro_use]
 extern crate invalid_punct_ident;
 
 lexer_failure!();
-//~^ ERROR proc macro panicked
-//~| ERROR unexpected closing delimiter: `)`
 
-fn main() {
-    let _recovery_witness: () = 0; //~ ERROR mismatched types
-}
+fn main() {}