default OOM handler: use non-unwinding panic (unless -Zoom=panic is set), to match std handler

Author: RalfJung

library/alloc/src/alloc.rs

@@ -402,7 +402,20 @@ pub mod __alloc_error_handler {
     // `#[alloc_error_handler]`.
     #[rustc_std_internal_symbol]
     pub unsafe fn __rdl_oom(size: usize, _align: usize) -> ! {
-        panic!("memory allocation of {size} bytes failed")
+        extern "Rust" {
+            // This symbol is emitted by rustc next to __rust_alloc_error_handler.
+            // Its value depends on the -Zoom={panic,abort} compiler option.
+            static __rust_alloc_error_handler_should_panic: u8;
+        }
+
+        #[allow(unused_unsafe)]
+        if unsafe { __rust_alloc_error_handler_should_panic != 0 } {
+            panic!("memory allocation of {size} bytes failed")
+        } else {
+            core::panicking::panic_nounwind_fmt(format_args!(
+                "memory allocation of {size} bytes failed"
+            ))
+        }
     }
 
     #[cfg(bootstrap)]

library/alloc/src/lib.rs

@@ -110,6 +110,7 @@
 #![feature(const_maybe_uninit_as_mut_ptr)]
 #![feature(const_refs_to_cell)]
 #![feature(core_intrinsics)]
+#![feature(core_panic)]
 #![feature(const_eval_select)]
 #![feature(const_pin)]
 #![feature(const_waker)]

library/core/src/panicking.rs

@@ -64,13 +64,17 @@ pub const fn panic_fmt(fmt: fmt::Arguments<'_>) -> ! {
     unsafe { panic_impl(&pi) }
 }
 
-/// Like `panic`, but without unwinding and track_caller to reduce the impact on codesize.
-/// (No `fmt` variant as a `fmt::Arguments` needs more space to be passed.)
+/// Like `panic_fmt`, but for non-unwinding panics.
+///
+/// Has to be a separate function so that it can carry the `rustc_nounwind` attribute.
 #[cfg_attr(not(feature = "panic_immediate_abort"), inline(never), cold)]
 #[cfg_attr(feature = "panic_immediate_abort", inline)]
-#[cfg_attr(not(bootstrap), lang = "panic_nounwind")] // needed by codegen for non-unwinding panics
+#[track_caller]
+// This attribute has the key side-effect that if the panic handler ignores `can_unwind`
+// and unwinds anyway, we will hit the "unwinding out of nounwind function" guard,
+// which causes a "panic in a function that cannot unwind".
 #[rustc_nounwind]
-pub fn panic_nounwind(msg: &'static str) -> ! {
+pub fn panic_nounwind_fmt(fmt: fmt::Arguments<'_>) -> ! {
     if cfg!(feature = "panic_immediate_abort") {
         super::intrinsics::abort()
     }
@@ -83,8 +87,6 @@ pub fn panic_nounwind(msg: &'static str) -> ! {
     }
 
     // PanicInfo with the `can_unwind` flag set to false forces an abort.
-    let pieces = [msg];
-    let fmt = fmt::Arguments::new_v1(&pieces, &[]);
     let pi = PanicInfo::internal_constructor(Some(&fmt), Location::caller(), false);
 
     // SAFETY: `panic_impl` is defined in safe Rust code and thus is safe to call.
@@ -112,6 +114,15 @@ pub const fn panic(expr: &'static str) -> ! {
     panic_fmt(fmt::Arguments::new_v1(&[expr], &[]));
 }
 
+/// Like `panic`, but without unwinding and track_caller to reduce the impact on codesize.
+#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never), cold)]
+#[cfg_attr(feature = "panic_immediate_abort", inline)]
+#[cfg_attr(not(bootstrap), lang = "panic_nounwind")] // needed by codegen for non-unwinding panics
+#[rustc_nounwind]
+pub fn panic_nounwind(expr: &'static str) -> ! {
+    panic_nounwind_fmt(fmt::Arguments::new_v1(&[expr], &[]));
+}
+
 #[inline]
 #[track_caller]
 #[rustc_diagnostic_item = "panic_str"]

library/std/src/alloc.rs

@@ -338,7 +338,7 @@ fn default_alloc_error_hook(layout: Layout) {
 
     #[allow(unused_unsafe)]
     if unsafe { __rust_alloc_error_handler_should_panic != 0 } {
-        panic!("memory allocation of {} bytes failed\n", layout.size());
+        panic!("memory allocation of {} bytes failed", layout.size());
     } else {
         rtprintpanic!("memory allocation of {} bytes failed\n", layout.size());
     }