From 4815754b93cf191eeaf69469677e667e7125d272 Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Sat, 5 Jul 2025 22:32:26 -0700 Subject: [PATCH] Allow matching enums without needing `alloca`s --- compiler/rustc_codegen_ssa/src/mir/analyze.rs | 147 +++++++++----- compiler/rustc_codegen_ssa/src/mir/locals.rs | 1 + compiler/rustc_codegen_ssa/src/mir/mod.rs | 1 + compiler/rustc_codegen_ssa/src/mir/operand.rs | 63 +++--- tests/codegen/array-cmp.rs | 10 +- tests/codegen/common_prim_int_ptr.rs | 6 +- tests/codegen/enum/enum-extract.rs | 187 ++++++++++++++++++ tests/codegen/enum/enum-match.rs | 27 ++- tests/codegen/enum/enum-two-variants-match.rs | 16 +- tests/codegen/intrinsics/cold_path3.rs | 4 +- tests/codegen/try_question_mark_nop.rs | 29 +-- tests/codegen/union-abi.rs | 16 +- 12 files changed, 385 insertions(+), 122 deletions(-) create mode 100644 tests/codegen/enum/enum-extract.rs diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs index 6d6465dd798b5..86f9826e40de2 100644 --- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs +++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs @@ -1,12 +1,13 @@ //! An analysis to determine which locals require allocas and //! which do not. +use rustc_abi as abi; use rustc_data_structures::graph::dominators::Dominators; use rustc_index::bit_set::DenseBitSet; use rustc_index::{IndexSlice, IndexVec}; use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::{self, DefLocation, Location, TerminatorKind, traversal}; -use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; +use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::{bug, span_bug}; use tracing::debug; @@ -99,63 +100,113 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'a, 'b, 'tcx, Bx> context: PlaceContext, location: Location, ) { - let cx = self.fx.cx; + const COPY_CONTEXT: PlaceContext = + PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy); + + // `PlaceElem::Index` is the only variant that can mention other `Local`s, + // so check for those up-front before any potential short-circuits. + for elem in place_ref.projection { + if let mir::PlaceElem::Index(index_local) = *elem { + self.visit_local(index_local, COPY_CONTEXT, location); + } + } - if let Some((place_base, elem)) = place_ref.last_projection() { - let mut base_context = if context.is_mutating_use() { - PlaceContext::MutatingUse(MutatingUseContext::Projection) - } else { - PlaceContext::NonMutatingUse(NonMutatingUseContext::Projection) - }; + // If our local is already memory, nothing can make it *more* memory + // so we don't need to bother checking the projections further. + if self.locals[place_ref.local] == LocalKind::Memory { + return; + } - // Allow uses of projections that are ZSTs or from scalar fields. - let is_consume = matches!( - context, - PlaceContext::NonMutatingUse( - NonMutatingUseContext::Copy | NonMutatingUseContext::Move, - ) - ); - if is_consume { - let base_ty = place_base.ty(self.fx.mir, cx.tcx()); - let base_ty = self.fx.monomorphize(base_ty); - - // ZSTs don't require any actual memory access. - let elem_ty = base_ty.projection_ty(cx.tcx(), self.fx.monomorphize(elem)).ty; - let span = self.fx.mir.local_decls[place_ref.local].source_info.span; - if cx.spanned_layout_of(elem_ty, span).is_zst() { - return; + if place_ref.is_indirect_first_projection() { + // If this starts with a `Deref`, we only need to record a read of the + // pointer being dereferenced, as all the subsequent projections are + // working on a place which is always supported. (And because we're + // looking at codegen MIR, it can only happen as the first projection.) + self.visit_local(place_ref.local, COPY_CONTEXT, location); + return; + } + + if !place_ref.projection.is_empty() { + if context.is_mutating_use() { + // If it's a mutating use it doesn't matter what the projections are, + // if there are *any* then we need a place to write. (For example, + // `_1 = Foo()` works in SSA but `_2.0 = Foo()` does not.) + let mut_projection = PlaceContext::MutatingUse(MutatingUseContext::Projection); + self.visit_local(place_ref.local, mut_projection, location); + return; + } + + // Scan through to ensure the only projections are those which + // `FunctionCx::maybe_codegen_consume_direct` can handle. + let base_ty = self.fx.monomorphized_place_ty(mir::PlaceRef::from(place_ref.local)); + let mut layout = self.fx.cx.layout_of(base_ty); + for elem in place_ref.projection { + if layout.is_zst() { + // Any further projections must still be ZSTs, so we're good. + break; } - if let mir::ProjectionElem::Field(..) = elem { - let layout = cx.spanned_layout_of(base_ty.ty, span); - if cx.is_backend_immediate(layout) || cx.is_backend_scalar_pair(layout) { - // Recurse with the same context, instead of `Projection`, - // potentially stopping at non-operand projections, - // which would trigger `not_ssa` on locals. - base_context = context; + #[track_caller] + fn compatible_projection(src: TyAndLayout<'_>, tgt: TyAndLayout<'_>) -> bool { + fn compatible_initness(a: abi::Scalar, b: abi::Scalar) -> bool { + !a.is_uninit_valid() || b.is_uninit_valid() + } + + use abi::BackendRepr::*; + match (src.backend_repr, tgt.backend_repr) { + _ if tgt.is_zst() => true, + (Scalar(a), Scalar(b)) + | (SimdVector { element: a, .. }, SimdVector { element: b, .. }) => { + compatible_initness(a, b) + } + (ScalarPair(a0, a1), Scalar(b)) => { + compatible_initness(a0, b) && compatible_initness(a1, b) + } + (ScalarPair(a0, a1), ScalarPair(b0, b1)) => { + compatible_initness(a0, b0) && compatible_initness(a1, b1) + } + // This arm is a hack; remove it as part of + // + (SimdVector { .. }, Memory { .. }) => true, + _ => bug!("Mismatched layouts in analysis\nsrc: {src:?}\ntgt: {tgt:?}"), } } - } - if let mir::ProjectionElem::Deref = elem { - // Deref projections typically only read the pointer. - base_context = PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy); - } + match *elem { + mir::PlaceElem::Field(fidx, ..) => { + let field_layout = layout.field(self.fx.cx, fidx.as_usize()); + if compatible_projection(layout, field_layout) { + layout = field_layout; + continue; + } + } + mir::PlaceElem::Downcast(_, vidx) => { + let variant_layout = layout.for_variant(self.fx.cx, vidx); + if compatible_projection(layout, variant_layout) { + layout = variant_layout; + continue; + } + } + + mir::PlaceElem::Index(..) + | mir::PlaceElem::ConstantIndex { .. } + | mir::PlaceElem::Subslice { .. } + | mir::PlaceElem::OpaqueCast(..) + | mir::PlaceElem::UnwrapUnsafeBinder(..) + | mir::PlaceElem::Subtype(..) => {} + + mir::PlaceElem::Deref => bug!("Deref after first position"), + } - self.process_place(&place_base, base_context, location); - // HACK(eddyb) this emulates the old `visit_projection_elem`, this - // entire `visit_place`-like `process_place` method should be rewritten, - // now that we have moved to the "slice of projections" representation. - if let mir::ProjectionElem::Index(local) = elem { - self.visit_local( - local, - PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy), - location, - ); + // If the above didn't `continue`, we can't handle the projection. + self.locals[place_ref.local] = LocalKind::Memory; + return; } - } else { - self.visit_local(place_ref.local, context, location); } + + // Even with supported projections, we still need to have `visit_local` + // check for things that can't be done in SSA (like `SharedBorrow`). + self.visit_local(place_ref.local, context, location); } } diff --git a/compiler/rustc_codegen_ssa/src/mir/locals.rs b/compiler/rustc_codegen_ssa/src/mir/locals.rs index 93f0ab36f2a2f..741c1f0a3610d 100644 --- a/compiler/rustc_codegen_ssa/src/mir/locals.rs +++ b/compiler/rustc_codegen_ssa/src/mir/locals.rs @@ -12,6 +12,7 @@ use tracing::{debug, warn}; use crate::mir::{FunctionCx, LocalRef}; use crate::traits::BuilderMethods; +#[derive(Debug)] pub(super) struct Locals<'tcx, V> { values: IndexVec>, } diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index fa69820d5d2ed..5d2fb1e197f3f 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -136,6 +136,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } +#[derive(Debug)] enum LocalRef<'tcx, V> { Place(PlaceRef<'tcx, V>), /// `UnsizedPlace(p)`: `p` itself is a thin pointer (indirect place). diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index b0d191528a891..5be3c91666e4e 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -126,7 +126,7 @@ pub struct OperandRef<'tcx, V> { pub layout: TyAndLayout<'tcx>, } -impl fmt::Debug for OperandRef<'_, V> { +impl fmt::Debug for OperandRef<'_, V> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "OperandRef({:?} @ {:?})", self.val, self.layout) } @@ -361,13 +361,17 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { let (in_scalar, imm) = match (self.val, self.layout.backend_repr) { // Extract a scalar component from a pair. (OperandValue::Pair(a_llval, b_llval), BackendRepr::ScalarPair(a, b)) => { - if offset.bytes() == 0 { + // This needs to look at `offset`, rather than `i`, because + // for a type like `Option`, the first thing in the pair + // is the tag, so `(_2 as Some).0` needs to read the *second* + // thing in the pair despite it being "field zero". + if offset == Size::ZERO { assert_eq!(field.size, a.size(bx.cx())); - (Some(a), a_llval) + (a, a_llval) } else { assert_eq!(offset, a.size(bx.cx()).align_to(b.align(bx.cx()).abi)); assert_eq!(field.size, b.size(bx.cx())); - (Some(b), b_llval) + (b, b_llval) } } @@ -378,23 +382,12 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { OperandValue::Immediate(match field.backend_repr { BackendRepr::SimdVector { .. } => imm, BackendRepr::Scalar(out_scalar) => { - let Some(in_scalar) = in_scalar else { - span_bug!( - fx.mir.span, - "OperandRef::extract_field({:?}): missing input scalar for output scalar", - self - ) - }; - if in_scalar != out_scalar { - // If the backend and backend_immediate types might differ, - // flip back to the backend type then to the new immediate. - // This avoids nop truncations, but still handles things like - // Bools in union fields needs to be truncated. - let backend = bx.from_immediate(imm); - bx.to_immediate_scalar(backend, out_scalar) - } else { - imm - } + // For a type like `Result` the layout is `Pair(i64, ptr)`. + // But if we're reading the `Ok` payload, we need to turn that `ptr` + // back into an integer. To avoid repeating logic we do that by + // calling the transmute code, which is legal thanks to the size + // assert we did when pulling it out of the pair. + transmute_scalar(bx, imm, in_scalar, out_scalar) } BackendRepr::ScalarPair(_, _) | BackendRepr::Memory { .. } => bug!(), }) @@ -875,7 +868,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { LocalRef::Operand(mut o) => { // Moves out of scalar and scalar pair fields are trivial. for elem in place_ref.projection.iter() { - match elem { + match *elem { mir::ProjectionElem::Field(f, _) => { assert!( !o.layout.ty.is_any_ptr(), @@ -884,17 +877,21 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { ); o = o.extract_field(self, bx, f.index()); } - mir::ProjectionElem::Index(_) - | mir::ProjectionElem::ConstantIndex { .. } => { - // ZSTs don't require any actual memory access. - // FIXME(eddyb) deduplicate this with the identical - // checks in `codegen_consume` and `extract_field`. - let elem = o.layout.field(bx.cx(), 0); - if elem.is_zst() { - o = OperandRef::zero_sized(elem); - } else { - return None; - } + mir::ProjectionElem::Downcast(_sym, variant_idx) => { + let layout = o.layout.for_variant(bx.cx(), variant_idx); + let val = match layout.backend_repr { + // The transmute here handles cases like `Result` + // where the immediate values need to change for + // the specific types in the cast-to variant. + BackendRepr::Scalar(..) | BackendRepr::ScalarPair(..) => { + self.codegen_transmute_operand(bx, o, layout) + } + BackendRepr::SimdVector { .. } | BackendRepr::Memory { .. } => { + o.val + } + }; + + o = OperandRef { val, layout }; } _ => return None, } diff --git a/tests/codegen/array-cmp.rs b/tests/codegen/array-cmp.rs index 0d33765540176..30bdc686a7e14 100644 --- a/tests/codegen/array-cmp.rs +++ b/tests/codegen/array-cmp.rs @@ -39,6 +39,10 @@ pub fn array_of_tuple_le(a: &[(i16, u16); 2], b: &[(i16, u16); 2]) -> bool { // CHECK: %[[EQ00:.+]] = icmp eq i16 %[[A00]], %[[B00]] // CHECK-NEXT: br i1 %[[EQ00]], label %[[L01:.+]], label %[[EXIT_S:.+]] + // CHECK: [[EXIT_S]]: + // CHECK: %[[RET_S:.+]] = icmp sle i16 + // CHECK: br label + // CHECK: [[L01]]: // CHECK: %[[PA01:.+]] = getelementptr{{.+}}i8, ptr %a, {{i32|i64}} 2 // CHECK: %[[PB01:.+]] = getelementptr{{.+}}i8, ptr %b, {{i32|i64}} 2 @@ -66,8 +70,12 @@ pub fn array_of_tuple_le(a: &[(i16, u16); 2], b: &[(i16, u16); 2]) -> bool { // CHECK: %[[EQ11:.+]] = icmp eq i16 %[[A11]], %[[B11]] // CHECK-NEXT: br i1 %[[EQ11]], label %[[DONE:.+]], label %[[EXIT_U]] + // CHECK: [[EXIT_U]]: + // CHECK: %[[RET_U:.+]] = icmp ule i16 + // CHECK: br label + // CHECK: [[DONE]]: - // CHECK: %[[RET:.+]] = phi i1 [ %{{.+}}, %[[EXIT_S]] ], [ %{{.+}}, %[[EXIT_U]] ], [ true, %[[L11]] ] + // CHECK: %[[RET:.+]] = phi i1 [ true, %[[L11]] ], [ %[[RET_S]], %[[EXIT_S]] ], [ %[[RET_U]], %[[EXIT_U]] ] // CHECK: ret i1 %[[RET]] a <= b diff --git a/tests/codegen/common_prim_int_ptr.rs b/tests/codegen/common_prim_int_ptr.rs index 53716adccbf21..5283a3e7ae1bb 100644 --- a/tests/codegen/common_prim_int_ptr.rs +++ b/tests/codegen/common_prim_int_ptr.rs @@ -40,9 +40,13 @@ pub unsafe fn extract_int(x: Result>) -> usize { } // CHECK-LABEL: @extract_box -// CHECK-SAME: (i{{[0-9]+}} {{[^%]+}} [[DISCRIMINANT:%[0-9]+]], ptr {{[^%]+}} [[PAYLOAD:%[0-9]+]]) +// CHECK-SAME: (i{{[0-9]+}} {{[^%]+}} [[DISCRIMINANT:%x.0]], ptr {{[^%]+}} [[PAYLOAD:%x.1]]) #[no_mangle] pub unsafe fn extract_box(x: Result>) -> Box { + // CHECK: [[NOT_OK:%.+]] = icmp ne i{{[0-9]+}} [[DISCRIMINANT]], 0 + // CHECK: call void @llvm.assume(i1 [[NOT_OK]]) + // CHECK: [[NOT_NULL:%.+]] = icmp ne ptr [[PAYLOAD]], null + // CHECK: call void @llvm.assume(i1 [[NOT_NULL]]) // CHECK: ret ptr [[PAYLOAD]] match x { Ok(_) => std::intrinsics::unreachable(), diff --git a/tests/codegen/enum/enum-extract.rs b/tests/codegen/enum/enum-extract.rs new file mode 100644 index 0000000000000..60736f027394a --- /dev/null +++ b/tests/codegen/enum/enum-extract.rs @@ -0,0 +1,187 @@ +//@ revisions: OPT DBG +//@ compile-flags: -Cno-prepopulate-passes -Cdebuginfo=0 +//@[OPT] compile-flags: -Copt-level=1 +//@[DBG] compile-flags: -Copt-level=0 +//@ min-llvm-version: 19 +//@ only-64bit + +#![crate_type = "lib"] + +// This tests various cases around consuming enums as SSA values in what we emit. +// Importantly, it checks things like correct `i1` handling for `bool` +// and for mixed integer/pointer payloads. + +use std::cmp::Ordering; +use std::mem::MaybeUninit; +use std::num::NonZero; +use std::ptr::NonNull; + +// This doesn't actually end up in an SSA value because `extract_field` +// doesn't know how to do the equivalent of `!noundef` without a load. +#[no_mangle] +fn use_option_u32(x: Option) -> u32 { + // CHECK-LABEL: @use_option_u32 + // OPT-SAME: (i32 noundef range(i32 0, 2) %0, i32 %1) + + // CHECK-NOT: alloca + // CHECK: %x = alloca [8 x i8] + // CHECK-NOT: alloca + // CHECK: %[[X0:.+]] = load i32, ptr %x + // CHECK: %[[DISCR:.+]] = zext i32 %[[X0]] to i64 + // CHECK: %[[IS_SOME:.+]] = trunc nuw i64 %[[DISCR]] to i1 + // OPT: %[[LIKELY:.+]] = call i1 @llvm.expect.i1(i1 %[[IS_SOME]], i1 true) + // OPT: br i1 %[[LIKELY]], label %[[BLOCK:.+]], + // DBG: br i1 %[[IS_SOME]], label %[[BLOCK:.+]], + + // CHECK: [[BLOCK]]: + // CHECK: %[[X1P:.+]] = getelementptr inbounds i8, ptr %x, i64 4 + // CHECK: %[[X1:.+]] = load i32, ptr %[[X1P]] + // OPT-SAME: !noundef + // CHECK: ret i32 %[[X1]] + + if let Some(val) = x { val } else { unreachable!() } +} + +#[no_mangle] +fn use_option_mu_i32(x: Option>) -> MaybeUninit { + // CHECK-LABEL: @use_option_mu_i32 + // OPT-SAME: (i32 noundef range(i32 0, 2) %x.0, i32 %x.1) + + // CHECK-NOT: alloca + // CHECK: %[[DISCR:.+]] = zext i32 %x.0 to i64 + // CHECK: %[[IS_SOME:.+]] = trunc nuw i64 %[[DISCR]] to i1 + // OPT: %[[LIKELY:.+]] = call i1 @llvm.expect.i1(i1 %[[IS_SOME]], i1 true) + // OPT: br i1 %[[LIKELY]], label %[[BLOCK:.+]], + // DBG: br i1 %[[IS_SOME]], label %[[BLOCK:.+]], + + // CHECK: [[BLOCK]]: + // CHECK: ret i32 %x.1 + + if let Some(val) = x { val } else { unreachable!() } +} + +#[no_mangle] +fn use_option_bool(x: Option) -> bool { + // CHECK-LABEL: @use_option_bool + // OPT-SAME: (i8 noundef range(i8 0, 3) %x) + + // CHECK-NOT: alloca + // CHECK: %[[IS_NONE:.+]] = icmp eq i8 %x, 2 + // CHECK: %[[DISCR:.+]] = select i1 %[[IS_NONE]], i64 0, i64 1 + // CHECK: %[[IS_SOME:.+]] = trunc nuw i64 %[[DISCR]] to i1 + // OPT: %[[LIKELY:.+]] = call i1 @llvm.expect.i1(i1 %[[IS_SOME]], i1 true) + // OPT: br i1 %[[LIKELY]], label %[[BLOCK:.+]], + // DBG: br i1 %[[IS_SOME]], label %[[BLOCK:.+]], + + // CHECK: [[BLOCK]]: + // CHECK: %val = trunc nuw i8 %x to i1 + // CHECK: ret i1 %val + + if let Some(val) = x { val } else { unreachable!() } +} + +#[no_mangle] +fn use_option_ordering(x: Option) -> Ordering { + // CHECK-LABEL: @use_option_ordering + // OPT-SAME: (i8 noundef range(i8 -1, 3) %x) + + // CHECK: %[[IS_NONE:.+]] = icmp eq i8 %x, 2 + // CHECK: %[[DISCR:.+]] = select i1 %[[IS_NONE]], i64 0, i64 1 + // CHECK: %[[IS_SOME:.+]] = trunc nuw i64 %[[DISCR]] to i1 + // OPT: %[[LIKELY:.+]] = call i1 @llvm.expect.i1(i1 %[[IS_SOME]], i1 true) + // OPT: br i1 %[[LIKELY]], label %[[BLOCK:.+]], + // DBG: br i1 %[[IS_SOME]], label %[[BLOCK:.+]], + + // CHECK: [[BLOCK]]: + // OPT: %[[SHIFTED:.+]] = sub i8 %x, -1 + // OPT: %[[IN_WIDTH:.+]] = icmp ule i8 %[[SHIFTED]], 3 + // OPT: call void @llvm.assume(i1 %[[IN_WIDTH]]) + // DBG-NOT: assume + // CHECK: ret i8 %x + + if let Some(val) = x { val } else { unreachable!() } +} + +#[no_mangle] +fn use_result_nzusize(x: Result, NonNull>) -> NonZero { + // CHECK-LABEL: @use_result_nzusize + // OPT-SAME: (i64 noundef range(i64 0, 2) %x.0, ptr noundef %x.1) + + // CHECK-NOT: alloca + // CHECK: %[[IS_ERR:.+]] = trunc nuw i64 %x.0 to i1 + // OPT: %[[UNLIKELY:.+]] = call i1 @llvm.expect.i1(i1 %[[IS_ERR]], i1 false) + // OPT: br i1 %[[UNLIKELY]], label %[[PANIC:.+]], label %[[BLOCK:.+]] + // DBG: br i1 %[[IS_ERR]], label %[[PANIC:.+]], label %[[BLOCK:.+]] + + // CHECK: [[BLOCK]]: + // CHECK: %val = ptrtoint ptr %x.1 to i64 + // CHECK: ret i64 %val + + if let Ok(val) = x { val } else { unreachable!() } +} + +#[no_mangle] +fn use_result_i32_char(x: Result) -> char { + // CHECK-LABEL: @use_result_i32_char + // OPT-SAME: (i32 noundef range(i32 0, 2) %x.0, i32 noundef %x.1) + + // CHECK-NOT: alloca + // CHECK: %[[IS_ERR_WIDE:.+]] = zext i32 %x.0 to i64 + // CHECK: %[[IS_ERR:.+]] = trunc nuw i64 %[[IS_ERR_WIDE]] to i1 + // OPT: %[[LIKELY:.+]] = call i1 @llvm.expect.i1(i1 %[[IS_ERR]], i1 true) + // OPT: br i1 %[[LIKELY]], label %[[BLOCK:.+]], label %[[PANIC:.+]] + // DBG: br i1 %[[IS_ERR]], label %[[BLOCK:.+]], label %[[PANIC:.+]] + + // CHECK: [[BLOCK]]: + // OPT: %[[RANGE:.+]] = icmp ule i32 %x.1, 1114111 + // OPT: call void @llvm.assume(i1 %[[RANGE]]) + // DBG-NOT: call + // CHECK: ret i32 %x.1 + + if let Err(val) = x { val } else { unreachable!() } +} + +#[repr(u64)] +enum BigEnum { + Foo = 100, + Bar = 200, +} + +#[no_mangle] +fn use_result_bigenum(x: Result) -> BigEnum { + // CHECK-LABEL: @use_result_bigenum + // OPT-SAME: (i64 noundef range(i64 0, 2) %x.0, i64 noundef %x.1) + + // CHECK-NOT: alloca + // CHECK: %[[IS_ERR:.+]] = trunc nuw i64 %x.0 to i1 + // OPT: %[[UNLIKELY:.+]] = call i1 @llvm.expect.i1(i1 %[[IS_ERR]], i1 false) + // OPT: br i1 %[[UNLIKELY]], label %[[PANIC:.+]], label %[[BLOCK:.+]] + // DBG: br i1 %[[IS_ERR]], label %[[PANIC:.+]], label %[[BLOCK:.+]] + + // CHECK: [[BLOCK]]: + // CHECK: ret i64 %x.1 + + if let Ok(val) = x { val } else { unreachable!() } +} + +struct WhateverError; + +#[no_mangle] +fn use_result_nonnull(x: Result, WhateverError>) -> NonNull { + // CHECK-LABEL: @use_result_nonnull + // OPT-SAME: (ptr noundef %x) + + // CHECK-NOT: alloca + // CHECK: %[[ADDR:.+]] = ptrtoint ptr %x to i64 + // CHECK: %[[IS_NULL:.+]] = icmp eq i64 %[[ADDR]], 0 + // CHECK: %[[DISCR:.+]] = select i1 %[[IS_NULL]], i64 1, i64 0 + // CHECK: %[[IS_ERR:.+]] = trunc nuw i64 %[[DISCR]] to i1 + // OPT: %[[UNLIKELY:.+]] = call i1 @llvm.expect.i1(i1 %[[IS_ERR]], i1 false) + // OPT: br i1 %[[UNLIKELY]], label %[[PANIC:.+]], label %[[BLOCK:.+]] + // DBG: br i1 %[[IS_ERR]], label %[[PANIC:.+]], label %[[BLOCK:.+]] + + // CHECK: [[BLOCK]]: + // CHECK: ret ptr %x + + if let Ok(val) = x { val } else { unreachable!() } +} diff --git a/tests/codegen/enum/enum-match.rs b/tests/codegen/enum/enum-match.rs index 98635008d068f..c4a7f248dd784 100644 --- a/tests/codegen/enum/enum-match.rs +++ b/tests/codegen/enum/enum-match.rs @@ -15,11 +15,10 @@ pub enum Enum0 { B, } -// CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match0(i8{{.+}}%0) +// CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match0(i8{{.+}}%e) // CHECK-NEXT: start: -// CHECK-NEXT: %[[IS_B:.+]] = icmp eq i8 %0, 2 -// CHECK-NEXT: %[[TRUNC:.+]] = and i8 %0, 1 -// CHECK-NEXT: %[[R:.+]] = select i1 %[[IS_B]], i8 13, i8 %[[TRUNC]] +// CHECK-NEXT: %[[IS_B:.+]] = icmp eq i8 %e, 2 +// CHECK-NEXT: %[[R:.+]] = select i1 %[[IS_B]], i8 13, i8 %e // CHECK-NEXT: ret i8 %[[R]] #[no_mangle] pub fn match0(e: Enum0) -> u8 { @@ -37,9 +36,9 @@ pub enum Enum1 { C, } -// CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match1(i8{{.+}}%0) +// CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match1(i8{{.+}}%e) // CHECK-NEXT: start: -// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %0, -2 +// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %e, -2 // CHECK-NEXT: %[[REL_VAR_WIDE:.+]] = zext i8 %[[REL_VAR]] to i64 // CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ult i8 %[[REL_VAR]], 2 // CHECK-NEXT: %[[NICHE_DISCR:.+]] = add nuw nsw i64 %[[REL_VAR_WIDE]], 1 @@ -98,9 +97,9 @@ pub enum Enum2 { E, } -// CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i8 [0-9]+, -?[0-9]+\))?}} i8 @match2(i8{{.+}}%0) +// CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i8 [0-9]+, -?[0-9]+\))?}} i8 @match2(i8{{.+}}%e) // CHECK-NEXT: start: -// CHECK-NEXT: %[[REL_VAR:.+]] = add i8 %0, 2 +// CHECK-NEXT: %[[REL_VAR:.+]] = add i8 %e, 2 // CHECK-NEXT: %[[REL_VAR_WIDE:.+]] = zext i8 %[[REL_VAR]] to i64 // CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ult i8 %[[REL_VAR]], 4 // CHECK-NEXT: %[[NICHE_DISCR:.+]] = add nuw nsw i64 %[[REL_VAR_WIDE]], 1 @@ -121,9 +120,9 @@ pub fn match2(e: Enum2) -> u8 { // And make sure it works even if the niched scalar is a pointer. // (For example, that we don't try to `sub` on pointers.) -// CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i16 -?[0-9]+, -?[0-9]+\))?}} i16 @match3(ptr{{.+}}%0) +// CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i16 -?[0-9]+, -?[0-9]+\))?}} i16 @match3(ptr{{.+}}%e) // CHECK-NEXT: start: -// CHECK-NEXT: %[[IS_NULL:.+]] = icmp eq ptr %0, null +// CHECK-NEXT: %[[IS_NULL:.+]] = icmp eq ptr %e, null // CHECK-NEXT: br i1 %[[IS_NULL]] #[no_mangle] pub fn match3(e: Option<&u8>) -> i16 { @@ -145,9 +144,9 @@ pub enum MiddleNiche { E, } -// CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i8 -?[0-9]+, -?[0-9]+\))?}} i8 @match4(i8{{.+}}%0) +// CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i8 -?[0-9]+, -?[0-9]+\))?}} i8 @match4(i8{{.+}}%e) // CHECK-NEXT: start: -// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %0, -2 +// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %e, -2 // CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ult i8 %[[REL_VAR]], 5 // CHECK-NEXT: %[[NOT_IMPOSSIBLE:.+]] = icmp ne i8 %[[REL_VAR]], 2 // CHECK-NEXT: call void @llvm.assume(i1 %[[NOT_IMPOSSIBLE]]) @@ -449,9 +448,9 @@ pub enum HugeVariantIndex { Possible259, } -// CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match5(i8{{.+}}%0) +// CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match5(i8{{.+}}%e) // CHECK-NEXT: start: -// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %0, -2 +// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %e, -2 // CHECK-NEXT: %[[REL_VAR_WIDE:.+]] = zext i8 %[[REL_VAR]] to i64 // CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ult i8 %[[REL_VAR]], 3 // CHECK-NEXT: %[[NOT_IMPOSSIBLE:.+]] = icmp ne i8 %[[REL_VAR]], 1 diff --git a/tests/codegen/enum/enum-two-variants-match.rs b/tests/codegen/enum/enum-two-variants-match.rs index 12d9edc4d6234..39a46f07ea29a 100644 --- a/tests/codegen/enum/enum-two-variants-match.rs +++ b/tests/codegen/enum/enum-two-variants-match.rs @@ -56,18 +56,16 @@ pub fn result_match(x: Result) -> u16 { } } -// CHECK-LABEL: @option_bool_match( +// CHECK-LABEL: @option_bool_match(i8{{.+}}%x) #[no_mangle] pub fn option_bool_match(x: Option) -> char { - // CHECK: %[[RAW:.+]] = load i8, ptr %x - // CHECK: %[[IS_NONE:.+]] = icmp eq i8 %[[RAW]], 2 + // CHECK: %[[IS_NONE:.+]] = icmp eq i8 %x, 2 // CHECK: %[[OPT_DISCR:.+]] = select i1 %[[IS_NONE]], i64 0, i64 1 // CHECK: %[[OPT_DISCR_T:.+]] = trunc nuw i64 %[[OPT_DISCR]] to i1 // CHECK: br i1 %[[OPT_DISCR_T]], label %[[BB_SOME:.+]], label %[[BB_NONE:.+]] // CHECK: [[BB_SOME]]: - // CHECK: %[[FIELD:.+]] = load i8, ptr %x - // CHECK: %[[FIELD_T:.+]] = trunc nuw i8 %[[FIELD]] to i1 + // CHECK: %[[FIELD_T:.+]] = trunc nuw i8 %x to i1 // CHECK: br i1 %[[FIELD_T]] match x { None => 'n', @@ -77,18 +75,16 @@ pub fn option_bool_match(x: Option) -> char { } use std::cmp::Ordering::{self, *}; -// CHECK-LABEL: @option_ordering_match( +// CHECK-LABEL: @option_ordering_match(i8{{.+}}%x) #[no_mangle] pub fn option_ordering_match(x: Option) -> char { - // CHECK: %[[RAW:.+]] = load i8, ptr %x - // CHECK: %[[IS_NONE:.+]] = icmp eq i8 %[[RAW]], 2 + // CHECK: %[[IS_NONE:.+]] = icmp eq i8 %x, 2 // CHECK: %[[OPT_DISCR:.+]] = select i1 %[[IS_NONE]], i64 0, i64 1 // CHECK: %[[OPT_DISCR_T:.+]] = trunc nuw i64 %[[OPT_DISCR]] to i1 // CHECK: br i1 %[[OPT_DISCR_T]], label %[[BB_SOME:.+]], label %[[BB_NONE:.+]] // CHECK: [[BB_SOME]]: - // CHECK: %[[FIELD:.+]] = load i8, ptr %x - // CHECK: switch i8 %[[FIELD]], label %[[UNREACHABLE:.+]] [ + // CHECK: switch i8 %x, label %[[UNREACHABLE:.+]] [ // CHECK-NEXT: i8 -1, label // CHECK-NEXT: i8 0, label // CHECK-NEXT: i8 1, label diff --git a/tests/codegen/intrinsics/cold_path3.rs b/tests/codegen/intrinsics/cold_path3.rs index bf3347de665db..2da1f61f67665 100644 --- a/tests/codegen/intrinsics/cold_path3.rs +++ b/tests/codegen/intrinsics/cold_path3.rs @@ -44,7 +44,7 @@ pub fn test(x: Option) { _ => path_a(), } - // CHECK-LABEL: @test( + // CHECK-LABEL: @test({{.+}} %0, {{.+}} %1) // CHECK: switch i32 %1, label %bb1 [ // CHECK: i32 0, label %bb6 // CHECK: i32 1, label %bb5 @@ -75,7 +75,7 @@ pub fn test2(x: Option) { } } - // CHECK-LABEL: @test2( + // CHECK-LABEL: @test2({{.+}} %0, {{.+}} %1) // CHECK: switch i32 %1, label %bb1 [ // CHECK: i32 10, label %bb5 // CHECK: i32 11, label %bb4 diff --git a/tests/codegen/try_question_mark_nop.rs b/tests/codegen/try_question_mark_nop.rs index 398c9a580bc30..def3907a9e873 100644 --- a/tests/codegen/try_question_mark_nop.rs +++ b/tests/codegen/try_question_mark_nop.rs @@ -1,6 +1,7 @@ //@ compile-flags: -Copt-level=3 -Z merge-functions=disabled //@ edition: 2021 -//@ only-x86_64 +//@ only-64bit +//@ needs-deterministic-layouts (opposite scalar pair orders breaks it) //@ revisions: NINETEEN TWENTY //@[NINETEEN] exact-llvm-major-version: 19 //@[TWENTY] min-llvm-version: 20 @@ -11,7 +12,7 @@ use std::ops::ControlFlow::{self, Break, Continue}; use std::ptr::NonNull; -// CHECK-LABEL: @option_nop_match_32 +// CHECK-LABEL: @option_nop_match_32({{.+}} %0, {{.+}} %1) #[no_mangle] pub fn option_nop_match_32(x: Option) -> Option { // CHECK: start: @@ -32,7 +33,7 @@ pub fn option_nop_match_32(x: Option) -> Option { } } -// CHECK-LABEL: @option_nop_traits_32 +// CHECK-LABEL: @option_nop_traits_32({{.+}} %0, {{.+}} %1) #[no_mangle] pub fn option_nop_traits_32(x: Option) -> Option { // CHECK: start: @@ -90,7 +91,7 @@ pub fn control_flow_nop_traits_32(x: ControlFlow) -> ControlFlow) -> Option { // CHECK: start: @@ -111,7 +112,7 @@ pub fn option_nop_match_64(x: Option) -> Option { } } -// CHECK-LABEL: @option_nop_traits_64 +// CHECK-LABEL: @option_nop_traits_64({{.+}} %0, {{.+}} %1) #[no_mangle] pub fn option_nop_traits_64(x: Option) -> Option { // CHECK: start: @@ -219,13 +220,19 @@ pub fn control_flow_nop_traits_128(x: ControlFlow) -> ControlFlow>) -> Result> { + // This also has an assume that if it's `Err` the pointer is non-null + // (since it *can* be null in the `Ok` case) but for the purpose of this + // test that's ok as the returned value is just the inputs paired up. + // CHECK: start: - // CHECK-NEXT: insertvalue { i{{[0-9]+}}, ptr } - // CHECK-NEXT: insertvalue { i{{[0-9]+}}, ptr } - // CHECK-NEXT: ret + // CHECK-NOT: insertvalue + // CHECK-NOT: ret + // CHECK: %[[TEMP1:.+]] = insertvalue { i64, ptr } poison, i64 %x.0, 0 + // CHECK-NEXT: %[[TEMP2:.+]] = insertvalue { i64, ptr } %[[TEMP1]], ptr %x.1, 1 + // CHECK-NEXT: ret { i64, ptr } %[[TEMP2]] match x { Ok(x) => Ok(x), Err(x) => Err(x), @@ -236,8 +243,8 @@ pub fn result_nop_match_ptr(x: Result>) -> Result> #[no_mangle] pub fn result_nop_traits_ptr(x: Result>) -> Result> { // CHECK: start: - // CHECK-NEXT: insertvalue { i{{[0-9]+}}, ptr } - // CHECK-NEXT: insertvalue { i{{[0-9]+}}, ptr } + // CHECK-NEXT: insertvalue { i64, ptr } + // CHECK-NEXT: insertvalue { i64, ptr } // CHECK-NEXT: ret try { x? } } diff --git a/tests/codegen/union-abi.rs b/tests/codegen/union-abi.rs index 28acc4de2f327..9c0f17f704d53 100644 --- a/tests/codegen/union-abi.rs +++ b/tests/codegen/union-abi.rs @@ -137,9 +137,21 @@ pub fn test_CUnionU128(_: CUnionU128) { pub union UnionBool { b: bool, } -// CHECK: define {{(dso_local )?}}noundef zeroext i1 @test_UnionBool(i8{{.*}} %b) +// CHECK: define {{(dso_local )?}}noundef zeroext i1 @test_UnionBool(i8 %0) #[no_mangle] pub fn test_UnionBool(b: UnionBool) -> bool { + // Note the lack of `noundef` on the parameter, because (at least for now) + // all unions are allowed to be fully-uninitialized. We thus write it to + // memory and read it via `!noundef` when asserting validity. + + // CHECK-NOT: alloca + // CHECK: %b = alloca [1 x i8], align 1 + // CHECK-NOT: alloca + // CHECK: store i8 %0, ptr %b, align 1 + // CHECK: %[[WIDE:.+]] = load i8, ptr %b, align 1 + // CHECK-SAME: !noundef + // CHECK: %[[BOOL:.+]] = trunc nuw i8 %[[WIDE]] to i1 + // CHECK: ret i1 %[[BOOL]] + unsafe { b.b } } -// CHECK: %_0 = trunc{{( nuw)?}} i8 %b to i1