Rollup of 9 pull requests

compiler/rustc_codegen_cranelift/example/float-minmax-pass.rs

@@ -33,24 +33,24 @@ fn main() {
     let n = f32x4([nan, nan, nan, nan]);
 
     unsafe {
-        let min0 = simd_fmin(x, y);
-        let min1 = simd_fmin(y, x);
+        let min0 = simd_minimum_number_nsz(x, y);
+        let min1 = simd_minimum_number_nsz(y, x);
         assert_eq!(min0.into_array(), min1.into_array());
         let e = f32x4([1.0, 1.0, 3.0, 3.0]);
         assert_eq!(min0.into_array(), e.into_array());
-        let minn = simd_fmin(x, n);
+        let minn = simd_minimum_number_nsz(x, n);
         assert_eq!(minn.into_array(), x.into_array());
-        let minn = simd_fmin(y, n);
+        let minn = simd_minimum_number_nsz(y, n);
         assert_eq!(minn.into_array(), y.into_array());
 
-        let max0 = simd_fmax(x, y);
-        let max1 = simd_fmax(y, x);
+        let max0 = simd_maximum_number_nsz(x, y);
+        let max1 = simd_maximum_number_nsz(y, x);
         assert_eq!(max0.into_array(), max1.into_array());
         let e = f32x4([2.0, 2.0, 4.0, 4.0]);
         assert_eq!(max0.into_array(), e.into_array());
-        let maxn = simd_fmax(x, n);
+        let maxn = simd_maximum_number_nsz(x, n);
         assert_eq!(maxn.into_array(), x.into_array());
-        let maxn = simd_fmax(y, n);
+        let maxn = simd_maximum_number_nsz(y, n);
         assert_eq!(maxn.into_array(), y.into_array());
     }
 }

compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

@@ -346,10 +346,6 @@ fn codegen_float_intrinsic_call<'tcx>(
         sym::log10f32 => ("log10f", 1, fx.tcx.types.f32, types::F32),
         sym::log10f64 => ("log10", 1, fx.tcx.types.f64, types::F64),
         sym::log10f128 => ("log10f128", 1, fx.tcx.types.f128, types::F128),
-        sym::fabsf16 => ("fabsf16", 1, fx.tcx.types.f16, types::F16),
-        sym::fabsf32 => ("fabsf", 1, fx.tcx.types.f32, types::F32),
-        sym::fabsf64 => ("fabs", 1, fx.tcx.types.f64, types::F64),
-        sym::fabsf128 => ("fabsf128", 1, fx.tcx.types.f128, types::F128),
         sym::fmaf16 => ("fmaf16", 3, fx.tcx.types.f16, types::F16),
         sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32),
         sym::fmaf64 => ("fma", 3, fx.tcx.types.f64, types::F64),
@@ -441,11 +437,7 @@ fn codegen_float_intrinsic_call<'tcx>(
         sym::copysignf32 | sym::copysignf64 => {
             CValue::by_val(fx.bcx.ins().fcopysign(args[0], args[1]), layout)
         }
-        sym::fabsf16 => CValue::by_val(codegen_f16_f128::abs_f16(fx, args[0]), layout),
-        sym::fabsf128 => CValue::by_val(codegen_f16_f128::abs_f128(fx, args[0]), layout),
-        sym::fabsf32
-        | sym::fabsf64
-        | sym::floorf32
+        sym::floorf32
         | sym::floorf64
         | sym::ceilf32
         | sym::ceilf64
@@ -456,7 +448,6 @@ fn codegen_float_intrinsic_call<'tcx>(
         | sym::sqrtf32
         | sym::sqrtf64 => {
             let val = match intrinsic {
-                sym::fabsf32 | sym::fabsf64 => fx.bcx.ins().fabs(args[0]),
                 sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(args[0]),
                 sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(args[0]),
                 sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(args[0]),
@@ -1179,6 +1170,28 @@ fn codegen_regular_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, old);
         }
 
+        sym::fabs => {
+            intrinsic_args!(fx, args => (arg); intrinsic);
+            let layout = arg.layout();
+            let ty::Float(float_ty) = layout.ty.kind() else {
+                span_bug!(
+                    source_info.span,
+                    "expected float type for fabs intrinsic: {:?}",
+                    layout.ty
+                );
+            };
+            let x = arg.load_scalar(fx);
+            let val = match float_ty {
+                FloatTy::F32 | FloatTy::F64 => fx.bcx.ins().fabs(x),
+                // FIXME(bytecodealliance/wasmtime#8312): Use `fabsf16` once Cranelift
+                // backend lowerings are implemented.
+                FloatTy::F16 => codegen_f16_f128::abs_f16(fx, x),
+                FloatTy::F128 => codegen_f16_f128::abs_f128(fx, x),
+            };
+            let val = CValue::by_val(val, layout);
+            ret.write_cvalue(fx, val);
+        }
+
         sym::minimumf16 => {
             intrinsic_args!(fx, args => (a, b); intrinsic);
             let a = a.load_scalar(fx);

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -493,7 +493,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             }
         }
 
-        sym::simd_fmin | sym::simd_fmax => {
+        sym::simd_minimum_number_nsz | sym::simd_maximum_number_nsz => {
             intrinsic_args!(fx, args => (x, y); intrinsic);
 
             if !x.layout().ty.is_simd() {
@@ -508,8 +508,12 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                     _ => unreachable!("{:?}", lane_ty),
                 }
                 match intrinsic {
-                    sym::simd_fmin => crate::num::codegen_float_min(fx, x_lane, y_lane),
-                    sym::simd_fmax => crate::num::codegen_float_max(fx, x_lane, y_lane),
+                    sym::simd_minimum_number_nsz => {
+                        crate::num::codegen_float_min(fx, x_lane, y_lane)
+                    }
+                    sym::simd_maximum_number_nsz => {
+                        crate::num::codegen_float_max(fx, x_lane, y_lane)
+                    }
                     _ => unreachable!(),
                 }
             });

compiler/rustc_codegen_cranelift/src/num.rs

@@ -500,8 +500,8 @@ fn codegen_ptr_binop<'tcx>(
 
 // In Rust floating point min and max don't propagate NaN (not even SNaN). In Cranelift they do
 // however. For this reason it is necessary to use `a.is_nan() ? b : (a >= b ? b : a)` for
-// `minnumf*` and `a.is_nan() ? b : (a <= b ? b : a)` for `maxnumf*`. NaN checks are done by
-// comparing a float against itself. Only in case of NaN is it not equal to itself.
+// `minimum_number_nsz` and `a.is_nan() ? b : (a <= b ? b : a)` for `maximum_number_nsz`. NaN checks
+// are done by comparing a float against itself. Only in case of NaN is it not equal to itself.
 pub(crate) fn codegen_float_min(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value {
     // FIXME(bytecodealliance/wasmtime#8312): Replace with Cranelift `fcmp` once
     // `f16`/`f128` backend lowerings have been added to Cranelift.

compiler/rustc_codegen_gcc/src/builder.rs

@@ -2278,6 +2278,8 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
         })
     }
 
+    /// Emits a SIMD min/max operation for floats. The semantics for each lane are: if one
+    /// side is NaN (QNaN or SNaN), the other side is returned.
     fn vector_extremum(
         &mut self,
         a: RValue<'gcc>,
@@ -2286,8 +2288,9 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
     ) -> RValue<'gcc> {
         let vector_type = a.get_type();
 
-        // mask out the NaNs in b and replace them with the corresponding lane in a, so when a and
-        // b get compared & spliced together, we get the numeric values instead of NaNs.
+        // Mask out the NaNs (both QNaN and SNaN) in b and replace them with the corresponding lane
+        // in a, so when a and b get compared & spliced together, we get the numeric values instead
+        // of NaNs.
         let b_nan_mask = self.context.new_comparison(self.location, ComparisonOp::NotEquals, b, b);
         let mask_type = b_nan_mask.get_type();
         let b_nan_mask_inverted =
@@ -2309,7 +2312,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
         self.context.new_bitcast(self.location, res, vector_type)
     }
 
-    pub fn vector_fmin(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
+    pub fn vector_minimum_number_nsz(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
         self.vector_extremum(a, b, ExtremumOperation::Min)
     }
 
@@ -2341,7 +2344,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
         unimplemented!();
     }
 
-    pub fn vector_fmax(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
+    pub fn vector_maximum_number_nsz(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
         self.vector_extremum(a, b, ExtremumOperation::Max)
     }
 

compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

@@ -23,11 +23,11 @@ use rustc_codegen_ssa::traits::{
     IntrinsicCallBuilderMethods, LayoutTypeCodegenMethods,
 };
 use rustc_data_structures::fx::FxHashSet;
-use rustc_middle::bug;
 #[cfg(feature = "master")]
 use rustc_middle::ty::layout::FnAbiOf;
 use rustc_middle::ty::layout::LayoutOf;
 use rustc_middle::ty::{self, Instance, Ty};
+use rustc_middle::{bug, span_bug};
 use rustc_span::{Span, Symbol, sym};
 use rustc_target::callconv::{ArgAbi, PassMode};
 
@@ -70,8 +70,6 @@ fn get_simple_intrinsic<'gcc, 'tcx>(
         // FIXME: calling `fma` from libc without FMA target feature uses expensive software emulation
         sym::fmuladdf32 => "fmaf", // FIXME: use gcc intrinsic analogous to llvm.fmuladd.f32
         sym::fmuladdf64 => "fma",  // FIXME: use gcc intrinsic analogous to llvm.fmuladd.f64
-        sym::fabsf32 => "fabsf",
-        sym::fabsf64 => "fabs",
         sym::minimumf32 => "fminimumf",
         sym::minimumf64 => "fminimum",
         sym::minimumf128 => {
@@ -194,58 +192,29 @@ fn get_simple_function<'gcc, 'tcx>(
 }
 
 fn get_simple_function_f128<'gcc, 'tcx>(
+    span: Span,
     cx: &CodegenCx<'gcc, 'tcx>,
     name: Symbol,
-) -> Option<Function<'gcc>> {
-    if !cx.supports_f128_type {
-        return None;
-    }
-
+) -> Function<'gcc> {
     let f128_type = cx.type_f128();
     let func_name = match name {
         sym::ceilf128 => "ceilf128",
-        sym::fabsf128 => "fabsf128",
+        sym::fabs => "fabsf128",
         sym::floorf128 => "floorf128",
         sym::truncf128 => "truncf128",
         sym::roundf128 => "roundf128",
         sym::round_ties_even_f128 => "roundevenf128",
         sym::sqrtf128 => "sqrtf128",
-        _ => return None,
+        _ => span_bug!(span, "used get_simple_function_f128 for non-unary f128 intrinsic"),
     };
-    Some(cx.context.new_function(
+    cx.context.new_function(
         None,
         FunctionType::Extern,
         f128_type,
         &[cx.context.new_parameter(None, f128_type, "a")],
         func_name,
         false,
-    ))
-}
-
-fn get_simple_function_f128_2args<'gcc, 'tcx>(
-    cx: &CodegenCx<'gcc, 'tcx>,
-    name: Symbol,
-) -> Option<Function<'gcc>> {
-    if !cx.supports_f128_type {
-        return None;
-    }
-
-    let f128_type = cx.type_f128();
-    let func_name = match name {
-        sym::copysignf128 => "copysignf128",
-        _ => return None,
-    };
-    Some(cx.context.new_function(
-        None,
-        FunctionType::Extern,
-        f128_type,
-        &[
-            cx.context.new_parameter(None, f128_type, "a"),
-            cx.context.new_parameter(None, f128_type, "b"),
-        ],
-        func_name,
-        false,
-    ))
+    )
 }
 
 fn f16_builtin<'gcc, 'tcx>(
@@ -257,7 +226,7 @@ fn f16_builtin<'gcc, 'tcx>(
     let builtin_name = match name {
         sym::ceilf16 => "__builtin_ceilf",
         sym::copysignf16 => "__builtin_copysignf",
-        sym::fabsf16 => "fabsf",
+        sym::fabs => "fabsf",
         sym::floorf16 => "__builtin_floorf",
         sym::fmaf16 => "fmaf",
         sym::powf16 => "__builtin_powf",
@@ -297,11 +266,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
         let fn_args = instance.args;
 
         let simple = get_simple_intrinsic(self, name);
-        // FIXME(antoyo): Only call get_simple_function_f128 and get_simple_function_f128_2args when
-        // it is the symbols for the supported f128 builtins.
-        let simple_func = get_simple_function(self, name)
-            .or_else(|| get_simple_function_f128(self, name))
-            .or_else(|| get_simple_function_f128_2args(self, name));
+        let simple_func = get_simple_function(self, name);
 
         let value = match name {
             _ if simple.is_some() => {
@@ -322,7 +287,6 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
             }
             sym::ceilf16
             | sym::copysignf16
-            | sym::fabsf16
             | sym::floorf16
             | sym::fmaf16
             | sym::powf16
@@ -331,6 +295,40 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
             | sym::round_ties_even_f16
             | sym::sqrtf16
             | sym::truncf16 => f16_builtin(self, name, args),
+            sym::ceilf128
+            | sym::floorf128
+            | sym::truncf128
+            | sym::roundf128
+            | sym::round_ties_even_f128
+            | sym::sqrtf128
+                if self.cx.supports_f128_type =>
+            {
+                let func = get_simple_function_f128(span, self, name);
+                self.cx.context.new_call(
+                    self.location,
+                    func,
+                    &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
+                )
+            }
+            sym::copysignf128 if self.cx.supports_f128_type => {
+                let f128_type = self.cx.type_f128();
+                let func = self.cx.context.new_function(
+                    None,
+                    FunctionType::Extern,
+                    f128_type,
+                    &[
+                        self.cx.context.new_parameter(None, f128_type, "a"),
+                        self.cx.context.new_parameter(None, f128_type, "b"),
+                    ],
+                    "copysignf128",
+                    false,
+                );
+                self.cx.context.new_call(
+                    self.location,
+                    func,
+                    &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
+                )
+            }
             sym::fmaf128 => {
                 let f128_type = self.cx.type_f128();
                 let func = self.cx.context.new_function(
@@ -488,6 +486,23 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
                     }
                 }
             }
+            sym::fabs => 'fabs: {
+                let ty = args[0].layout.ty;
+                let ty::Float(float_ty) = *ty.kind() else {
+                    span_bug!(span, "expected float type for fabs intrinsic: {:?}", ty);
+                };
+                let func = match float_ty {
+                    ty::FloatTy::F16 => break 'fabs f16_builtin(self, name, args),
+                    ty::FloatTy::F32 => self.context.get_builtin_function("fabsf"),
+                    ty::FloatTy::F64 => self.context.get_builtin_function("fabs"),
+                    ty::FloatTy::F128 => get_simple_function_f128(span, self, name),
+                };
+                self.cx.context.new_call(
+                    self.location,
+                    func,
+                    &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
+                )
+            }
 
             sym::raw_eq => {
                 use rustc_abi::BackendRepr::*;

compiler/rustc_codegen_gcc/src/intrinsic/simd.rs

@@ -811,7 +811,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
             }};
         }
         let ty::Float(ref f) = *in_elem.kind() else {
-            return_error!(InvalidMonomorphization::FloatingPointType { span, name, in_ty });
+            return_error!(InvalidMonomorphization::BasicFloatType { span, name, ty: in_ty });
         };
         let elem_ty = bx.cx.type_float_from_ty(*f);
         let (elem_ty_str, elem_ty, cast_type) = match f.bit_width() {
@@ -1222,8 +1222,8 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         simd_and: Uint, Int => and;
         simd_or: Uint, Int => or; // FIXME(antoyo): calling `or` might not work on vectors.
         simd_xor: Uint, Int => xor;
-        simd_fmin: Float => vector_fmin;
-        simd_fmax: Float => vector_fmax;
+        simd_minimum_number_nsz: Float => vector_minimum_number_nsz;
+        simd_maximum_number_nsz: Float => vector_maximum_number_nsz;
     }
 
     macro_rules! arith_unary {

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1605,12 +1605,16 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         *self = Self::build(self.cx, next_bb);
     }
 
-    pub(crate) fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
-        self.call_intrinsic("llvm.minnum", &[self.val_ty(lhs)], &[lhs, rhs])
+    pub(crate) fn minimum_number_nsz(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
+        let call = self.call_intrinsic("llvm.minimumnum", &[self.val_ty(lhs)], &[lhs, rhs]);
+        unsafe { llvm::LLVMRustSetNoSignedZeros(call) };
+        call
     }
 
-    pub(crate) fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
-        self.call_intrinsic("llvm.maxnum", &[self.val_ty(lhs)], &[lhs, rhs])
+    pub(crate) fn maximum_number_nsz(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
+        let call = self.call_intrinsic("llvm.maximumnum", &[self.val_ty(lhs)], &[lhs, rhs]);
+        unsafe { llvm::LLVMRustSetNoSignedZeros(call) };
+        call
     }
 
     pub(crate) fn insert_element(