Integrate Arm SVE types and intrinsics into stdarch

compiler/rustc_abi/src/callconv.rs

@@ -82,6 +82,8 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
                 }))
             }
 
+            BackendRepr::ScalableVector { .. } => Err(Heterogeneous),
+
             BackendRepr::ScalarPair(..) | BackendRepr::Memory { sized: true } => {
                 // Helper for computing `homogeneous_aggregate`, allowing a custom
                 // starting offset (used below for handling variants).

compiler/rustc_abi/src/layout.rs

@@ -11,7 +11,7 @@ use tracing::{debug, trace};
 use crate::{
     AbiAlign, Align, BackendRepr, FieldsShape, HasDataLayout, IndexSlice, IndexVec, Integer,
     LayoutData, Niche, NonZeroUsize, Primitive, ReprOptions, Scalar, Size, StructKind, TagEncoding,
-    Variants, WrappingRange,
+    TargetDataLayout, Variants, WrappingRange,
 };
 
 mod coroutine;
@@ -143,58 +143,32 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
         })
     }
 
-    pub fn simd_type<
+    pub fn scalable_vector_type<FieldIdx, VariantIdx, F>(
+        &self,
+        element: F,
+        count: u64,
+    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
+    where
         FieldIdx: Idx,
         VariantIdx: Idx,
         F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
-    >(
+    {
+        vector_type_layout(VectorKind::Scalable, self.cx.data_layout(), element, count)
+    }
+
+    pub fn simd_type<FieldIdx, VariantIdx, F>(
         &self,
         element: F,
         count: u64,
         repr_packed: bool,
-    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {
-        let elt = element.as_ref();
-        if count == 0 {
-            return Err(LayoutCalculatorError::ZeroLengthSimdType);
-        } else if count > crate::MAX_SIMD_LANES {
-            return Err(LayoutCalculatorError::OversizedSimdType {
-                max_lanes: crate::MAX_SIMD_LANES,
-            });
-        }
-
-        let BackendRepr::Scalar(e_repr) = elt.backend_repr else {
-            return Err(LayoutCalculatorError::NonPrimitiveSimdType(element));
-        };
-
-        // Compute the size and alignment of the vector
-        let dl = self.cx.data_layout();
-        let size =
-            elt.size.checked_mul(count, dl).ok_or_else(|| LayoutCalculatorError::SizeOverflow)?;
-        let (repr, align) = if repr_packed && !count.is_power_of_two() {
-            // Non-power-of-two vectors have padding up to the next power-of-two.
-            // If we're a packed repr, remove the padding while keeping the alignment as close
-            // to a vector as possible.
-            (BackendRepr::Memory { sized: true }, AbiAlign { abi: Align::max_aligned_factor(size) })
-        } else {
-            (BackendRepr::SimdVector { element: e_repr, count }, dl.llvmlike_vector_align(size))
-        };
-        let size = size.align_to(align.abi);
-
-        Ok(LayoutData {
-            variants: Variants::Single { index: VariantIdx::new(0) },
-            fields: FieldsShape::Arbitrary {
-                offsets: [Size::ZERO].into(),
-                memory_index: [0].into(),
-            },
-            backend_repr: repr,
-            largest_niche: elt.largest_niche,
-            uninhabited: false,
-            size,
-            align,
-            max_repr_align: None,
-            unadjusted_abi_align: elt.align.abi,
-            randomization_seed: elt.randomization_seed.wrapping_add(Hash64::new(count)),
-        })
+    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
+    where
+        FieldIdx: Idx,
+        VariantIdx: Idx,
+        F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
+    {
+        let kind = if repr_packed { VectorKind::PackedFixed } else { VectorKind::Fixed };
+        vector_type_layout(kind, self.cx.data_layout(), element, count)
     }
 
     /// Compute the layout for a coroutine.
@@ -455,6 +429,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
                 BackendRepr::Scalar(..)
                 | BackendRepr::ScalarPair(..)
                 | BackendRepr::SimdVector { .. }
+                | BackendRepr::ScalableVector { .. }
                 | BackendRepr::Memory { .. } => repr,
             },
         };
@@ -526,7 +501,8 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
                     hide_niches(a);
                     hide_niches(b);
                 }
-                BackendRepr::SimdVector { element, count: _ } => hide_niches(element),
+                BackendRepr::SimdVector { element, .. }
+                | BackendRepr::ScalableVector { element, .. } => hide_niches(element),
                 BackendRepr::Memory { sized: _ } => {}
             }
             st.largest_niche = None;
@@ -1525,3 +1501,67 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
         s
     }
 }
+
+enum VectorKind {
+    /// `#[rustc_scalable_vector]`
+    Scalable,
+    /// `#[repr(simd, packed)]`
+    PackedFixed,
+    /// `#[repr(simd)]`
+    Fixed,
+}
+
+fn vector_type_layout<FieldIdx, VariantIdx, F>(
+    kind: VectorKind,
+    dl: &TargetDataLayout,
+    element: F,
+    count: u64,
+) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
+where
+    FieldIdx: Idx,
+    VariantIdx: Idx,
+    F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
+{
+    let elt = element.as_ref();
+    if count == 0 {
+        return Err(LayoutCalculatorError::ZeroLengthSimdType);
+    } else if count > crate::MAX_SIMD_LANES {
+        return Err(LayoutCalculatorError::OversizedSimdType { max_lanes: crate::MAX_SIMD_LANES });
+    }
+
+    let BackendRepr::Scalar(element) = elt.backend_repr else {
+        return Err(LayoutCalculatorError::NonPrimitiveSimdType(element));
+    };
+
+    // Compute the size and alignment of the vector
+    let size =
+        elt.size.checked_mul(count, dl).ok_or_else(|| LayoutCalculatorError::SizeOverflow)?;
+    let (repr, align) = match kind {
+        VectorKind::Scalable => {
+            (BackendRepr::ScalableVector { element, count }, dl.llvmlike_vector_align(size))
+        }
+        // Non-power-of-two vectors have padding up to the next power-of-two.
+        // If we're a packed repr, remove the padding while keeping the alignment as close
+        // to a vector as possible.
+        VectorKind::PackedFixed if !count.is_power_of_two() => {
+            (BackendRepr::Memory { sized: true }, AbiAlign { abi: Align::max_aligned_factor(size) })
+        }
+        VectorKind::PackedFixed | VectorKind::Fixed => {
+            (BackendRepr::SimdVector { element, count }, dl.llvmlike_vector_align(size))
+        }
+    };
+    let size = size.align_to(align.abi);
+
+    Ok(LayoutData {
+        variants: Variants::Single { index: VariantIdx::new(0) },
+        fields: FieldsShape::Arbitrary { offsets: [Size::ZERO].into(), memory_index: [0].into() },
+        backend_repr: repr,
+        largest_niche: elt.largest_niche,
+        uninhabited: false,
+        size,
+        align,
+        max_repr_align: None,
+        unadjusted_abi_align: elt.align.abi,
+        randomization_seed: elt.randomization_seed.wrapping_add(Hash64::new(count)),
+    })
+}

compiler/rustc_abi/src/lib.rs

@@ -93,9 +93,11 @@ bitflags! {
         // Other flags can still inhibit reordering and thus randomization.
         // The seed stored in `ReprOptions.field_shuffle_seed`.
         const RANDOMIZE_LAYOUT   = 1 << 4;
+        const IS_SCALABLE        = 1 << 5;
         // Any of these flags being set prevent field reordering optimisation.
         const FIELD_ORDER_UNOPTIMIZABLE   = ReprFlags::IS_C.bits()
                                  | ReprFlags::IS_SIMD.bits()
+                                 | ReprFlags::IS_SCALABLE.bits()
                                  | ReprFlags::IS_LINEAR.bits();
         const ABI_UNOPTIMIZABLE = ReprFlags::IS_C.bits() | ReprFlags::IS_SIMD.bits();
     }
@@ -132,6 +134,19 @@ impl IntegerType {
     }
 }
 
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(
+    feature = "nightly",
+    derive(Encodable_NoContext, Decodable_NoContext, HashStable_Generic)
+)]
+pub enum ScalableElt {
+    /// `N` in `rustc_scalable_vector(N)` - the element count of the scalable vector
+    ElementCount(u128),
+    /// `rustc_scalable_vector` w/out `N`, used for tuple types of scalable vectors that only
+    /// contain other scalable vectors
+    Container,
+}
+
 /// Represents the repr options provided by the user.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, Default)]
 #[cfg_attr(
@@ -143,6 +158,8 @@ pub struct ReprOptions {
     pub align: Option<Align>,
     pub pack: Option<Align>,
     pub flags: ReprFlags,
+    /// `#[rustc_scalable_vector]`
+    pub scalable: Option<ScalableElt>,
     /// The seed to be used for randomizing a type's layout
     ///
     /// Note: This could technically be a `u128` which would
@@ -159,6 +176,11 @@ impl ReprOptions {
         self.flags.contains(ReprFlags::IS_SIMD)
     }
 
+    #[inline]
+    pub fn scalable(&self) -> bool {
+        self.flags.contains(ReprFlags::IS_SCALABLE)
+    }
+
     #[inline]
     pub fn c(&self) -> bool {
         self.flags.contains(ReprFlags::IS_C)
@@ -1731,6 +1753,10 @@ impl AddressSpace {
 pub enum BackendRepr {
     Scalar(Scalar),
     ScalarPair(Scalar, Scalar),
+    ScalableVector {
+        element: Scalar,
+        count: u64,
+    },
     SimdVector {
         element: Scalar,
         count: u64,
@@ -1749,6 +1775,9 @@ impl BackendRepr {
         match *self {
             BackendRepr::Scalar(_)
             | BackendRepr::ScalarPair(..)
+            // FIXME(repr_scalable): Scalable vectors are `Sized` while the `sized_hierarchy`
+            // feature is not yet fully implemented
+            | BackendRepr::ScalableVector { .. }
             | BackendRepr::SimdVector { .. } => false,
             BackendRepr::Memory { sized } => !sized,
         }
@@ -1789,7 +1818,9 @@ impl BackendRepr {
             BackendRepr::Scalar(s) => Some(s.align(cx).abi),
             BackendRepr::ScalarPair(s1, s2) => Some(s1.align(cx).max(s2.align(cx)).abi),
             // The align of a Vector can vary in surprising ways
-            BackendRepr::SimdVector { .. } | BackendRepr::Memory { .. } => None,
+            BackendRepr::SimdVector { .. }
+            | BackendRepr::Memory { .. }
+            | BackendRepr::ScalableVector { .. } => None,
         }
     }
 
@@ -1811,7 +1842,9 @@ impl BackendRepr {
                 Some(size)
             }
             // The size of a Vector can vary in surprising ways
-            BackendRepr::SimdVector { .. } | BackendRepr::Memory { .. } => None,
+            BackendRepr::SimdVector { .. }
+            | BackendRepr::Memory { .. }
+            | BackendRepr::ScalableVector { .. } => None,
         }
     }
 
@@ -1826,6 +1859,9 @@ impl BackendRepr {
                 BackendRepr::SimdVector { element: element.to_union(), count }
             }
             BackendRepr::Memory { .. } => BackendRepr::Memory { sized: true },
+            BackendRepr::ScalableVector { element, count } => {
+                BackendRepr::ScalableVector { element: element.to_union(), count }
+            }
         }
     }
 
@@ -2066,7 +2102,9 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
     /// Returns `true` if this is an aggregate type (including a ScalarPair!)
     pub fn is_aggregate(&self) -> bool {
         match self.backend_repr {
-            BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } => false,
+            BackendRepr::Scalar(_)
+            | BackendRepr::SimdVector { .. }
+            | BackendRepr::ScalableVector { .. } => false,
             BackendRepr::ScalarPair(..) | BackendRepr::Memory { .. } => true,
         }
     }
@@ -2160,6 +2198,19 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
         self.is_sized() && self.size.bytes() == 0 && self.align.abi.bytes() == 1
     }
 
+    /// Returns `true` if the size of the type is only known at runtime.
+    pub fn is_runtime_sized(&self) -> bool {
+        matches!(self.backend_repr, BackendRepr::ScalableVector { .. })
+    }
+
+    /// Returns the elements count of a scalable vector.
+    pub fn scalable_vector_element_count(&self) -> Option<u64> {
+        match self.backend_repr {
+            BackendRepr::ScalableVector { count, .. } => Some(count),
+            _ => None,
+        }
+    }
+
     /// Returns `true` if the type is a ZST and not unsized.
     ///
     /// Note that this does *not* imply that the type is irrelevant for layout! It can still have
@@ -2168,6 +2219,7 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
         match self.backend_repr {
             BackendRepr::Scalar(_)
             | BackendRepr::ScalarPair(..)
+            | BackendRepr::ScalableVector { .. }
             | BackendRepr::SimdVector { .. } => false,
             BackendRepr::Memory { sized } => sized && self.size.bytes() == 0,
         }

compiler/rustc_ast_passes/messages.ftl

@@ -228,6 +228,8 @@ ast_passes_precise_capturing_duplicated = duplicate `use<...>` precise capturing
 
 ast_passes_precise_capturing_not_allowed_here = `use<...>` precise capturing syntax not allowed in {$loc}
 
+ast_passes_scalable_vector_not_tuple_struct = scalable vectors must be tuple structs
+
 ast_passes_static_without_body =
     free static item without body
     .suggestion = provide a definition for the static

compiler/rustc_ast_passes/src/ast_validation.rs

@@ -1182,6 +1182,14 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
             }
             ItemKind::Struct(ident, generics, vdata) => {
                 self.with_tilde_const(Some(TildeConstReason::Struct { span: item.span }), |this| {
+                    // Scalable vectors can only be tuple structs
+                    let is_scalable_vector =
+                        item.attrs.iter().any(|attr| attr.has_name(sym::rustc_scalable_vector));
+                    if is_scalable_vector && !matches!(vdata, VariantData::Tuple(..)) {
+                        this.dcx()
+                            .emit_err(errors::ScalableVectorNotTupleStruct { span: item.span });
+                    }
+
                     match vdata {
                         VariantData::Struct { fields, .. } => {
                             this.visit_attrs_vis_ident(&item.attrs, &item.vis, ident);

compiler/rustc_ast_passes/src/errors.rs

@@ -907,3 +907,10 @@ pub(crate) struct AbiMustNotHaveReturnType {
     pub span: Span,
     pub abi: ExternAbi,
 }
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_scalable_vector_not_tuple_struct)]
+pub(crate) struct ScalableVectorNotTupleStruct {
+    #[primary_span]
+    pub span: Span,
+}

compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs

@@ -51,3 +51,26 @@ impl<S: Stage> SingleAttributeParser<S> for RustcObjectLifetimeDefaultParser {
         Some(AttributeKind::RustcObjectLifetimeDefault)
     }
 }
+
+pub(crate) struct RustcScalableVectorParser;
+
+impl<S: Stage> SingleAttributeParser<S> for RustcScalableVectorParser {
+    const PATH: &[rustc_span::Symbol] = &[sym::rustc_scalable_vector];
+    const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost;
+    const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::Error;
+    const TEMPLATE: AttributeTemplate = template!(Word, List: "count");
+
+    fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option<AttributeKind> {
+        if args.no_args().is_ok() {
+            return Some(AttributeKind::RustcScalableVector {
+                element_count: None,
+                span: cx.attr_span,
+            });
+        }
+
+        parse_single_integer(cx, args).map(|n| AttributeKind::RustcScalableVector {
+            element_count: Some(n),
+            span: cx.attr_span,
+        })
+    }
+}