Skip to content

Commit a05dd71

Browse files
davidtwcoJamieCunliffe
davidtwco
and
JamieCunliffe
committed
codegen: implement repr(scalable)
Introduces `BackendRepr::ScalableVector` corresponding to scalable vector types annotated with `repr(scalable)` which lowers to a scalable vector type in LLVM. Co-authored-by: Jamie Cunliffe <Jamie.Cunliffe@arm.com>
1 parent 7f6f8c8 commit a05dd71

File tree

41 files changed

+553
-74
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+553
-74
lines changed

‎compiler/rustc_abi/src/callconv.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
8282
}))
8383
}
8484

85+
BackendRepr::ScalableVector { .. } => Err(Heterogeneous),
86+
8587
BackendRepr::ScalarPair(..) | BackendRepr::Memory { sized: true } => {
8688
// Helper for computing `homogeneous_aggregate`, allowing a custom
8789
// starting offset (used below for handling variants).

‎compiler/rustc_abi/src/layout.rs‎

Lines changed: 87 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ use tracing::{debug, trace};
1111
use crate::{
1212
AbiAlign, Align, BackendRepr, FieldsShape, HasDataLayout, IndexSlice, IndexVec, Integer,
1313
LayoutData, Niche, NonZeroUsize, Primitive, ReprOptions, Scalar, Size, StructKind, TagEncoding,
14-
Variants, WrappingRange,
14+
TargetDataLayout, Variants, WrappingRange,
1515
};
1616

1717
mod coroutine;
@@ -143,58 +143,32 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
143143
})
144144
}
145145

146-
pub fn simd_type<
146+
pub fn scalable_vector_type<FieldIdx, VariantIdx, F>(
147+
&self,
148+
element: F,
149+
count: u64,
150+
) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
151+
where
147152
FieldIdx: Idx,
148153
VariantIdx: Idx,
149154
F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
150-
>(
155+
{
156+
vector_type_layout(VectorKind::Scalable, self.cx.data_layout(), element, count)
157+
}
158+
159+
pub fn simd_type<FieldIdx, VariantIdx, F>(
151160
&self,
152161
element: F,
153162
count: u64,
154163
repr_packed: bool,
155-
) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {
156-
let elt = element.as_ref();
157-
if count == 0 {
158-
return Err(LayoutCalculatorError::ZeroLengthSimdType);
159-
} else if count > crate::MAX_SIMD_LANES {
160-
return Err(LayoutCalculatorError::OversizedSimdType {
161-
max_lanes: crate::MAX_SIMD_LANES,
162-
});
163-
}
164-
165-
let BackendRepr::Scalar(e_repr) = elt.backend_repr else {
166-
return Err(LayoutCalculatorError::NonPrimitiveSimdType(element));
167-
};
168-
169-
// Compute the size and alignment of the vector
170-
let dl = self.cx.data_layout();
171-
let size =
172-
elt.size.checked_mul(count, dl).ok_or_else(|| LayoutCalculatorError::SizeOverflow)?;
173-
let (repr, align) = if repr_packed && !count.is_power_of_two() {
174-
// Non-power-of-two vectors have padding up to the next power-of-two.
175-
// If we're a packed repr, remove the padding while keeping the alignment as close
176-
// to a vector as possible.
177-
(BackendRepr::Memory { sized: true }, Align::max_aligned_factor(size))
178-
} else {
179-
(BackendRepr::SimdVector { element: e_repr, count }, dl.llvmlike_vector_align(size))
180-
};
181-
let size = size.align_to(align);
182-
183-
Ok(LayoutData {
184-
variants: Variants::Single { index: VariantIdx::new(0) },
185-
fields: FieldsShape::Arbitrary {
186-
offsets: [Size::ZERO].into(),
187-
memory_index: [0].into(),
188-
},
189-
backend_repr: repr,
190-
largest_niche: elt.largest_niche,
191-
uninhabited: false,
192-
size,
193-
align: AbiAlign::new(align),
194-
max_repr_align: None,
195-
unadjusted_abi_align: elt.align.abi,
196-
randomization_seed: elt.randomization_seed.wrapping_add(Hash64::new(count)),
197-
})
164+
) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
165+
where
166+
FieldIdx: Idx,
167+
VariantIdx: Idx,
168+
F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
169+
{
170+
let kind = if repr_packed { VectorKind::PackedFixed } else { VectorKind::Fixed };
171+
vector_type_layout(kind, self.cx.data_layout(), element, count)
198172
}
199173

200174
/// Compute the layout for a coroutine.
@@ -453,6 +427,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
453427
BackendRepr::Scalar(..)
454428
| BackendRepr::ScalarPair(..)
455429
| BackendRepr::SimdVector { .. }
430+
| BackendRepr::ScalableVector { .. }
456431
| BackendRepr::Memory { .. } => repr,
457432
},
458433
};
@@ -524,7 +499,8 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
524499
hide_niches(a);
525500
hide_niches(b);
526501
}
527-
BackendRepr::SimdVector { element, count: _ } => hide_niches(element),
502+
BackendRepr::SimdVector { element, .. }
503+
| BackendRepr::ScalableVector { element, .. } => hide_niches(element),
528504
BackendRepr::Memory { sized: _ } => {}
529505
}
530506
st.largest_niche = None;
@@ -1501,3 +1477,67 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
15011477
s
15021478
}
15031479
}
1480+
1481+
enum VectorKind {
1482+
/// `#[rustc_scalable_vector]`
1483+
Scalable,
1484+
/// `#[repr(simd, packed)]`
1485+
PackedFixed,
1486+
/// `#[repr(simd)]`
1487+
Fixed,
1488+
}
1489+
1490+
fn vector_type_layout<FieldIdx, VariantIdx, F>(
1491+
kind: VectorKind,
1492+
dl: &TargetDataLayout,
1493+
element: F,
1494+
count: u64,
1495+
) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
1496+
where
1497+
FieldIdx: Idx,
1498+
VariantIdx: Idx,
1499+
F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
1500+
{
1501+
let elt = element.as_ref();
1502+
if count == 0 {
1503+
return Err(LayoutCalculatorError::ZeroLengthSimdType);
1504+
} else if count > crate::MAX_SIMD_LANES {
1505+
return Err(LayoutCalculatorError::OversizedSimdType { max_lanes: crate::MAX_SIMD_LANES });
1506+
}
1507+
1508+
let BackendRepr::Scalar(element) = elt.backend_repr else {
1509+
return Err(LayoutCalculatorError::NonPrimitiveSimdType(element));
1510+
};
1511+
1512+
// Compute the size and alignment of the vector
1513+
let size =
1514+
elt.size.checked_mul(count, dl).ok_or_else(|| LayoutCalculatorError::SizeOverflow)?;
1515+
let (repr, align) = match kind {
1516+
VectorKind::Scalable => {
1517+
(BackendRepr::ScalableVector { element, count }, dl.llvmlike_vector_align(size))
1518+
}
1519+
// Non-power-of-two vectors have padding up to the next power-of-two.
1520+
// If we're a packed repr, remove the padding while keeping the alignment as close
1521+
// to a vector as possible.
1522+
VectorKind::PackedFixed if !count.is_power_of_two() => {
1523+
(BackendRepr::Memory { sized: true }, Align::max_aligned_factor(size))
1524+
}
1525+
VectorKind::PackedFixed | VectorKind::Fixed => {
1526+
(BackendRepr::SimdVector { element, count }, dl.llvmlike_vector_align(size))
1527+
}
1528+
};
1529+
let size = size.align_to(align);
1530+
1531+
Ok(LayoutData {
1532+
variants: Variants::Single { index: VariantIdx::new(0) },
1533+
fields: FieldsShape::Arbitrary { offsets: [Size::ZERO].into(), memory_index: [0].into() },
1534+
backend_repr: repr,
1535+
largest_niche: elt.largest_niche,
1536+
uninhabited: false,
1537+
size,
1538+
align: AbiAlign::new(align),
1539+
max_repr_align: None,
1540+
unadjusted_abi_align: elt.align.abi,
1541+
randomization_seed: elt.randomization_seed.wrapping_add(Hash64::new(count)),
1542+
})
1543+
}

‎compiler/rustc_abi/src/lib.rs‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1758,6 +1758,10 @@ impl AddressSpace {
17581758
pub enum BackendRepr {
17591759
Scalar(Scalar),
17601760
ScalarPair(Scalar, Scalar),
1761+
ScalableVector {
1762+
element: Scalar,
1763+
count: u64,
1764+
},
17611765
SimdVector {
17621766
element: Scalar,
17631767
count: u64,
@@ -1776,6 +1780,9 @@ impl BackendRepr {
17761780
match *self {
17771781
BackendRepr::Scalar(_)
17781782
| BackendRepr::ScalarPair(..)
1783+
// FIXME(repr_scalable): Scalable vectors are `Sized` while the `sized_hierarchy`
1784+
// feature is not yet fully implemented
1785+
| BackendRepr::ScalableVector { .. }
17791786
| BackendRepr::SimdVector { .. } => false,
17801787
BackendRepr::Memory { sized } => !sized,
17811788
}
@@ -1816,7 +1823,9 @@ impl BackendRepr {
18161823
BackendRepr::Scalar(s) => Some(s.align(cx).abi),
18171824
BackendRepr::ScalarPair(s1, s2) => Some(s1.align(cx).max(s2.align(cx)).abi),
18181825
// The align of a Vector can vary in surprising ways
1819-
BackendRepr::SimdVector { .. } | BackendRepr::Memory { .. } => None,
1826+
BackendRepr::SimdVector { .. }
1827+
| BackendRepr::Memory { .. }
1828+
| BackendRepr::ScalableVector { .. } => None,
18201829
}
18211830
}
18221831

@@ -1838,7 +1847,9 @@ impl BackendRepr {
18381847
Some(size)
18391848
}
18401849
// The size of a Vector can vary in surprising ways
1841-
BackendRepr::SimdVector { .. } | BackendRepr::Memory { .. } => None,
1850+
BackendRepr::SimdVector { .. }
1851+
| BackendRepr::Memory { .. }
1852+
| BackendRepr::ScalableVector { .. } => None,
18421853
}
18431854
}
18441855

@@ -1853,6 +1864,9 @@ impl BackendRepr {
18531864
BackendRepr::SimdVector { element: element.to_union(), count }
18541865
}
18551866
BackendRepr::Memory { .. } => BackendRepr::Memory { sized: true },
1867+
BackendRepr::ScalableVector { element, count } => {
1868+
BackendRepr::ScalableVector { element: element.to_union(), count }
1869+
}
18561870
}
18571871
}
18581872

@@ -2093,7 +2107,9 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
20932107
/// Returns `true` if this is an aggregate type (including a ScalarPair!)
20942108
pub fn is_aggregate(&self) -> bool {
20952109
match self.backend_repr {
2096-
BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } => false,
2110+
BackendRepr::Scalar(_)
2111+
| BackendRepr::SimdVector { .. }
2112+
| BackendRepr::ScalableVector { .. } => false,
20972113
BackendRepr::ScalarPair(..) | BackendRepr::Memory { .. } => true,
20982114
}
20992115
}
@@ -2187,6 +2203,19 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
21872203
self.is_sized() && self.size.bytes() == 0 && self.align.bytes() == 1
21882204
}
21892205

2206+
/// Returns `true` if the size of the type is only known at runtime.
2207+
pub fn is_runtime_sized(&self) -> bool {
2208+
matches!(self.backend_repr, BackendRepr::ScalableVector { .. })
2209+
}
2210+
2211+
/// Returns the elements count of a scalable vector.
2212+
pub fn scalable_vector_element_count(&self) -> Option<u64> {
2213+
match self.backend_repr {
2214+
BackendRepr::ScalableVector { count, .. } => Some(count),
2215+
_ => None,
2216+
}
2217+
}
2218+
21902219
/// Returns `true` if the type is a ZST and not unsized.
21912220
///
21922221
/// Note that this does *not* imply that the type is irrelevant for layout! It can still have
@@ -2195,6 +2224,7 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
21952224
match self.backend_repr {
21962225
BackendRepr::Scalar(_)
21972226
| BackendRepr::ScalarPair(..)
2227+
| BackendRepr::ScalableVector { .. }
21982228
| BackendRepr::SimdVector { .. } => false,
21992229
BackendRepr::Memory { sized } => sized && self.size.bytes() == 0,
22002230
}

‎compiler/rustc_codegen_gcc/src/builder.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -943,6 +943,10 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
943943
.get_address(self.location)
944944
}
945945

946+
fn scalable_alloca(&mut self, _elt: u64, _align: Align, _element_ty: Ty<'_>) -> RValue<'gcc> {
947+
todo!()
948+
}
949+
946950
fn load(&mut self, pointee_ty: Type<'gcc>, ptr: RValue<'gcc>, align: Align) -> RValue<'gcc> {
947951
let block = self.llbb();
948952
let function = block.get_function();

‎compiler/rustc_codegen_gcc/src/intrinsic/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -531,7 +531,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
531531
let layout = self.layout_of(tp_ty).layout;
532532
let _use_integer_compare = match layout.backend_repr() {
533533
Scalar(_) | ScalarPair(_, _) => true,
534-
SimdVector { .. } => false,
534+
SimdVector { .. } | ScalableVector { .. } => false,
535535
Memory { .. } => {
536536
// For rusty ABIs, small aggregates are actually passed
537537
// as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),

‎compiler/rustc_codegen_gcc/src/type_of.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ fn uncached_gcc_type<'gcc, 'tcx>(
8585
);
8686
}
8787
BackendRepr::Memory { .. } => {}
88+
BackendRepr::ScalableVector { .. } => todo!(),
8889
}
8990

9091
let name = match *layout.ty.kind() {
@@ -178,7 +179,9 @@ pub trait LayoutGccExt<'tcx> {
178179
impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
179180
fn is_gcc_immediate(&self) -> bool {
180181
match self.backend_repr {
181-
BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } => true,
182+
BackendRepr::Scalar(_)
183+
| BackendRepr::SimdVector { .. }
184+
| BackendRepr::ScalableVector { .. } => true,
182185
BackendRepr::ScalarPair(..) | BackendRepr::Memory { .. } => false,
183186
}
184187
}
@@ -188,6 +191,7 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
188191
BackendRepr::ScalarPair(..) => true,
189192
BackendRepr::Scalar(_)
190193
| BackendRepr::SimdVector { .. }
194+
| BackendRepr::ScalableVector { .. }
191195
| BackendRepr::Memory { .. } => false,
192196
}
193197
}

‎compiler/rustc_codegen_llvm/src/builder.rs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -613,6 +613,25 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
613613
}
614614
}
615615

616+
fn scalable_alloca(&mut self, elt: u64, align: Align, element_ty: Ty<'_>) -> Self::Value {
617+
let mut bx = Builder::with_cx(self.cx);
618+
bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
619+
let llvm_ty = match element_ty.kind() {
620+
ty::Bool => bx.type_i1(),
621+
ty::Int(int_ty) => self.cx.type_int_from_ty(*int_ty),
622+
ty::Uint(uint_ty) => self.cx.type_uint_from_ty(*uint_ty),
623+
ty::Float(float_ty) => self.cx.type_float_from_ty(*float_ty),
624+
_ => unreachable!("scalable vectors can only contain a bool, int, uint or float"),
625+
};
626+
627+
unsafe {
628+
let ty = llvm::LLVMScalableVectorType(llvm_ty, elt.try_into().unwrap());
629+
let alloca = llvm::LLVMBuildAlloca(&bx.llbuilder, ty, UNNAMED);
630+
llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
631+
alloca
632+
}
633+
}
634+
616635
fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
617636
unsafe {
618637
let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);

0 commit comments

Comments
 (0)