diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 72e7b27a1f97c..369fe12539fa8 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -876,11 +876,15 @@ pub trait AttributeExt: Debug { /// a doc comment) will return `false`. fn is_doc_comment(&self) -> Option; + /// Returns true if the attribute's first *and only* path segment is equal to the passed-in + /// symbol. #[inline] fn has_name(&self, name: Symbol) -> bool { self.name().map(|x| x == name).unwrap_or(false) } + /// Returns true if the attribute's first *and only* path segment is any of the passed-in + /// symbols. #[inline] fn has_any_name(&self, names: &[Symbol]) -> bool { names.iter().any(|&name| self.has_name(name)) @@ -889,6 +893,7 @@ pub trait AttributeExt: Debug { /// get the span of the entire attribute fn span(&self) -> Span; + /// Returns whether the attribute is a path, without any arguments. fn is_word(&self) -> bool; fn path(&self) -> SmallVec<[Symbol; 1]> { @@ -911,11 +916,14 @@ pub trait AttributeExt: Debug { /// * `#[deprecated(note = "note", ...)]` returns `Some("note")`. fn deprecation_note(&self) -> Option; + /// Returns whether this attribute is any of the proc macro attributes. + /// i.e. `proc_macro`, `proc_macro_attribute` or `proc_macro_derive`. fn is_proc_macro_attr(&self) -> bool { [sym::proc_macro, sym::proc_macro_attribute, sym::proc_macro_derive] .iter() .any(|kind| self.has_name(*kind)) } + /// Returns true if this attribute is `#[automatically_deived]`. fn is_automatically_derived_attr(&self) -> bool; /// Returns the documentation and its kind if this is a doc comment or a sugared doc comment. diff --git a/compiler/rustc_codegen_cranelift/patches/0028-stdlib-Ensure-va_end-doesn-t-get-emitted-unless-VaList-is-a.patch b/compiler/rustc_codegen_cranelift/patches/0028-stdlib-Ensure-va_end-doesn-t-get-emitted-unless-VaList-is-a.patch deleted file mode 100644 index 2aa93164674f6..0000000000000 --- a/compiler/rustc_codegen_cranelift/patches/0028-stdlib-Ensure-va_end-doesn-t-get-emitted-unless-VaList-is-a.patch +++ /dev/null @@ -1,25 +0,0 @@ -From 116abc64add4d617104993a7a3011f20bcf31ef2 Mon Sep 17 00:00:00 2001 -From: bjorn3 <17426603+bjorn3@users.noreply.github.com> -Date: Mon, 26 Jan 2026 16:20:58 +0000 -Subject: [PATCH] Ensure va_end doesn't get emitted unless VaList is actually - used - ---- - library/core/src/ffi/va_list.rs | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/library/core/src/ffi/va_list.rs b/library/core/src/ffi/va_list.rs -index d0f1553..75129af 100644 ---- a/library/core/src/ffi/va_list.rs -+++ b/library/core/src/ffi/va_list.rs -@@ -217,6 +217,7 @@ impl Clone for VaList<'_> { - - #[rustc_const_unstable(feature = "const_c_variadic", issue = "151787")] - impl<'f> const Drop for VaList<'f> { -+ #[inline] - fn drop(&mut self) { - // SAFETY: this variable argument list is being dropped, so won't be read from again. - unsafe { va_end(self) } --- -2.43.0 - diff --git a/compiler/rustc_data_structures/src/vec_cache.rs b/compiler/rustc_data_structures/src/vec_cache.rs index aea5924b8802d..6d026bb2c7f74 100644 --- a/compiler/rustc_data_structures/src/vec_cache.rs +++ b/compiler/rustc_data_structures/src/vec_cache.rs @@ -6,12 +6,16 @@ //! //! This is currently used for query caching. -use std::fmt::Debug; +use std::fmt::{self, Debug}; use std::marker::PhantomData; +use std::ops::{Index, IndexMut}; use std::sync::atomic::{AtomicPtr, AtomicU32, AtomicUsize, Ordering}; use rustc_index::Idx; +#[cfg(test)] +mod tests; + struct Slot { // We never construct &Slot so it's fine for this to not be in an UnsafeCell. value: V, @@ -28,7 +32,7 @@ struct Slot { #[derive(Copy, Clone, Debug)] struct SlotIndex { // the index of the bucket in VecCache (0 to 20) - bucket_idx: usize, + bucket_idx: BucketIndex, // the index of the slot within the bucket index_in_bucket: usize, } @@ -42,7 +46,7 @@ const ENTRIES_BY_BUCKET: [usize; BUCKETS] = { let mut key = 0; loop { let si = SlotIndex::from_index(key); - entries[si.bucket_idx] = si.entries(); + entries[si.bucket_idx.to_usize()] = si.bucket_idx.capacity(); if key == 0 { key = 1; } else if key == (1 << 31) { @@ -57,48 +61,24 @@ const ENTRIES_BY_BUCKET: [usize; BUCKETS] = { const BUCKETS: usize = 21; impl SlotIndex { - /// The total possible number of entries in the bucket - const fn entries(&self) -> usize { - if self.bucket_idx == 0 { 1 << 12 } else { 1 << (self.bucket_idx + 11) } - } - - // This unpacks a flat u32 index into identifying which bucket it belongs to and the offset - // within that bucket. As noted in the VecCache docs, buckets double in size with each index. - // Typically that would mean 31 buckets (2^0 + 2^1 ... + 2^31 = u32::MAX - 1), but to reduce - // the size of the VecCache struct and avoid uselessly small allocations, we instead have the - // first bucket have 2**12 entries. To simplify the math, the second bucket also 2**12 entries, - // and buckets double from there. - // - // We assert that [0, 2**32 - 1] uniquely map through this function to individual, consecutive - // slots (see `slot_index_exhaustive` in tests). + /// Unpacks a flat 32-bit index into a [`BucketIndex`] and a slot offset within that bucket. #[inline] const fn from_index(idx: u32) -> Self { - const FIRST_BUCKET_SHIFT: usize = 12; - if idx < (1 << FIRST_BUCKET_SHIFT) { - return SlotIndex { bucket_idx: 0, index_in_bucket: idx as usize }; - } - // We already ruled out idx 0, so this `ilog2` never panics (and the check optimizes away) - let bucket = idx.ilog2() as usize; - let entries = 1 << bucket; - SlotIndex { - bucket_idx: bucket - FIRST_BUCKET_SHIFT + 1, - index_in_bucket: idx as usize - entries, - } + let (bucket_idx, index_in_bucket) = BucketIndex::from_flat_index(idx as usize); + SlotIndex { bucket_idx, index_in_bucket } } // SAFETY: Buckets must be managed solely by functions here (i.e., get/put on SlotIndex) and // `self` comes from SlotIndex::from_index #[inline] unsafe fn get(&self, buckets: &[AtomicPtr>; 21]) -> Option<(V, u32)> { - // SAFETY: `bucket_idx` is ilog2(u32).saturating_sub(11), which is at most 21, i.e., - // in-bounds of buckets. See `from_index` for computation. - let bucket = unsafe { buckets.get_unchecked(self.bucket_idx) }; + let bucket = &buckets[self.bucket_idx]; let ptr = bucket.load(Ordering::Acquire); // Bucket is not yet initialized: then we obviously won't find this entry in that bucket. if ptr.is_null() { return None; } - debug_assert!(self.index_in_bucket < self.entries()); + debug_assert!(self.index_in_bucket < self.bucket_idx.capacity()); // SAFETY: `bucket` was allocated (so <= isize in total bytes) to hold `entries`, so this // must be inbounds. let slot = unsafe { ptr.add(self.index_in_bucket) }; @@ -131,7 +111,7 @@ impl SlotIndex { #[cold] #[inline(never)] - fn initialize_bucket(bucket: &AtomicPtr>, bucket_idx: usize) -> *mut Slot { + fn initialize_bucket(bucket: &AtomicPtr>, bucket_idx: BucketIndex) -> *mut Slot { static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); // If we are initializing the bucket, then acquire a global lock. @@ -145,8 +125,8 @@ impl SlotIndex { // OK, now under the allocator lock, if we're still null then it's definitely us that will // initialize this bucket. if ptr.is_null() { - let bucket_len = SlotIndex { bucket_idx, index_in_bucket: 0 }.entries(); - let bucket_layout = std::alloc::Layout::array::>(bucket_len).unwrap(); + let bucket_layout = + std::alloc::Layout::array::>(bucket_idx.capacity()).unwrap(); // This is more of a sanity check -- this code is very cold, so it's safe to pay a // little extra cost here. assert!(bucket_layout.size() > 0); @@ -167,12 +147,10 @@ impl SlotIndex { /// Returns true if this successfully put into the map. #[inline] fn put(&self, buckets: &[AtomicPtr>; 21], value: V, extra: u32) -> bool { - // SAFETY: `bucket_idx` is ilog2(u32).saturating_sub(11), which is at most 21, i.e., - // in-bounds of buckets. - let bucket = unsafe { buckets.get_unchecked(self.bucket_idx) }; + let bucket = &buckets[self.bucket_idx]; let ptr = self.bucket_ptr(bucket); - debug_assert!(self.index_in_bucket < self.entries()); + debug_assert!(self.index_in_bucket < self.bucket_idx.capacity()); // SAFETY: `bucket` was allocated (so <= isize in total bytes) to hold `entries`, so this // must be inbounds. let slot = unsafe { ptr.add(self.index_in_bucket) }; @@ -209,12 +187,10 @@ impl SlotIndex { /// Inserts into the map, given that the slot is unique, so it won't race with other threads. #[inline] unsafe fn put_unique(&self, buckets: &[AtomicPtr>; 21], value: V, extra: u32) { - // SAFETY: `bucket_idx` is ilog2(u32).saturating_sub(11), which is at most 21, i.e., - // in-bounds of buckets. - let bucket = unsafe { buckets.get_unchecked(self.bucket_idx) }; + let bucket = &buckets[self.bucket_idx]; let ptr = self.bucket_ptr(bucket); - debug_assert!(self.index_in_bucket < self.entries()); + debug_assert!(self.index_in_bucket < self.bucket_idx.capacity()); // SAFETY: `bucket` was allocated (so <= isize in total bytes) to hold `entries`, so this // must be inbounds. let slot = unsafe { ptr.add(self.index_in_bucket) }; @@ -254,7 +230,7 @@ pub struct VecCache { // ... // Bucket 19: 1073741824 // Bucket 20: 2147483648 - // The total number of entries if all buckets are initialized is u32::MAX-1. + // The total number of entries if all buckets are initialized is 2^32. buckets: [AtomicPtr>; BUCKETS], // In the compiler's current usage these are only *read* during incremental and self-profiling. @@ -289,7 +265,7 @@ unsafe impl Drop for VecCache { assert!(!std::mem::needs_drop::()); assert!(!std::mem::needs_drop::()); - for (idx, bucket) in self.buckets.iter().enumerate() { + for (idx, bucket) in BucketIndex::enumerate_buckets(&self.buckets) { let bucket = bucket.load(Ordering::Acquire); if !bucket.is_null() { let layout = std::alloc::Layout::array::>(ENTRIES_BY_BUCKET[idx]).unwrap(); @@ -299,7 +275,7 @@ unsafe impl Drop for VecCache { } } - for (idx, bucket) in self.present.iter().enumerate() { + for (idx, bucket) in BucketIndex::enumerate_buckets(&self.present) { let bucket = bucket.load(Ordering::Acquire); if !bucket.is_null() { let layout = std::alloc::Layout::array::>(ENTRIES_BY_BUCKET[idx]).unwrap(); @@ -365,5 +341,164 @@ where } } -#[cfg(test)] -mod tests; +/// Index into an array of buckets. +/// +/// Using an enum lets us tell the compiler that values range from 0 to 20, +/// allowing array bounds checks to be optimized away, +/// without having to resort to pattern types or other unstable features. +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(usize)] +enum BucketIndex { + // tidy-alphabetical-start + Bucket00, + Bucket01, + Bucket02, + Bucket03, + Bucket04, + Bucket05, + Bucket06, + Bucket07, + Bucket08, + Bucket09, + Bucket10, + Bucket11, + Bucket12, + Bucket13, + Bucket14, + Bucket15, + Bucket16, + Bucket17, + Bucket18, + Bucket19, + Bucket20, + // tidy-alphabetical-end +} + +impl Debug for BucketIndex { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + Debug::fmt(&self.to_usize(), f) + } +} + +impl BucketIndex { + /// Capacity of bucket 0 (and also of bucket 1). + const BUCKET_0_CAPACITY: usize = 1 << (Self::NONZERO_BUCKET_SHIFT_ADJUST + 1); + /// Adjustment factor from the highest-set-bit-position of a flat index, + /// to its corresponding bucket number. + /// + /// For example, the first flat-index in bucket 2 is 8192. + /// Its highest-set-bit-position is `(8192).ilog2() == 13`, and subtracting + /// the adjustment factor of 11 gives the bucket number of 2. + const NONZERO_BUCKET_SHIFT_ADJUST: usize = 11; + + #[inline(always)] + const fn to_usize(self) -> usize { + self as usize + } + + #[inline(always)] + const fn from_raw(raw: usize) -> Self { + match raw { + // tidy-alphabetical-start + 00 => Self::Bucket00, + 01 => Self::Bucket01, + 02 => Self::Bucket02, + 03 => Self::Bucket03, + 04 => Self::Bucket04, + 05 => Self::Bucket05, + 06 => Self::Bucket06, + 07 => Self::Bucket07, + 08 => Self::Bucket08, + 09 => Self::Bucket09, + 10 => Self::Bucket10, + 11 => Self::Bucket11, + 12 => Self::Bucket12, + 13 => Self::Bucket13, + 14 => Self::Bucket14, + 15 => Self::Bucket15, + 16 => Self::Bucket16, + 17 => Self::Bucket17, + 18 => Self::Bucket18, + 19 => Self::Bucket19, + 20 => Self::Bucket20, + // tidy-alphabetical-end + _ => panic!("bucket index out of range"), + } + } + + /// Total number of slots in this bucket. + #[inline(always)] + const fn capacity(self) -> usize { + match self { + Self::Bucket00 => Self::BUCKET_0_CAPACITY, + // Bucket 1 has a capacity of `1 << (1 + 11) == pow(2, 12) == 4096`. + // Bucket 2 has a capacity of `1 << (2 + 11) == pow(2, 13) == 8192`. + _ => 1 << (self.to_usize() + Self::NONZERO_BUCKET_SHIFT_ADJUST), + } + } + + /// Converts a flat index in the range `0..=u32::MAX` into a bucket index, + /// and a slot offset within that bucket. + /// + /// Panics if `flat > u32::MAX`. + #[inline(always)] + const fn from_flat_index(flat: usize) -> (Self, usize) { + if flat > u32::MAX as usize { + panic!(); + } + + // If the index is in bucket 0, the conversion is trivial. + // This also avoids calling `ilog2` when `flat == 0`. + if flat < Self::BUCKET_0_CAPACITY { + return (Self::Bucket00, flat); + } + + // General-case conversion for a non-zero bucket index. + // + // | bucket | slot + // flat | ilog2 | index | offset + // ------------------------------ + // 4096 | 12 | 1 | 0 + // 4097 | 12 | 1 | 1 + // ... + // 8191 | 12 | 1 | 4095 + // 8192 | 13 | 2 | 0 + let highest_bit_pos = flat.ilog2() as usize; + let bucket_index = + BucketIndex::from_raw(highest_bit_pos - Self::NONZERO_BUCKET_SHIFT_ADJUST); + + // Clear the highest-set bit (which selects the bucket) to get the + // slot offset within this bucket. + let slot_offset = flat - (1 << highest_bit_pos); + + (bucket_index, slot_offset) + } + + #[inline(always)] + fn iter_all() -> impl ExactSizeIterator { + (0usize..BUCKETS).map(BucketIndex::from_raw) + } + + #[inline(always)] + fn enumerate_buckets(buckets: &[T; BUCKETS]) -> impl ExactSizeIterator { + BucketIndex::iter_all().zip(buckets) + } +} + +impl Index for [T; BUCKETS] { + type Output = T; + + #[inline(always)] + fn index(&self, index: BucketIndex) -> &Self::Output { + // The optimizer should be able to see that see that a bucket index is + // always in-bounds, and omit the runtime bounds check. + &self[index.to_usize()] + } +} + +impl IndexMut for [T; BUCKETS] { + #[inline(always)] + fn index_mut(&mut self, index: BucketIndex) -> &mut Self::Output { + &mut self[index.to_usize()] + } +} diff --git a/compiler/rustc_data_structures/src/vec_cache/tests.rs b/compiler/rustc_data_structures/src/vec_cache/tests.rs index f588442eee62a..f12937ad565da 100644 --- a/compiler/rustc_data_structures/src/vec_cache/tests.rs +++ b/compiler/rustc_data_structures/src/vec_cache/tests.rs @@ -1,10 +1,46 @@ use super::*; +#[test] +#[should_panic(expected = "bucket index out of range")] +fn bucket_index_n_buckets() { + BucketIndex::from_raw(BUCKETS); +} + +#[test] +fn bucket_index_round_trip() { + for i in 0..BUCKETS { + assert_eq!(BucketIndex::from_raw(i).to_usize(), i); + } +} + +#[test] +fn bucket_index_iter_all_len() { + let len = BucketIndex::iter_all().len(); + assert_eq!(len, BUCKETS); + + let len = BucketIndex::iter_all().collect::>().len(); + assert_eq!(len, BUCKETS); + + let len = BucketIndex::enumerate_buckets(&[(); BUCKETS]).len(); + assert_eq!(len, BUCKETS); +} + +#[test] +fn bucket_index_capacity() { + // Check that the combined capacity of all buckets is 2^32 slots. + // That's 1 larger than `u32::MAX`, so store the total as a `u64`. + let mut total = 0u64; + for i in BucketIndex::iter_all() { + total += u64::try_from(i.capacity()).unwrap(); + } + assert_eq!(total, 1 << 32); +} + #[test] #[cfg(not(miri))] -fn vec_cache_empty() { +fn vec_cache_empty_exhaustive() { let cache: VecCache = VecCache::default(); - for key in 0..u32::MAX { + for key in 0..=u32::MAX { assert!(cache.lookup(&key).is_none()); } } @@ -70,8 +106,8 @@ fn slot_entries_table() { #[test] fn bucket_entries_matches() { - for i in 0..BUCKETS { - assert_eq!(SlotIndex { bucket_idx: i, index_in_bucket: 0 }.entries(), ENTRIES_BY_BUCKET[i]); + for i in BucketIndex::iter_all() { + assert_eq!(i.capacity(), ENTRIES_BY_BUCKET[i]); } } @@ -84,13 +120,13 @@ fn slot_index_exhaustive() { } let slot_idx = SlotIndex::from_index(0); assert_eq!(slot_idx.index_in_bucket, 0); - assert_eq!(slot_idx.bucket_idx, 0); + assert_eq!(slot_idx.bucket_idx, BucketIndex::Bucket00); let mut prev = slot_idx; for idx in 1..=u32::MAX { let slot_idx = SlotIndex::from_index(idx); // SAFETY: Ensure indices don't go out of bounds of buckets. - assert!(slot_idx.index_in_bucket < slot_idx.entries()); + assert!(slot_idx.index_in_bucket < slot_idx.bucket_idx.capacity()); if prev.bucket_idx == slot_idx.bucket_idx { assert_eq!(prev.index_in_bucket + 1, slot_idx.index_in_bucket); @@ -98,8 +134,8 @@ fn slot_index_exhaustive() { assert_eq!(slot_idx.index_in_bucket, 0); } - assert_eq!(buckets[slot_idx.bucket_idx], slot_idx.entries() as u32); - assert_eq!(ENTRIES_BY_BUCKET[slot_idx.bucket_idx], slot_idx.entries(), "{}", idx); + assert_eq!(buckets[slot_idx.bucket_idx], slot_idx.bucket_idx.capacity() as u32); + assert_eq!(ENTRIES_BY_BUCKET[slot_idx.bucket_idx], slot_idx.bucket_idx.capacity(), "{idx}",); prev = slot_idx; } diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs index 9526143fcace5..71424f7275c94 100644 --- a/compiler/rustc_hir/src/hir.rs +++ b/compiler/rustc_hir/src/hir.rs @@ -1264,6 +1264,8 @@ pub struct HashIgnoredAttrId { pub attr_id: AttrId, } +/// Many functions on this type have their documentation in the [`AttributeExt`] trait, +/// since they defer their implementation directly to that trait. #[derive(Clone, Debug, Encodable, Decodable, HashStable_Generic)] pub enum Attribute { /// A parsed built-in attribute. diff --git a/compiler/rustc_mir_transform/src/copy_prop.rs b/compiler/rustc_mir_transform/src/copy_prop.rs index 89ee96317ac93..e04cb26e89902 100644 --- a/compiler/rustc_mir_transform/src/copy_prop.rs +++ b/compiler/rustc_mir_transform/src/copy_prop.rs @@ -58,7 +58,8 @@ impl<'tcx> crate::MirPass<'tcx> for CopyProp { } } -/// Utility to help performing substitution of `*pattern` by `target`. +/// Utility to help performing substitution: for all key-value pairs in `copy_classes`, +/// all occurrences of the key get replaced by the value. struct Replacer<'a, 'tcx> { tcx: TyCtxt<'tcx>, unified: DenseBitSet, diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs index 245ee6ec1cb75..e9a20aa016550 100644 --- a/compiler/rustc_mir_transform/src/gvn.rs +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -48,8 +48,15 @@ //! # Handling of references //! //! We handle references by assigning a different "provenance" index to each Ref/RawPtr rvalue. -//! This ensure that we do not spuriously merge borrows that should not be merged. Meanwhile, we -//! consider all the derefs of an immutable reference to a freeze type to give the same value: +//! This ensure that we do not spuriously merge borrows that should not be merged. For instance: +//! ```ignore (MIR) +//! _x = &_a; +//! _a = 0; +//! _y = &_a; // cannot be turned into `_y = _x`! +//! ``` +//! +//! On top of that, we consider all the derefs of an immutable reference to a freeze type to give +//! the same value: //! ```ignore (MIR) //! _a = *_b // _b is &Freeze //! _c = *_b // replaced by _c = _a diff --git a/library/core/src/ffi/va_list.rs b/library/core/src/ffi/va_list.rs index b6dfe3a541842..f0f58a0f83430 100644 --- a/library/core/src/ffi/va_list.rs +++ b/library/core/src/ffi/va_list.rs @@ -244,7 +244,7 @@ impl VaList<'_> { #[rustc_const_unstable(feature = "const_c_variadic", issue = "151787")] impl<'f> const Clone for VaList<'f> { - #[inline] + #[inline] // Avoid codegen when not used to help backends that don't support VaList. fn clone(&self) -> Self { // We only implement Clone and not Copy because some future target might not be able to // implement Copy (e.g. because it allocates). For the same reason we use an intrinsic @@ -256,6 +256,7 @@ impl<'f> const Clone for VaList<'f> { #[rustc_const_unstable(feature = "const_c_variadic", issue = "151787")] impl<'f> const Drop for VaList<'f> { + #[inline] // Avoid codegen when not used to help backends that don't support VaList. fn drop(&mut self) { // SAFETY: this variable argument list is being dropped, so won't be read from again. unsafe { va_end(self) } @@ -326,7 +327,7 @@ impl<'f> VaList<'f> { /// /// Calling this function with an incompatible type, an invalid value, or when there /// are no more variable arguments, is unsound. - #[inline] + #[inline] // Avoid codegen when not used to help backends that don't support VaList. #[rustc_const_unstable(feature = "const_c_variadic", issue = "151787")] pub const unsafe fn arg(&mut self) -> T { // SAFETY: the caller must uphold the safety contract for `va_arg`.