From 71d656edde5b2e588a129951beef1c5c7dbe35e7 Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Sat, 14 Feb 2026 17:41:51 +0300 Subject: [PATCH 1/5] LinkedGraph: Use IndexVec instead of Vec in LinkedGraph nodes --- .../src/graph/linked_graph/mod.rs | 35 ++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/compiler/rustc_data_structures/src/graph/linked_graph/mod.rs b/compiler/rustc_data_structures/src/graph/linked_graph/mod.rs index ecb0095626b4a..6d53b155e9a2a 100644 --- a/compiler/rustc_data_structures/src/graph/linked_graph/mod.rs +++ b/compiler/rustc_data_structures/src/graph/linked_graph/mod.rs @@ -23,6 +23,7 @@ use std::fmt::Debug; use rustc_index::bit_set::DenseBitSet; +use rustc_index::{Idx, IndexSlice, IndexVec}; use tracing::debug; #[cfg(test)] @@ -45,7 +46,7 @@ mod tests; /// and does not implement those traits, so it has its own implementations of a /// few basic graph algorithms. pub struct LinkedGraph { - nodes: Vec>, + nodes: IndexVec>, edges: Vec>, } @@ -62,7 +63,7 @@ pub struct Edge { pub data: E, } -#[derive(Copy, Clone, PartialEq, Debug)] +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] pub struct NodeIndex(pub usize); #[derive(Copy, Clone, PartialEq, Debug)] @@ -87,19 +88,29 @@ impl NodeIndex { } } +impl Idx for NodeIndex { + fn new(idx: usize) -> NodeIndex { + NodeIndex(idx) + } + + fn index(self) -> usize { + self.0 + } +} + impl LinkedGraph { pub fn new() -> Self { - Self { nodes: Vec::new(), edges: Vec::new() } + Self { nodes: IndexVec::new(), edges: Vec::new() } } pub fn with_capacity(nodes: usize, edges: usize) -> Self { - Self { nodes: Vec::with_capacity(nodes), edges: Vec::with_capacity(edges) } + Self { nodes: IndexVec::with_capacity(nodes), edges: Vec::with_capacity(edges) } } // # Simple accessors #[inline] - pub fn all_nodes(&self) -> &[Node] { + pub fn all_nodes(&self) -> &IndexSlice> { &self.nodes } @@ -131,15 +142,15 @@ impl LinkedGraph { } pub fn mut_node_data(&mut self, idx: NodeIndex) -> &mut N { - &mut self.nodes[idx.0].data + &mut self.nodes[idx].data } pub fn node_data(&self, idx: NodeIndex) -> &N { - &self.nodes[idx.0].data + &self.nodes[idx].data } pub fn node(&self, idx: NodeIndex) -> &Node { - &self.nodes[idx.0] + &self.nodes[idx] } // # Edge construction and queries @@ -154,16 +165,16 @@ impl LinkedGraph { let idx = self.next_edge_index(); // read current first of the list of edges from each node - let source_first = self.nodes[source.0].first_edge[OUTGOING.repr]; - let target_first = self.nodes[target.0].first_edge[INCOMING.repr]; + let source_first = self.nodes[source].first_edge[OUTGOING.repr]; + let target_first = self.nodes[target].first_edge[INCOMING.repr]; // create the new edge, with the previous firsts from each node // as the next pointers self.edges.push(Edge { next_edge: [source_first, target_first], source, target, data }); // adjust the firsts for each node target be the next object. - self.nodes[source.0].first_edge[OUTGOING.repr] = idx; - self.nodes[target.0].first_edge[INCOMING.repr] = idx; + self.nodes[source].first_edge[OUTGOING.repr] = idx; + self.nodes[target].first_edge[INCOMING.repr] = idx; idx } From 96697d41ed0c2d70dc4bb64cb4e660da1b82150d Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Sat, 14 Feb 2026 17:53:00 +0300 Subject: [PATCH 2/5] LinkedGraph: support adding nodes and edges in arbitrary order If an edge uses some not-yet-known node, we just leave the node's data empty, that data can be added later. Use this support to avoid skipping edges in DepGraphQuery --- .../src/graph/linked_graph/mod.rs | 24 ++++++++++++++----- .../src/graph/linked_graph/tests.rs | 2 +- .../rustc_middle/src/dep_graph/retained.rs | 17 ++++--------- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/compiler/rustc_data_structures/src/graph/linked_graph/mod.rs b/compiler/rustc_data_structures/src/graph/linked_graph/mod.rs index 6d53b155e9a2a..2223e85a24957 100644 --- a/compiler/rustc_data_structures/src/graph/linked_graph/mod.rs +++ b/compiler/rustc_data_structures/src/graph/linked_graph/mod.rs @@ -52,7 +52,7 @@ pub struct LinkedGraph { pub struct Node { first_edge: [EdgeIndex; 2], // see module comment - pub data: N, + pub data: Option, } #[derive(Debug)] @@ -135,18 +135,30 @@ impl LinkedGraph { NodeIndex(self.nodes.len()) } + fn ensure_node(&mut self, idx: NodeIndex) -> &mut Node { + self.nodes.ensure_contains_elem(idx, || Node { + first_edge: [INVALID_EDGE_INDEX, INVALID_EDGE_INDEX], + data: None, + }) + } + + pub fn add_node_with_idx(&mut self, idx: NodeIndex, data: N) { + let old_data = self.ensure_node(idx).data.replace(data); + debug_assert!(old_data.is_none()); + } + pub fn add_node(&mut self, data: N) -> NodeIndex { let idx = self.next_node_index(); - self.nodes.push(Node { first_edge: [INVALID_EDGE_INDEX, INVALID_EDGE_INDEX], data }); + self.add_node_with_idx(idx, data); idx } pub fn mut_node_data(&mut self, idx: NodeIndex) -> &mut N { - &mut self.nodes[idx].data + self.nodes[idx].data.as_mut().unwrap() } pub fn node_data(&self, idx: NodeIndex) -> &N { - &self.nodes[idx].data + self.nodes[idx].data.as_ref().unwrap() } pub fn node(&self, idx: NodeIndex) -> &Node { @@ -165,8 +177,8 @@ impl LinkedGraph { let idx = self.next_edge_index(); // read current first of the list of edges from each node - let source_first = self.nodes[source].first_edge[OUTGOING.repr]; - let target_first = self.nodes[target].first_edge[INCOMING.repr]; + let source_first = self.ensure_node(source).first_edge[OUTGOING.repr]; + let target_first = self.ensure_node(target).first_edge[INCOMING.repr]; // create the new edge, with the previous firsts from each node // as the next pointers diff --git a/compiler/rustc_data_structures/src/graph/linked_graph/tests.rs b/compiler/rustc_data_structures/src/graph/linked_graph/tests.rs index 357aa81a57ca3..da416cd638d47 100644 --- a/compiler/rustc_data_structures/src/graph/linked_graph/tests.rs +++ b/compiler/rustc_data_structures/src/graph/linked_graph/tests.rs @@ -40,7 +40,7 @@ fn each_node() { let expected = ["A", "B", "C", "D", "E", "F"]; graph.each_node(|idx, node| { assert_eq!(&expected[idx.0], graph.node_data(idx)); - assert_eq!(expected[idx.0], node.data); + assert_eq!(expected[idx.0], node.data.unwrap()); true }); } diff --git a/compiler/rustc_middle/src/dep_graph/retained.rs b/compiler/rustc_middle/src/dep_graph/retained.rs index 4427982e3708f..626b3b7821794 100644 --- a/compiler/rustc_middle/src/dep_graph/retained.rs +++ b/compiler/rustc_middle/src/dep_graph/retained.rs @@ -1,6 +1,5 @@ use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::graph::linked_graph::{Direction, INCOMING, LinkedGraph, NodeIndex}; -use rustc_index::IndexVec; use super::{DepNode, DepNodeIndex}; @@ -13,7 +12,6 @@ use super::{DepNode, DepNodeIndex}; pub struct RetainedDepGraph { pub inner: LinkedGraph, pub indices: FxHashMap, - pub dep_index_to_index: IndexVec>, } impl RetainedDepGraph { @@ -23,27 +21,22 @@ impl RetainedDepGraph { let inner = LinkedGraph::with_capacity(node_count, edge_count); let indices = FxHashMap::default(); - let dep_index_to_index = IndexVec::new(); - Self { inner, indices, dep_index_to_index } + Self { inner, indices } } pub fn push(&mut self, index: DepNodeIndex, node: DepNode, edges: &[DepNodeIndex]) { - let source = self.inner.add_node(node); - self.dep_index_to_index.insert(index, source); + let source = NodeIndex(index.as_usize()); + self.inner.add_node_with_idx(source, node); self.indices.insert(node, source); for &target in edges.iter() { - // We may miss the edges that are pushed while the `DepGraphQuery` is being accessed. - // Skip them to issues. - if let Some(&Some(target)) = self.dep_index_to_index.get(target) { - self.inner.add_edge(source, target, ()); - } + self.inner.add_edge(source, NodeIndex(target.as_usize()), ()); } } pub fn nodes(&self) -> Vec<&DepNode> { - self.inner.all_nodes().iter().map(|n| &n.data).collect() + self.inner.all_nodes().iter().map(|n| n.data.as_ref().unwrap()).collect() } pub fn edges(&self) -> Vec<(&DepNode, &DepNode)> { From 86c5da6b8be5f77354d08748188942c436710a36 Mon Sep 17 00:00:00 2001 From: cyrgani Date: Fri, 13 Mar 2026 10:28:04 +0000 Subject: [PATCH 3/5] delete some duplicated tests --- tests/ui/moves/move-1-unique.rs | 25 ------------------------- tests/ui/moves/move-2-unique.rs | 10 ---------- tests/ui/moves/move-4-unique.rs | 18 ------------------ tests/ui/moves/move-arg-2-unique.rs | 12 ------------ tests/ui/moves/move-scalar.rs | 10 ---------- 5 files changed, 75 deletions(-) delete mode 100644 tests/ui/moves/move-1-unique.rs delete mode 100644 tests/ui/moves/move-2-unique.rs delete mode 100644 tests/ui/moves/move-4-unique.rs delete mode 100644 tests/ui/moves/move-arg-2-unique.rs delete mode 100644 tests/ui/moves/move-scalar.rs diff --git a/tests/ui/moves/move-1-unique.rs b/tests/ui/moves/move-1-unique.rs deleted file mode 100644 index c97bfaaaf1a5a..0000000000000 --- a/tests/ui/moves/move-1-unique.rs +++ /dev/null @@ -1,25 +0,0 @@ -//@ run-pass -#![allow(unused_mut)] -#![allow(dead_code)] - -#[derive(Clone)] -struct Triple { - x: isize, - y: isize, - z: isize, -} - -fn test(x: bool, foo: Box) -> isize { - let bar = foo; - let mut y: Box; - if x { y = bar; } else { y = Box::new(Triple{x: 4, y: 5, z: 6}); } - return y.y; -} - -pub fn main() { - let x: Box<_> = Box::new(Triple{x: 1, y: 2, z: 3}); - assert_eq!(test(true, x.clone()), 2); - assert_eq!(test(true, x.clone()), 2); - assert_eq!(test(true, x.clone()), 2); - assert_eq!(test(false, x), 5); -} diff --git a/tests/ui/moves/move-2-unique.rs b/tests/ui/moves/move-2-unique.rs deleted file mode 100644 index 2204ea95741db..0000000000000 --- a/tests/ui/moves/move-2-unique.rs +++ /dev/null @@ -1,10 +0,0 @@ -//@ run-pass -#![allow(dead_code)] - -struct X { x: isize, y: isize, z: isize } - -pub fn main() { - let x: Box<_> = Box::new(X {x: 1, y: 2, z: 3}); - let y = x; - assert_eq!(y.y, 2); -} diff --git a/tests/ui/moves/move-4-unique.rs b/tests/ui/moves/move-4-unique.rs deleted file mode 100644 index 09e0f11a8b442..0000000000000 --- a/tests/ui/moves/move-4-unique.rs +++ /dev/null @@ -1,18 +0,0 @@ -//@ run-pass -#![allow(dead_code)] - -struct Triple {a: isize, b: isize, c: isize} - -fn test(foo: Box) -> Box { - let foo = foo; - let bar = foo; - let baz = bar; - let quux = baz; - return quux; -} - -pub fn main() { - let x = Box::new(Triple{a: 1, b: 2, c: 3}); - let y = test(x); - assert_eq!(y.c, 3); -} diff --git a/tests/ui/moves/move-arg-2-unique.rs b/tests/ui/moves/move-arg-2-unique.rs deleted file mode 100644 index d9a03be0ed2dc..0000000000000 --- a/tests/ui/moves/move-arg-2-unique.rs +++ /dev/null @@ -1,12 +0,0 @@ -//@ run-pass - -fn test(foo: Box> ) { assert_eq!((*foo)[0], 10); } - -pub fn main() { - let x = Box::new(vec![10]); - // Test forgetting a local by move-in - test(x); - - // Test forgetting a temporary by move-in. - test(Box::new(vec![10])); -} diff --git a/tests/ui/moves/move-scalar.rs b/tests/ui/moves/move-scalar.rs deleted file mode 100644 index e8cf5632b322a..0000000000000 --- a/tests/ui/moves/move-scalar.rs +++ /dev/null @@ -1,10 +0,0 @@ -//@ run-pass -#![allow(unused_mut)] - -pub fn main() { - - let y: isize = 42; - let mut x: isize; - x = y; - assert_eq!(x, 42); -} From 6699c13683a302e6457be7f9fb930ef8102b1982 Mon Sep 17 00:00:00 2001 From: Zalathar Date: Thu, 12 Mar 2026 21:29:15 +1100 Subject: [PATCH 4/5] Move the big `rustc_query_impl` macro into a physical `query_impl.rs` Moving the macro and its expansion into the same physical file resolves a lot of tension in the current module arrangement. Code in the macro is now free to use plain imports in the same file, and there is no longer any question of whether `mod query_impl` should be declared inside the macro, or surrounding a separate expansion site. --- compiler/rustc_query_impl/src/execution.rs | 2 +- compiler/rustc_query_impl/src/lib.rs | 14 +- compiler/rustc_query_impl/src/plumbing.rs | 263 +---------------- .../rustc_query_impl/src/profiling_support.rs | 2 +- compiler/rustc_query_impl/src/query_impl.rs | 266 ++++++++++++++++++ 5 files changed, 275 insertions(+), 272 deletions(-) create mode 100644 compiler/rustc_query_impl/src/query_impl.rs diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs index 2b535d64cf706..eae91a2b2e566 100644 --- a/compiler/rustc_query_impl/src/execution.rs +++ b/compiler/rustc_query_impl/src/execution.rs @@ -18,9 +18,9 @@ use rustc_span::{DUMMY_SP, Span}; use tracing::warn; use crate::dep_graph::{DepNode, DepNodeIndex}; -use crate::for_each_query_vtable; use crate::job::{QueryJobInfo, QueryJobMap, find_cycle_in_stack, report_cycle}; use crate::plumbing::{current_query_job, next_job_id, start_query}; +use crate::query_impl::for_each_query_vtable; #[inline] fn equivalent_key(k: K) -> impl Fn(&(K, V)) -> bool { diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index 44c97d19914c9..c62ea3e289fcb 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -10,26 +10,24 @@ use rustc_data_structures::sync::AtomicU64; use rustc_middle::dep_graph; -use rustc_middle::queries::{self, ExternProviders, Providers, TaggedQueryKey}; +use rustc_middle::queries::{ExternProviders, Providers}; +use rustc_middle::query::QueryCache; use rustc_middle::query::on_disk_cache::OnDiskCache; use rustc_middle::query::plumbing::{QuerySystem, QueryVTable}; -use rustc_middle::query::{AsLocalQueryKey, QueryCache, QueryMode}; use rustc_middle::ty::TyCtxt; -use rustc_span::Span; pub use crate::dep_kind_vtables::make_dep_kind_vtables; pub use crate::execution::{CollectActiveJobsKind, collect_active_jobs_from_all_queries}; pub use crate::job::{QueryJobMap, break_query_cycles, print_query_stack}; -#[macro_use] -mod plumbing; - mod dep_kind_vtables; mod error; mod execution; mod from_cycle_error; mod job; +mod plumbing; mod profiling_support; +mod query_impl; /// Trait that knows how to look up the [`QueryVTable`] for a particular query. /// @@ -51,7 +49,7 @@ pub fn query_system<'tcx>( on_disk_cache: Option, incremental: bool, ) -> QuerySystem<'tcx> { - let mut query_vtables = make_query_vtables(incremental); + let mut query_vtables = query_impl::make_query_vtables(incremental); from_cycle_error::specialize_query_vtables(&mut query_vtables); QuerySystem { arenas: Default::default(), @@ -63,8 +61,6 @@ pub fn query_system<'tcx>( } } -rustc_middle::rustc_with_all_queries! { define_queries! } - pub fn provide(providers: &mut rustc_middle::util::Providers) { providers.hooks.alloc_self_profile_query_strings = profiling_support::alloc_self_profile_query_strings; diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index bece2e56c0a86..a346836057d93 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -1,7 +1,3 @@ -//! The implementation of the query system itself. This defines the macros that -//! generate the actual methods on tcx which find and execute the provider, -//! manage the caches, and so forth. - use std::num::NonZero; use rustc_data_structures::sync::{DynSend, DynSync}; @@ -29,10 +25,8 @@ use rustc_span::def_id::LOCAL_CRATE; use crate::error::{QueryOverflow, QueryOverflowNote}; use crate::execution::{all_inactive, force_query}; use crate::job::find_dep_kind_root; -use crate::{ - CollectActiveJobsKind, GetQueryVTable, collect_active_jobs_from_all_queries, - for_each_query_vtable, -}; +use crate::query_impl::for_each_query_vtable; +use crate::{CollectActiveJobsKind, GetQueryVTable, collect_active_jobs_from_all_queries}; fn depth_limit_error<'tcx>(tcx: TyCtxt<'tcx>, job: QueryJobId) { let job_map = collect_active_jobs_from_all_queries(tcx, CollectActiveJobsKind::Full); @@ -283,256 +277,3 @@ pub(crate) fn force_from_dep_node_inner<'tcx, Q: GetQueryVTable<'tcx>>( false } } - -macro_rules! define_queries { - ( - // Note: `$K` and `$V` are unused but present so this can be called by - // `rustc_with_all_queries`. - queries { - $( - $(#[$attr:meta])* - fn $name:ident($K:ty) -> $V:ty - { - // Search for (QMODLIST) to find all occurrences of this query modifier list. - anon: $anon:literal, - arena_cache: $arena_cache:literal, - cache_on_disk: $cache_on_disk:literal, - depth_limit: $depth_limit:literal, - eval_always: $eval_always:literal, - feedable: $feedable:literal, - no_hash: $no_hash:literal, - returns_error_guaranteed: $returns_error_guaranteed:literal, - separate_provide_extern: $separate_provide_extern:literal, - } - )* - } - // Non-queries are unused here. - non_queries { $($_:tt)* } - ) => { - pub(crate) mod query_impl { $(pub(crate) mod $name { - use super::super::*; - use ::rustc_middle::query::erase::{self, Erased}; - - // It seems to be important that every query has its own monomorphic - // copy of `execute_query_incr` and `execute_query_non_incr`. - // Trying to inline these wrapper functions into their generic - // "inner" helpers tends to break `tests/run-make/short-ice`. - - pub(crate) mod execute_query_incr { - use super::*; - - // Adding `__rust_end_short_backtrace` marker to backtraces so that we emit the frames - // when `RUST_BACKTRACE=1`, add a new mod with `$name` here is to allow duplicate naming - #[inline(never)] - pub(crate) fn __rust_end_short_backtrace<'tcx>( - tcx: TyCtxt<'tcx>, - span: Span, - key: queries::$name::Key<'tcx>, - mode: QueryMode, - ) -> Option>> { - #[cfg(debug_assertions)] - let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered(); - execution::execute_query_incr_inner( - &tcx.query_system.query_vtables.$name, - tcx, - span, - key, - mode - ) - } - } - - pub(crate) mod execute_query_non_incr { - use super::*; - - #[inline(never)] - pub(crate) fn __rust_end_short_backtrace<'tcx>( - tcx: TyCtxt<'tcx>, - span: Span, - key: queries::$name::Key<'tcx>, - __mode: QueryMode, - ) -> Option>> { - Some(execution::execute_query_non_incr_inner( - &tcx.query_system.query_vtables.$name, - tcx, - span, - key, - )) - } - } - - /// Defines an `invoke_provider` function that calls the query's provider, - /// to be used as a function pointer in the query's vtable. - /// - /// To mark a short-backtrace boundary, the function's actual name - /// (after demangling) must be `__rust_begin_short_backtrace`. - mod invoke_provider_fn { - use super::*; - use ::rustc_middle::queries::$name::{Key, Value, provided_to_erased}; - - #[inline(never)] - pub(crate) fn __rust_begin_short_backtrace<'tcx>( - tcx: TyCtxt<'tcx>, - key: Key<'tcx>, - ) -> Erased> { - #[cfg(debug_assertions)] - let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered(); - - // Call the actual provider function for this query. - - #[cfg($separate_provide_extern)] - let provided_value = if let Some(local_key) = key.as_local_key() { - (tcx.query_system.local_providers.$name)(tcx, local_key) - } else { - (tcx.query_system.extern_providers.$name)(tcx, key) - }; - - #[cfg(not($separate_provide_extern))] - let provided_value = (tcx.query_system.local_providers.$name)(tcx, key); - - rustc_middle::ty::print::with_reduced_queries!({ - tracing::trace!(?provided_value); - }); - - // Erase the returned value, because `QueryVTable` uses erased values. - // For queries with `arena_cache`, this also arena-allocates the value. - provided_to_erased(tcx, provided_value) - } - } - - pub(crate) fn make_query_vtable<'tcx>(incremental: bool) - -> QueryVTable<'tcx, queries::$name::Cache<'tcx>> - { - QueryVTable { - name: stringify!($name), - anon: $anon, - eval_always: $eval_always, - depth_limit: $depth_limit, - feedable: $feedable, - dep_kind: dep_graph::DepKind::$name, - state: Default::default(), - cache: Default::default(), - - invoke_provider_fn: self::invoke_provider_fn::__rust_begin_short_backtrace, - - #[cfg($cache_on_disk)] - will_cache_on_disk_for_key_fn: - rustc_middle::queries::_cache_on_disk_if_fns::$name, - #[cfg(not($cache_on_disk))] - will_cache_on_disk_for_key_fn: |_, _| false, - - #[cfg($cache_on_disk)] - try_load_from_disk_fn: |tcx, key, prev_index, index| { - // Check the `cache_on_disk_if` condition for this key. - if !rustc_middle::queries::_cache_on_disk_if_fns::$name(tcx, key) { - return None; - } - - let value: queries::$name::ProvidedValue<'tcx> = - $crate::plumbing::try_load_from_disk(tcx, prev_index, index)?; - - // Arena-alloc the value if appropriate, and erase it. - Some(queries::$name::provided_to_erased(tcx, value)) - }, - #[cfg(not($cache_on_disk))] - try_load_from_disk_fn: |_tcx, _key, _prev_index, _index| None, - - #[cfg($cache_on_disk)] - is_loadable_from_disk_fn: |tcx, key, index| -> bool { - rustc_middle::queries::_cache_on_disk_if_fns::$name(tcx, key) && - $crate::plumbing::loadable_from_disk(tcx, index) - }, - #[cfg(not($cache_on_disk))] - is_loadable_from_disk_fn: |_tcx, _key, _index| false, - - // The default just emits `err` and then aborts. - // `from_cycle_error::specialize_query_vtables` overwrites this default for - // certain queries. - value_from_cycle_error: |_tcx, _key, _cycle, err| { - $crate::from_cycle_error::default(err) - }, - - #[cfg($no_hash)] - hash_value_fn: None, - #[cfg(not($no_hash))] - hash_value_fn: Some(|hcx, erased_value: &erase::Erased>| { - let value = erase::restore_val(*erased_value); - rustc_middle::dep_graph::hash_result(hcx, &value) - }), - - format_value: |value| format!("{:?}", erase::restore_val::>(*value)), - create_tagged_key: TaggedQueryKey::$name, - execute_query_fn: if incremental { - query_impl::$name::execute_query_incr::__rust_end_short_backtrace - } else { - query_impl::$name::execute_query_non_incr::__rust_end_short_backtrace - }, - } - } - - /// Marker type that implements [`GetQueryVTable`] for this query. - pub(crate) enum VTableGetter {} - - impl<'tcx> GetQueryVTable<'tcx> for VTableGetter { - type Cache = rustc_middle::queries::$name::Cache<'tcx>; - - #[inline(always)] - fn query_vtable(tcx: TyCtxt<'tcx>) -> &'tcx QueryVTable<'tcx, Self::Cache> { - &tcx.query_system.query_vtables.$name - } - } - })*} - - pub fn make_query_vtables<'tcx>(incremental: bool) -> queries::QueryVTables<'tcx> { - queries::QueryVTables { - $( - $name: query_impl::$name::make_query_vtable(incremental), - )* - } - } - - /// Given a filter condition (e.g. `ALL` or `CACHE_ON_DISK`), a `tcx`, - /// and a closure expression that accepts `&QueryVTable`, this macro - /// calls that closure with each query vtable that satisfies the filter - /// condition. - /// - /// This needs to be a macro, because the vtables can have different - /// key/value/cache types for different queries. - /// - /// This macro's argument syntax is specifically intended to look like - /// plain Rust code, so that `for_each_query_vtable!(..)` calls will be - /// formatted by rustfmt. - /// - /// To avoid too much nested-macro complication, filter conditions are - /// implemented by hand as needed. - macro_rules! for_each_query_vtable { - // Call with all queries. - (ALL, $tcx:expr, $closure:expr) => {{ - let tcx: rustc_middle::ty::TyCtxt<'_> = $tcx; - $( - let query: &rustc_middle::query::plumbing::QueryVTable<'_, _> = - &tcx.query_system.query_vtables.$name; - $closure(query); - )* - }}; - - // Only call with queries that can potentially cache to disk. - // - // This allows the use of trait bounds that only need to be satisfied - // by the subset of queries that actually cache to disk. - (CACHE_ON_DISK, $tcx:expr, $closure:expr) => {{ - let tcx: rustc_middle::ty::TyCtxt<'_> = $tcx; - $( - #[cfg($cache_on_disk)] - { - let query: &rustc_middle::query::plumbing::QueryVTable<'_, _> = - &tcx.query_system.query_vtables.$name; - $closure(query); - } - )* - }} - } - - pub(crate) use for_each_query_vtable; - } -} diff --git a/compiler/rustc_query_impl/src/profiling_support.rs b/compiler/rustc_query_impl/src/profiling_support.rs index c88bc8845ebdb..c34938cdb3866 100644 --- a/compiler/rustc_query_impl/src/profiling_support.rs +++ b/compiler/rustc_query_impl/src/profiling_support.rs @@ -10,7 +10,7 @@ use rustc_middle::query::QueryCache; use rustc_middle::query::plumbing::QueryVTable; use rustc_middle::ty::TyCtxt; -use crate::for_each_query_vtable; +use crate::query_impl::for_each_query_vtable; pub(crate) struct QueryKeyStringCache { def_id_cache: FxHashMap, diff --git a/compiler/rustc_query_impl/src/query_impl.rs b/compiler/rustc_query_impl/src/query_impl.rs new file mode 100644 index 0000000000000..f833ec3871042 --- /dev/null +++ b/compiler/rustc_query_impl/src/query_impl.rs @@ -0,0 +1,266 @@ +use rustc_middle::queries::TaggedQueryKey; +use rustc_middle::query::plumbing::QueryVTable; +use rustc_middle::query::{AsLocalQueryKey, QueryMode}; +use rustc_middle::ty::TyCtxt; +use rustc_middle::{dep_graph, queries}; +use rustc_span::Span; + +use crate::{GetQueryVTable, execution, query_impl}; + +macro_rules! define_queries { + ( + // Note: `$K` and `$V` are unused but present so this can be called by + // `rustc_with_all_queries`. + queries { + $( + $(#[$attr:meta])* + fn $name:ident($K:ty) -> $V:ty + { + // Search for (QMODLIST) to find all occurrences of this query modifier list. + anon: $anon:literal, + arena_cache: $arena_cache:literal, + cache_on_disk: $cache_on_disk:literal, + depth_limit: $depth_limit:literal, + eval_always: $eval_always:literal, + feedable: $feedable:literal, + no_hash: $no_hash:literal, + returns_error_guaranteed: $returns_error_guaranteed:literal, + separate_provide_extern: $separate_provide_extern:literal, + } + )* + } + // Non-queries are unused here. + non_queries { $($_:tt)* } + ) => { + // This macro expects to be expanded into `crate::query_impl`, which is this file. + $( + pub(crate) mod $name { + use super::*; + use ::rustc_middle::query::erase::{self, Erased}; + + // It seems to be important that every query has its own monomorphic + // copy of `execute_query_incr` and `execute_query_non_incr`. + // Trying to inline these wrapper functions into their generic + // "inner" helpers tends to break `tests/run-make/short-ice`. + + pub(crate) mod execute_query_incr { + use super::*; + + // Adding `__rust_end_short_backtrace` marker to backtraces so that we emit the frames + // when `RUST_BACKTRACE=1`, add a new mod with `$name` here is to allow duplicate naming + #[inline(never)] + pub(crate) fn __rust_end_short_backtrace<'tcx>( + tcx: TyCtxt<'tcx>, + span: Span, + key: queries::$name::Key<'tcx>, + mode: QueryMode, + ) -> Option>> { + #[cfg(debug_assertions)] + let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered(); + execution::execute_query_incr_inner( + &tcx.query_system.query_vtables.$name, + tcx, + span, + key, + mode + ) + } + } + + pub(crate) mod execute_query_non_incr { + use super::*; + + #[inline(never)] + pub(crate) fn __rust_end_short_backtrace<'tcx>( + tcx: TyCtxt<'tcx>, + span: Span, + key: queries::$name::Key<'tcx>, + __mode: QueryMode, + ) -> Option>> { + Some(execution::execute_query_non_incr_inner( + &tcx.query_system.query_vtables.$name, + tcx, + span, + key, + )) + } + } + + /// Defines an `invoke_provider` function that calls the query's provider, + /// to be used as a function pointer in the query's vtable. + /// + /// To mark a short-backtrace boundary, the function's actual name + /// (after demangling) must be `__rust_begin_short_backtrace`. + mod invoke_provider_fn { + use super::*; + use ::rustc_middle::queries::$name::{Key, Value, provided_to_erased}; + + #[inline(never)] + pub(crate) fn __rust_begin_short_backtrace<'tcx>( + tcx: TyCtxt<'tcx>, + key: Key<'tcx>, + ) -> Erased> { + #[cfg(debug_assertions)] + let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered(); + + // Call the actual provider function for this query. + + #[cfg($separate_provide_extern)] + let provided_value = if let Some(local_key) = key.as_local_key() { + (tcx.query_system.local_providers.$name)(tcx, local_key) + } else { + (tcx.query_system.extern_providers.$name)(tcx, key) + }; + + #[cfg(not($separate_provide_extern))] + let provided_value = (tcx.query_system.local_providers.$name)(tcx, key); + + rustc_middle::ty::print::with_reduced_queries!({ + tracing::trace!(?provided_value); + }); + + // Erase the returned value, because `QueryVTable` uses erased values. + // For queries with `arena_cache`, this also arena-allocates the value. + provided_to_erased(tcx, provided_value) + } + } + + pub(crate) fn make_query_vtable<'tcx>(incremental: bool) + -> QueryVTable<'tcx, queries::$name::Cache<'tcx>> + { + QueryVTable { + name: stringify!($name), + anon: $anon, + eval_always: $eval_always, + depth_limit: $depth_limit, + feedable: $feedable, + dep_kind: dep_graph::DepKind::$name, + state: Default::default(), + cache: Default::default(), + + invoke_provider_fn: self::invoke_provider_fn::__rust_begin_short_backtrace, + + #[cfg($cache_on_disk)] + will_cache_on_disk_for_key_fn: + rustc_middle::queries::_cache_on_disk_if_fns::$name, + #[cfg(not($cache_on_disk))] + will_cache_on_disk_for_key_fn: |_, _| false, + + #[cfg($cache_on_disk)] + try_load_from_disk_fn: |tcx, key, prev_index, index| { + // Check the `cache_on_disk_if` condition for this key. + if !rustc_middle::queries::_cache_on_disk_if_fns::$name(tcx, key) { + return None; + } + + let value: queries::$name::ProvidedValue<'tcx> = + $crate::plumbing::try_load_from_disk(tcx, prev_index, index)?; + + // Arena-alloc the value if appropriate, and erase it. + Some(queries::$name::provided_to_erased(tcx, value)) + }, + #[cfg(not($cache_on_disk))] + try_load_from_disk_fn: |_tcx, _key, _prev_index, _index| None, + + #[cfg($cache_on_disk)] + is_loadable_from_disk_fn: |tcx, key, index| -> bool { + rustc_middle::queries::_cache_on_disk_if_fns::$name(tcx, key) && + $crate::plumbing::loadable_from_disk(tcx, index) + }, + #[cfg(not($cache_on_disk))] + is_loadable_from_disk_fn: |_tcx, _key, _index| false, + + // The default just emits `err` and then aborts. + // `from_cycle_error::specialize_query_vtables` overwrites this default for + // certain queries. + value_from_cycle_error: |_tcx, _key, _cycle, err| { + $crate::from_cycle_error::default(err) + }, + + #[cfg($no_hash)] + hash_value_fn: None, + #[cfg(not($no_hash))] + hash_value_fn: Some(|hcx, erased_value: &erase::Erased>| { + let value = erase::restore_val(*erased_value); + rustc_middle::dep_graph::hash_result(hcx, &value) + }), + + format_value: |value| format!("{:?}", erase::restore_val::>(*value)), + create_tagged_key: TaggedQueryKey::$name, + execute_query_fn: if incremental { + query_impl::$name::execute_query_incr::__rust_end_short_backtrace + } else { + query_impl::$name::execute_query_non_incr::__rust_end_short_backtrace + }, + } + } + + /// Marker type that implements [`GetQueryVTable`] for this query. + pub(crate) enum VTableGetter {} + + impl<'tcx> GetQueryVTable<'tcx> for VTableGetter { + type Cache = rustc_middle::queries::$name::Cache<'tcx>; + + #[inline(always)] + fn query_vtable(tcx: TyCtxt<'tcx>) -> &'tcx QueryVTable<'tcx, Self::Cache> { + &tcx.query_system.query_vtables.$name + } + } + } + )* + + pub(crate) fn make_query_vtables<'tcx>(incremental: bool) -> queries::QueryVTables<'tcx> { + queries::QueryVTables { + $( + $name: query_impl::$name::make_query_vtable(incremental), + )* + } + } + + /// Given a filter condition (e.g. `ALL` or `CACHE_ON_DISK`), a `tcx`, + /// and a closure expression that accepts `&QueryVTable`, this macro + /// calls that closure with each query vtable that satisfies the filter + /// condition. + /// + /// This needs to be a macro, because the vtables can have different + /// key/value/cache types for different queries. + /// + /// This macro's argument syntax is specifically intended to look like + /// plain Rust code, so that `for_each_query_vtable!(..)` calls will be + /// formatted by rustfmt. + /// + /// To avoid too much nested-macro complication, filter conditions are + /// implemented by hand as needed. + macro_rules! for_each_query_vtable { + // Call with all queries. + (ALL, $tcx:expr, $closure:expr) => {{ + let tcx: rustc_middle::ty::TyCtxt<'_> = $tcx; + $( + let query: &rustc_middle::query::plumbing::QueryVTable<'_, _> = + &tcx.query_system.query_vtables.$name; + $closure(query); + )* + }}; + + // Only call with queries that can potentially cache to disk. + // + // This allows the use of trait bounds that only need to be satisfied + // by the subset of queries that actually cache to disk. + (CACHE_ON_DISK, $tcx:expr, $closure:expr) => {{ + let tcx: rustc_middle::ty::TyCtxt<'_> = $tcx; + $( + #[cfg($cache_on_disk)] + { + let query: &rustc_middle::query::plumbing::QueryVTable<'_, _> = + &tcx.query_system.query_vtables.$name; + $closure(query); + } + )* + }} + } + + pub(crate) use for_each_query_vtable; + } +} + +rustc_middle::rustc_with_all_queries! { define_queries! } From 69ea0c506d3be9f53809feb631c62d327d4b617e Mon Sep 17 00:00:00 2001 From: Zalathar Date: Thu, 12 Mar 2026 21:29:15 +1100 Subject: [PATCH 5/5] Use more fully-qualified paths and local imports in `query_impl` --- compiler/rustc_query_impl/src/query_impl.rs | 51 ++++++++++++--------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/compiler/rustc_query_impl/src/query_impl.rs b/compiler/rustc_query_impl/src/query_impl.rs index f833ec3871042..d5fb90871e76e 100644 --- a/compiler/rustc_query_impl/src/query_impl.rs +++ b/compiler/rustc_query_impl/src/query_impl.rs @@ -1,11 +1,11 @@ use rustc_middle::queries::TaggedQueryKey; +use rustc_middle::query::erase::{self, Erased}; use rustc_middle::query::plumbing::QueryVTable; use rustc_middle::query::{AsLocalQueryKey, QueryMode}; use rustc_middle::ty::TyCtxt; -use rustc_middle::{dep_graph, queries}; use rustc_span::Span; -use crate::{GetQueryVTable, execution, query_impl}; +use crate::GetQueryVTable; macro_rules! define_queries { ( @@ -36,7 +36,6 @@ macro_rules! define_queries { $( pub(crate) mod $name { use super::*; - use ::rustc_middle::query::erase::{self, Erased}; // It seems to be important that every query has its own monomorphic // copy of `execute_query_incr` and `execute_query_non_incr`. @@ -45,6 +44,7 @@ macro_rules! define_queries { pub(crate) mod execute_query_incr { use super::*; + use rustc_middle::queries::$name::{Key, Value}; // Adding `__rust_end_short_backtrace` marker to backtraces so that we emit the frames // when `RUST_BACKTRACE=1`, add a new mod with `$name` here is to allow duplicate naming @@ -52,12 +52,12 @@ macro_rules! define_queries { pub(crate) fn __rust_end_short_backtrace<'tcx>( tcx: TyCtxt<'tcx>, span: Span, - key: queries::$name::Key<'tcx>, + key: Key<'tcx>, mode: QueryMode, - ) -> Option>> { + ) -> Option>> { #[cfg(debug_assertions)] let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered(); - execution::execute_query_incr_inner( + crate::execution::execute_query_incr_inner( &tcx.query_system.query_vtables.$name, tcx, span, @@ -69,15 +69,16 @@ macro_rules! define_queries { pub(crate) mod execute_query_non_incr { use super::*; + use rustc_middle::queries::$name::{Key, Value}; #[inline(never)] pub(crate) fn __rust_end_short_backtrace<'tcx>( tcx: TyCtxt<'tcx>, span: Span, - key: queries::$name::Key<'tcx>, + key: Key<'tcx>, __mode: QueryMode, - ) -> Option>> { - Some(execution::execute_query_non_incr_inner( + ) -> Option>> { + Some(crate::execution::execute_query_non_incr_inner( &tcx.query_system.query_vtables.$name, tcx, span, @@ -93,7 +94,7 @@ macro_rules! define_queries { /// (after demangling) must be `__rust_begin_short_backtrace`. mod invoke_provider_fn { use super::*; - use ::rustc_middle::queries::$name::{Key, Value, provided_to_erased}; + use rustc_middle::queries::$name::{Key, Value, provided_to_erased}; #[inline(never)] pub(crate) fn __rust_begin_short_backtrace<'tcx>( @@ -126,15 +127,17 @@ macro_rules! define_queries { } pub(crate) fn make_query_vtable<'tcx>(incremental: bool) - -> QueryVTable<'tcx, queries::$name::Cache<'tcx>> + -> QueryVTable<'tcx, rustc_middle::queries::$name::Cache<'tcx>> { + use rustc_middle::queries::$name::Value; + QueryVTable { name: stringify!($name), anon: $anon, eval_always: $eval_always, depth_limit: $depth_limit, feedable: $feedable, - dep_kind: dep_graph::DepKind::$name, + dep_kind: rustc_middle::dep_graph::DepKind::$name, state: Default::default(), cache: Default::default(), @@ -148,16 +151,18 @@ macro_rules! define_queries { #[cfg($cache_on_disk)] try_load_from_disk_fn: |tcx, key, prev_index, index| { + use rustc_middle::queries::$name::{ProvidedValue, provided_to_erased}; + // Check the `cache_on_disk_if` condition for this key. if !rustc_middle::queries::_cache_on_disk_if_fns::$name(tcx, key) { return None; } - let value: queries::$name::ProvidedValue<'tcx> = + let loaded_value: ProvidedValue<'tcx> = $crate::plumbing::try_load_from_disk(tcx, prev_index, index)?; // Arena-alloc the value if appropriate, and erase it. - Some(queries::$name::provided_to_erased(tcx, value)) + Some(provided_to_erased(tcx, loaded_value)) }, #[cfg(not($cache_on_disk))] try_load_from_disk_fn: |_tcx, _key, _prev_index, _index| None, @@ -180,17 +185,19 @@ macro_rules! define_queries { #[cfg($no_hash)] hash_value_fn: None, #[cfg(not($no_hash))] - hash_value_fn: Some(|hcx, erased_value: &erase::Erased>| { + hash_value_fn: Some(|hcx, erased_value: &erase::Erased>| { let value = erase::restore_val(*erased_value); rustc_middle::dep_graph::hash_result(hcx, &value) }), - format_value: |value| format!("{:?}", erase::restore_val::>(*value)), + format_value: |erased_value: &erase::Erased>| { + format!("{:?}", erase::restore_val(*erased_value)) + }, create_tagged_key: TaggedQueryKey::$name, execute_query_fn: if incremental { - query_impl::$name::execute_query_incr::__rust_end_short_backtrace + crate::query_impl::$name::execute_query_incr::__rust_end_short_backtrace } else { - query_impl::$name::execute_query_non_incr::__rust_end_short_backtrace + crate::query_impl::$name::execute_query_non_incr::__rust_end_short_backtrace }, } } @@ -209,10 +216,12 @@ macro_rules! define_queries { } )* - pub(crate) fn make_query_vtables<'tcx>(incremental: bool) -> queries::QueryVTables<'tcx> { - queries::QueryVTables { + pub(crate) fn make_query_vtables<'tcx>(incremental: bool) + -> rustc_middle::queries::QueryVTables<'tcx> + { + rustc_middle::queries::QueryVTables { $( - $name: query_impl::$name::make_query_vtable(incremental), + $name: crate::query_impl::$name::make_query_vtable(incremental), )* } }