From 7a4cb402aac4e2cf9dc44332d9cc66c762522a92 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sun, 30 Apr 2023 20:48:29 +0000 Subject: [PATCH 1/9] Enable DestinationPropagation by default. --- compiler/rustc_mir_transform/src/dest_prop.rs | 2 +- compiler/rustc_mir_transform/src/lib.rs | 4 +- compiler/rustc_mir_transform/src/nrvo.rs | 234 ----------------- ...o.DestinationPropagation.panic-abort.diff} | 31 +-- ....DestinationPropagation.panic-unwind.diff} | 31 +-- .../nrvo_borrowed.rs} | 4 +- .../{ => dest-prop}/nrvo_miscompile_111005.rs | 4 +- ..._111005.wrong.DestinationPropagation.diff} | 7 +- tests/mir-opt/pre-codegen/checked_ops.rs | 3 +- ...b_at_home.PreCodegen.after.panic-abort.mir | 6 +- ..._at_home.PreCodegen.after.panic-unwind.mir | 6 +- ...py.enum_clone_as_copy.PreCodegen.after.mir | 4 +- .../derived_ord.demo_le.PreCodegen.after.mir | 35 ++- tests/mir-opt/pre-codegen/derived_ord.rs | 5 +- ....{impl#0}-partial_cmp.PreCodegen.after.mir | 32 ++- ...ward_loop.PreCodegen.after.panic-abort.mir | 68 +++-- ...ard_loop.PreCodegen.after.panic-unwind.mir | 68 +++-- ...ated_loop.PreCodegen.after.panic-abort.mir | 236 ++++++++---------- ...ted_loop.PreCodegen.after.panic-unwind.mir | 72 +++--- ...ward_loop.PreCodegen.after.panic-abort.mir | 180 ++++++------- ...ard_loop.PreCodegen.after.panic-unwind.mir | 180 ++++++------- ...ange_loop.PreCodegen.after.panic-abort.mir | 52 ++-- ...nge_loop.PreCodegen.after.panic-unwind.mir | 52 ++-- ...erse_loop.PreCodegen.after.panic-abort.mir | 74 +++--- ...rse_loop.PreCodegen.after.panic-unwind.mir | 74 +++--- ...e_ord.demo_ge_partial.PreCodegen.after.mir | 18 +- ...ple_ord.demo_le_total.PreCodegen.after.mir | 18 +- 27 files changed, 579 insertions(+), 921 deletions(-) delete mode 100644 compiler/rustc_mir_transform/src/nrvo.rs rename tests/mir-opt/{nrvo_simple.nrvo.RenameReturnPlace.panic-abort.diff => dest-prop/nrvo_borrowed.nrvo.DestinationPropagation.panic-abort.diff} (52%) rename tests/mir-opt/{nrvo_simple.nrvo.RenameReturnPlace.panic-unwind.diff => dest-prop/nrvo_borrowed.nrvo.DestinationPropagation.panic-unwind.diff} (52%) rename tests/mir-opt/{nrvo_simple.rs => dest-prop/nrvo_borrowed.rs} (69%) rename tests/mir-opt/{ => dest-prop}/nrvo_miscompile_111005.rs (79%) rename tests/mir-opt/{nrvo_miscompile_111005.wrong.RenameReturnPlace.diff => dest-prop/nrvo_miscompile_111005.wrong.DestinationPropagation.diff} (61%) diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index 4c94a6c524e00..6cc0a9c01d940 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -158,7 +158,7 @@ impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation { // 2. Despite being an overall perf improvement, this still causes a 30% regression in // keccak. We can temporarily fix this by bounding function size, but in the long term // we should fix this by being smarter about invalidating analysis results. - sess.mir_opt_level() >= 3 + sess.mir_opt_level() >= 2 } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 08f25276cecc1..23fc4aa5817b4 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -154,7 +154,6 @@ declare_passes! { mod match_branches : MatchBranchSimplification; mod mentioned_items : MentionedItems; mod multiple_return_terminators : MultipleReturnTerminators; - mod nrvo : RenameReturnPlace; mod post_drop_elaboration : CheckLiveDrops; mod prettify : ReorderBasicBlocks, ReorderLocals; mod promote_consts : PromoteTemps; @@ -710,7 +709,6 @@ pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<' &jump_threading::JumpThreading, &early_otherwise_branch::EarlyOtherwiseBranch, &simplify_comparison_integral::SimplifyComparisonIntegral, - &dest_prop::DestinationPropagation, &o1(simplify_branches::SimplifyConstCondition::Final), &o1(remove_noop_landing_pads::RemoveNoopLandingPads), &o1(simplify::SimplifyCfg::Final), @@ -718,7 +716,7 @@ pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<' &strip_debuginfo::StripDebugInfo, ©_prop::CopyProp, &dead_store_elimination::DeadStoreElimination::Final, - &nrvo::RenameReturnPlace, + &dest_prop::DestinationPropagation, &simplify::SimplifyLocals::Final, &multiple_return_terminators::MultipleReturnTerminators, &large_enums::EnumSizeOpt { discrepancy: 128 }, diff --git a/compiler/rustc_mir_transform/src/nrvo.rs b/compiler/rustc_mir_transform/src/nrvo.rs deleted file mode 100644 index 965002aae04c7..0000000000000 --- a/compiler/rustc_mir_transform/src/nrvo.rs +++ /dev/null @@ -1,234 +0,0 @@ -//! See the docs for [`RenameReturnPlace`]. - -use rustc_hir::Mutability; -use rustc_index::bit_set::DenseBitSet; -use rustc_middle::bug; -use rustc_middle::mir::visit::{MutVisitor, NonUseContext, PlaceContext, Visitor}; -use rustc_middle::mir::{self, BasicBlock, Local, Location}; -use rustc_middle::ty::TyCtxt; -use tracing::{debug, trace}; - -/// This pass looks for MIR that always copies the same local into the return place and eliminates -/// the copy by renaming all uses of that local to `_0`. -/// -/// This allows LLVM to perform an optimization similar to the named return value optimization -/// (NRVO) that is guaranteed in C++. This avoids a stack allocation and `memcpy` for the -/// relatively common pattern of allocating a buffer on the stack, mutating it, and returning it by -/// value like so: -/// -/// ```rust -/// fn foo(init: fn(&mut [u8; 1024])) -> [u8; 1024] { -/// let mut buf = [0; 1024]; -/// init(&mut buf); -/// buf -/// } -/// ``` -/// -/// For now, this pass is very simple and only capable of eliminating a single copy. A more general -/// version of copy propagation, such as the one based on non-overlapping live ranges in [#47954] and -/// [#71003], could yield even more benefits. -/// -/// [#47954]: https://github.com/rust-lang/rust/pull/47954 -/// [#71003]: https://github.com/rust-lang/rust/pull/71003 -pub(super) struct RenameReturnPlace; - -impl<'tcx> crate::MirPass<'tcx> for RenameReturnPlace { - fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - // unsound: #111005 - sess.mir_opt_level() > 0 && sess.opts.unstable_opts.unsound_mir_opts - } - - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut mir::Body<'tcx>) { - let def_id = body.source.def_id(); - let Some(returned_local) = local_eligible_for_nrvo(body) else { - debug!("`{:?}` was ineligible for NRVO", def_id); - return; - }; - - debug!( - "`{:?}` was eligible for NRVO, making {:?} the return place", - def_id, returned_local - ); - - RenameToReturnPlace { tcx, to_rename: returned_local }.visit_body_preserves_cfg(body); - - // Clean up the `NOP`s we inserted for statements made useless by our renaming. - for block_data in body.basic_blocks.as_mut_preserves_cfg() { - block_data.statements.retain(|stmt| stmt.kind != mir::StatementKind::Nop); - } - - // Overwrite the debuginfo of `_0` with that of the renamed local. - let (renamed_decl, ret_decl) = - body.local_decls.pick2_mut(returned_local, mir::RETURN_PLACE); - - // Sometimes, the return place is assigned a local of a different but coercible type, for - // example `&mut T` instead of `&T`. Overwriting the `LocalInfo` for the return place means - // its type may no longer match the return type of its function. This doesn't cause a - // problem in codegen because these two types are layout-compatible, but may be unexpected. - debug!("_0: {:?} = {:?}: {:?}", ret_decl.ty, returned_local, renamed_decl.ty); - ret_decl.clone_from(renamed_decl); - - // The return place is always mutable. - ret_decl.mutability = Mutability::Mut; - } - - fn is_required(&self) -> bool { - false - } -} - -/// MIR that is eligible for the NRVO must fulfill two conditions: -/// 1. The return place must not be read prior to the `Return` terminator. -/// 2. A simple assignment of a whole local to the return place (e.g., `_0 = _1`) must be the -/// only definition of the return place reaching the `Return` terminator. -/// -/// If the MIR fulfills both these conditions, this function returns the `Local` that is assigned -/// to the return place along all possible paths through the control-flow graph. -fn local_eligible_for_nrvo(body: &mir::Body<'_>) -> Option { - if IsReturnPlaceRead::run(body) { - return None; - } - - let mut copied_to_return_place = None; - for block in body.basic_blocks.indices() { - // Look for blocks with a `Return` terminator. - if !matches!(body[block].terminator().kind, mir::TerminatorKind::Return) { - continue; - } - - // Look for an assignment of a single local to the return place prior to the `Return`. - let returned_local = find_local_assigned_to_return_place(block, body)?; - match body.local_kind(returned_local) { - // FIXME: Can we do this for arguments as well? - mir::LocalKind::Arg => return None, - - mir::LocalKind::ReturnPointer => bug!("Return place was assigned to itself?"), - mir::LocalKind::Temp => {} - } - - // If multiple different locals are copied to the return place. We can't pick a - // single one to rename. - if copied_to_return_place.is_some_and(|old| old != returned_local) { - return None; - } - - copied_to_return_place = Some(returned_local); - } - - copied_to_return_place -} - -fn find_local_assigned_to_return_place(start: BasicBlock, body: &mir::Body<'_>) -> Option { - let mut block = start; - let mut seen = DenseBitSet::new_empty(body.basic_blocks.len()); - - // Iterate as long as `block` has exactly one predecessor that we have not yet visited. - while seen.insert(block) { - trace!("Looking for assignments to `_0` in {:?}", block); - - let local = body[block].statements.iter().rev().find_map(as_local_assigned_to_return_place); - if local.is_some() { - return local; - } - - match body.basic_blocks.predecessors()[block].as_slice() { - &[pred] => block = pred, - _ => return None, - } - } - - None -} - -// If this statement is an assignment of an unprojected local to the return place, -// return that local. -fn as_local_assigned_to_return_place(stmt: &mir::Statement<'_>) -> Option { - if let mir::StatementKind::Assign(box (lhs, rhs)) = &stmt.kind { - if lhs.as_local() == Some(mir::RETURN_PLACE) { - if let mir::Rvalue::Use(mir::Operand::Copy(rhs) | mir::Operand::Move(rhs)) = rhs { - return rhs.as_local(); - } - } - } - - None -} - -struct RenameToReturnPlace<'tcx> { - to_rename: Local, - tcx: TyCtxt<'tcx>, -} - -/// Replaces all uses of `self.to_rename` with `_0`. -impl<'tcx> MutVisitor<'tcx> for RenameToReturnPlace<'tcx> { - fn tcx(&self) -> TyCtxt<'tcx> { - self.tcx - } - - fn visit_statement(&mut self, stmt: &mut mir::Statement<'tcx>, loc: Location) { - // Remove assignments of the local being replaced to the return place, since it is now the - // return place: - // _0 = _1 - if as_local_assigned_to_return_place(stmt) == Some(self.to_rename) { - stmt.kind = mir::StatementKind::Nop; - return; - } - - // Remove storage annotations for the local being replaced: - // StorageLive(_1) - if let mir::StatementKind::StorageLive(local) | mir::StatementKind::StorageDead(local) = - stmt.kind - { - if local == self.to_rename { - stmt.kind = mir::StatementKind::Nop; - return; - } - } - - self.super_statement(stmt, loc) - } - - fn visit_terminator(&mut self, terminator: &mut mir::Terminator<'tcx>, loc: Location) { - // Ignore the implicit "use" of the return place in a `Return` statement. - if let mir::TerminatorKind::Return = terminator.kind { - return; - } - - self.super_terminator(terminator, loc); - } - - fn visit_local(&mut self, l: &mut Local, ctxt: PlaceContext, _: Location) { - if *l == mir::RETURN_PLACE { - assert_eq!(ctxt, PlaceContext::NonUse(NonUseContext::VarDebugInfo)); - } else if *l == self.to_rename { - *l = mir::RETURN_PLACE; - } - } -} - -struct IsReturnPlaceRead(bool); - -impl IsReturnPlaceRead { - fn run(body: &mir::Body<'_>) -> bool { - let mut vis = IsReturnPlaceRead(false); - vis.visit_body(body); - vis.0 - } -} - -impl<'tcx> Visitor<'tcx> for IsReturnPlaceRead { - fn visit_local(&mut self, l: Local, ctxt: PlaceContext, _: Location) { - if l == mir::RETURN_PLACE && ctxt.is_use() && !ctxt.is_place_assignment() { - self.0 = true; - } - } - - fn visit_terminator(&mut self, terminator: &mir::Terminator<'tcx>, loc: Location) { - // Ignore the implicit "use" of the return place in a `Return` statement. - if let mir::TerminatorKind::Return = terminator.kind { - return; - } - - self.super_terminator(terminator, loc); - } -} diff --git a/tests/mir-opt/nrvo_simple.nrvo.RenameReturnPlace.panic-abort.diff b/tests/mir-opt/dest-prop/nrvo_borrowed.nrvo.DestinationPropagation.panic-abort.diff similarity index 52% rename from tests/mir-opt/nrvo_simple.nrvo.RenameReturnPlace.panic-abort.diff rename to tests/mir-opt/dest-prop/nrvo_borrowed.nrvo.DestinationPropagation.panic-abort.diff index 6dce3ec5303d7..e9fbcf20a7228 100644 --- a/tests/mir-opt/nrvo_simple.nrvo.RenameReturnPlace.panic-abort.diff +++ b/tests/mir-opt/dest-prop/nrvo_borrowed.nrvo.DestinationPropagation.panic-abort.diff @@ -1,5 +1,5 @@ -- // MIR for `nrvo` before RenameReturnPlace -+ // MIR for `nrvo` after RenameReturnPlace +- // MIR for `nrvo` before DestinationPropagation ++ // MIR for `nrvo` after DestinationPropagation fn nrvo(_1: for<'a> fn(&'a mut [u8; 1024])) -> [u8; 1024] { debug init => _1; @@ -10,32 +10,33 @@ let mut _5: &mut [u8; 1024]; let mut _6: &mut [u8; 1024]; scope 1 { -- debug buf => _2; -+ debug buf => _0; + debug buf => _2; } bb0: { -- StorageLive(_2); -- _2 = [const 0_u8; 1024]; -+ _0 = [const 0_u8; 1024]; + StorageLive(_2); + _2 = [const 0_u8; 1024]; StorageLive(_3); - StorageLive(_4); - _4 = copy _1; +- StorageLive(_4); +- _4 = copy _1; ++ nop; ++ nop; StorageLive(_5); StorageLive(_6); -- _6 = &mut _2; -+ _6 = &mut _0; + _6 = &mut _2; _5 = &mut (*_6); - _3 = move _4(move _5) -> [return: bb1, unwind unreachable]; +- _3 = move _4(move _5) -> [return: bb1, unwind unreachable]; ++ _3 = move _1(move _5) -> [return: bb1, unwind unreachable]; } bb1: { StorageDead(_5); - StorageDead(_4); +- StorageDead(_4); ++ nop; StorageDead(_6); StorageDead(_3); -- _0 = copy _2; -- StorageDead(_2); + _0 = copy _2; + StorageDead(_2); return; } } diff --git a/tests/mir-opt/nrvo_simple.nrvo.RenameReturnPlace.panic-unwind.diff b/tests/mir-opt/dest-prop/nrvo_borrowed.nrvo.DestinationPropagation.panic-unwind.diff similarity index 52% rename from tests/mir-opt/nrvo_simple.nrvo.RenameReturnPlace.panic-unwind.diff rename to tests/mir-opt/dest-prop/nrvo_borrowed.nrvo.DestinationPropagation.panic-unwind.diff index 54cbe2871f15a..95d5fe1b9304b 100644 --- a/tests/mir-opt/nrvo_simple.nrvo.RenameReturnPlace.panic-unwind.diff +++ b/tests/mir-opt/dest-prop/nrvo_borrowed.nrvo.DestinationPropagation.panic-unwind.diff @@ -1,5 +1,5 @@ -- // MIR for `nrvo` before RenameReturnPlace -+ // MIR for `nrvo` after RenameReturnPlace +- // MIR for `nrvo` before DestinationPropagation ++ // MIR for `nrvo` after DestinationPropagation fn nrvo(_1: for<'a> fn(&'a mut [u8; 1024])) -> [u8; 1024] { debug init => _1; @@ -10,32 +10,33 @@ let mut _5: &mut [u8; 1024]; let mut _6: &mut [u8; 1024]; scope 1 { -- debug buf => _2; -+ debug buf => _0; + debug buf => _2; } bb0: { -- StorageLive(_2); -- _2 = [const 0_u8; 1024]; -+ _0 = [const 0_u8; 1024]; + StorageLive(_2); + _2 = [const 0_u8; 1024]; StorageLive(_3); - StorageLive(_4); - _4 = copy _1; +- StorageLive(_4); +- _4 = copy _1; ++ nop; ++ nop; StorageLive(_5); StorageLive(_6); -- _6 = &mut _2; -+ _6 = &mut _0; + _6 = &mut _2; _5 = &mut (*_6); - _3 = move _4(move _5) -> [return: bb1, unwind continue]; +- _3 = move _4(move _5) -> [return: bb1, unwind continue]; ++ _3 = move _1(move _5) -> [return: bb1, unwind continue]; } bb1: { StorageDead(_5); - StorageDead(_4); +- StorageDead(_4); ++ nop; StorageDead(_6); StorageDead(_3); -- _0 = copy _2; -- StorageDead(_2); + _0 = copy _2; + StorageDead(_2); return; } } diff --git a/tests/mir-opt/nrvo_simple.rs b/tests/mir-opt/dest-prop/nrvo_borrowed.rs similarity index 69% rename from tests/mir-opt/nrvo_simple.rs rename to tests/mir-opt/dest-prop/nrvo_borrowed.rs index df540472e1ccb..6f3076d4c13e9 100644 --- a/tests/mir-opt/nrvo_simple.rs +++ b/tests/mir-opt/dest-prop/nrvo_borrowed.rs @@ -1,8 +1,8 @@ // skip-filecheck // EMIT_MIR_FOR_EACH_PANIC_STRATEGY -//@ test-mir-pass: RenameReturnPlace +//@ test-mir-pass: DestinationPropagation -// EMIT_MIR nrvo_simple.nrvo.RenameReturnPlace.diff +// EMIT_MIR nrvo_borrowed.nrvo.DestinationPropagation.diff fn nrvo(init: fn(&mut [u8; 1024])) -> [u8; 1024] { let mut buf = [0; 1024]; init(&mut buf); diff --git a/tests/mir-opt/nrvo_miscompile_111005.rs b/tests/mir-opt/dest-prop/nrvo_miscompile_111005.rs similarity index 79% rename from tests/mir-opt/nrvo_miscompile_111005.rs rename to tests/mir-opt/dest-prop/nrvo_miscompile_111005.rs index 03008fa819156..22d4a8b59c5ce 100644 --- a/tests/mir-opt/nrvo_miscompile_111005.rs +++ b/tests/mir-opt/dest-prop/nrvo_miscompile_111005.rs @@ -1,13 +1,13 @@ // skip-filecheck // This is a miscompilation, #111005 to track -//@ test-mir-pass: RenameReturnPlace +//@ test-mir-pass: DestinationPropagation #![feature(custom_mir, core_intrinsics)] extern crate core; use core::intrinsics::mir::*; -// EMIT_MIR nrvo_miscompile_111005.wrong.RenameReturnPlace.diff +// EMIT_MIR nrvo_miscompile_111005.wrong.DestinationPropagation.diff #[custom_mir(dialect = "runtime", phase = "initial")] pub fn wrong(arg: char) -> char { mir! { diff --git a/tests/mir-opt/nrvo_miscompile_111005.wrong.RenameReturnPlace.diff b/tests/mir-opt/dest-prop/nrvo_miscompile_111005.wrong.DestinationPropagation.diff similarity index 61% rename from tests/mir-opt/nrvo_miscompile_111005.wrong.RenameReturnPlace.diff rename to tests/mir-opt/dest-prop/nrvo_miscompile_111005.wrong.DestinationPropagation.diff index d248c76f261b6..60cf9236f6c2a 100644 --- a/tests/mir-opt/nrvo_miscompile_111005.wrong.RenameReturnPlace.diff +++ b/tests/mir-opt/dest-prop/nrvo_miscompile_111005.wrong.DestinationPropagation.diff @@ -1,5 +1,5 @@ -- // MIR for `wrong` before RenameReturnPlace -+ // MIR for `wrong` after RenameReturnPlace +- // MIR for `wrong` before DestinationPropagation ++ // MIR for `wrong` after DestinationPropagation fn wrong(_1: char) -> char { let mut _0: char; @@ -9,8 +9,9 @@ - _2 = copy _1; - _0 = copy _2; - _2 = const 'b'; ++ nop; + _0 = copy _1; -+ _0 = const 'b'; ++ _1 = const 'b'; return; } } diff --git a/tests/mir-opt/pre-codegen/checked_ops.rs b/tests/mir-opt/pre-codegen/checked_ops.rs index 8fd340503f548..dfbc0f4a1108d 100644 --- a/tests/mir-opt/pre-codegen/checked_ops.rs +++ b/tests/mir-opt/pre-codegen/checked_ops.rs @@ -44,8 +44,7 @@ pub fn saturating_sub_at_home(lhs: u32, rhs: u32) -> u32 { // CHECK-LABEL: fn saturating_sub_at_home // CHECK: [[DELTA:_[0-9]+]] = SubUnchecked(copy _1, copy _2) // CHECK: [[TEMP1:_.+]] = Option::::Some({{move|copy}} [[DELTA]]); - // CHECK: [[TEMP2:_.+]] = {{move|copy}} (([[TEMP1]] as Some).0: u32); - // CHECK: _0 = {{move|copy}} [[TEMP2]]; + // CHECK: _0 = {{move|copy}} (([[TEMP1]] as Some).0: u32); u32::checked_sub(lhs, rhs).unwrap_or(0) } diff --git a/tests/mir-opt/pre-codegen/checked_ops.saturating_sub_at_home.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/checked_ops.saturating_sub_at_home.PreCodegen.after.panic-abort.mir index 5b4fdeda85731..dc7567b57e70a 100644 --- a/tests/mir-opt/pre-codegen/checked_ops.saturating_sub_at_home.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/checked_ops.saturating_sub_at_home.PreCodegen.after.panic-abort.mir @@ -10,7 +10,6 @@ fn saturating_sub_at_home(_1: u32, _2: u32) -> u32 { let mut _4: u32; } scope 2 (inlined Option::::unwrap_or) { - let _6: u32; scope 3 { } } @@ -28,10 +27,7 @@ fn saturating_sub_at_home(_1: u32, _2: u32) -> u32 { _5 = Option::::Some(move _4); StorageDead(_4); StorageDead(_3); - StorageLive(_6); - _6 = move ((_5 as Some).0: u32); - _0 = move _6; - StorageDead(_6); + _0 = move ((_5 as Some).0: u32); goto -> bb3; } diff --git a/tests/mir-opt/pre-codegen/checked_ops.saturating_sub_at_home.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/checked_ops.saturating_sub_at_home.PreCodegen.after.panic-unwind.mir index 5b4fdeda85731..dc7567b57e70a 100644 --- a/tests/mir-opt/pre-codegen/checked_ops.saturating_sub_at_home.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/checked_ops.saturating_sub_at_home.PreCodegen.after.panic-unwind.mir @@ -10,7 +10,6 @@ fn saturating_sub_at_home(_1: u32, _2: u32) -> u32 { let mut _4: u32; } scope 2 (inlined Option::::unwrap_or) { - let _6: u32; scope 3 { } } @@ -28,10 +27,7 @@ fn saturating_sub_at_home(_1: u32, _2: u32) -> u32 { _5 = Option::::Some(move _4); StorageDead(_4); StorageDead(_3); - StorageLive(_6); - _6 = move ((_5 as Some).0: u32); - _0 = move _6; - StorageDead(_6); + _0 = move ((_5 as Some).0: u32); goto -> bb3; } diff --git a/tests/mir-opt/pre-codegen/clone_as_copy.enum_clone_as_copy.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/clone_as_copy.enum_clone_as_copy.PreCodegen.after.mir index 9f88e1961ec88..e67f362ee04ae 100644 --- a/tests/mir-opt/pre-codegen/clone_as_copy.enum_clone_as_copy.PreCodegen.after.mir +++ b/tests/mir-opt/pre-codegen/clone_as_copy.enum_clone_as_copy.PreCodegen.after.mir @@ -6,8 +6,8 @@ fn enum_clone_as_copy(_1: &Enum1) -> Enum1 { scope 1 (inlined ::clone) { debug self => _1; let mut _2: isize; - let mut _3: &AllCopy; - let mut _4: &NestCopy; + let _3: &AllCopy; + let _4: &NestCopy; scope 2 { debug __self_0 => _3; scope 6 (inlined ::clone) { diff --git a/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir index 8746cb0899166..e235fa35c0238 100644 --- a/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir +++ b/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir @@ -5,8 +5,9 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool { debug b => _2; let mut _0: bool; scope 1 (inlined ::le) { - let mut _11: std::option::Option; + let mut _6: std::option::Option; scope 2 (inlined Option::::is_some_and:: bool {std::cmp::Ordering::is_le}>) { + let mut _11: isize; let _12: std::cmp::Ordering; scope 3 { scope 4 (inlined bool {std::cmp::Ordering::is_le} as FnOnce<(std::cmp::Ordering,)>>::call_once - shim(fn(std::cmp::Ordering) -> bool {std::cmp::Ordering::is_le})) { @@ -19,7 +20,6 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool { } } scope 7 (inlined ::partial_cmp) { - let mut _6: std::option::Option; let mut _7: i8; scope 8 { } @@ -38,9 +38,8 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool { bb0: { StorageLive(_12); - StorageLive(_11); - StorageLive(_5); StorageLive(_6); + StorageLive(_5); StorageLive(_7); StorageLive(_3); _3 = copy ((*_1).0: char); @@ -63,30 +62,44 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool { _10 = Cmp(move _8, move _9); StorageDead(_9); StorageDead(_8); - _11 = Option::::Some(move _10); + _6 = Option::::Some(move _10); StorageDead(_10); StorageDead(_7); - StorageDead(_6); StorageDead(_5); - goto -> bb3; + StorageLive(_11); + goto -> bb4; } bb2: { - _11 = copy _6; StorageDead(_7); - StorageDead(_6); StorageDead(_5); - goto -> bb3; + StorageLive(_11); + _11 = discriminant(_6); + switchInt(move _11) -> [0: bb3, 1: bb4, otherwise: bb6]; } bb3: { - _12 = move ((_11 as Some).0: std::cmp::Ordering); + _0 = const false; + goto -> bb5; + } + + bb4: { + _12 = move ((_6 as Some).0: std::cmp::Ordering); StorageLive(_13); _13 = discriminant(_12); _0 = Le(move _13, const 0_i8); StorageDead(_13); + goto -> bb5; + } + + bb5: { StorageDead(_11); + StorageDead(_6); StorageDead(_12); return; } + + bb6: { + unreachable; + } } diff --git a/tests/mir-opt/pre-codegen/derived_ord.rs b/tests/mir-opt/pre-codegen/derived_ord.rs index 73ae923a6cb9c..823e0f6d09c2b 100644 --- a/tests/mir-opt/pre-codegen/derived_ord.rs +++ b/tests/mir-opt/pre-codegen/derived_ord.rs @@ -25,7 +25,10 @@ pub fn demo_le(a: &MultiField, b: &MultiField) -> bool { // CHECK: Cmp(move [[A1]], move [[B1]]); // CHECK: [[D1:_[0-9]+]] = discriminant({{.+}}); - // CHECK: _0 = Le(move [[D1]], const 0_i8); + // CHECK: switchInt(move [[D1]]) -> [0: bb{{[0-9]+}}, 1: bb{{[0-9]+}}, otherwise: bb{{[0-9]+}}]; + + // CHECK: [[D2:_[0-9]+]] = discriminant({{.+}}); + // CHECK: _0 = Le(move [[D2]], const 0_i8); *a <= *b } diff --git a/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir index de25eebee77ff..5993bd79d274c 100644 --- a/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir +++ b/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir @@ -4,10 +4,9 @@ fn ::partial_cmp(_1: &MultiField, _2: &M debug self => _1; debug other => _2; let mut _0: std::option::Option; - let mut _6: std::option::Option; - let mut _7: i8; + let mut _6: i8; scope 1 { - debug cmp => _6; + debug cmp => _0; } scope 2 (inlined std::cmp::impls::::partial_cmp) { let mut _3: char; @@ -15,9 +14,9 @@ fn ::partial_cmp(_1: &MultiField, _2: &M let mut _5: std::cmp::Ordering; } scope 3 (inlined std::cmp::impls::::partial_cmp) { + let mut _7: i16; let mut _8: i16; - let mut _9: i16; - let mut _10: std::cmp::Ordering; + let mut _9: std::cmp::Ordering; } bb0: { @@ -28,27 +27,26 @@ fn ::partial_cmp(_1: &MultiField, _2: &M _5 = Cmp(move _3, move _4); StorageDead(_4); StorageDead(_3); - _6 = Option::::Some(copy _5); - _7 = discriminant(_5); - switchInt(move _7) -> [0: bb1, otherwise: bb2]; + _0 = Option::::Some(copy _5); + _6 = discriminant(_5); + switchInt(move _6) -> [0: bb1, otherwise: bb2]; } bb1: { - StorageLive(_10); - StorageLive(_8); - _8 = copy ((*_1).1: i16); StorageLive(_9); - _9 = copy ((*_2).1: i16); - _10 = Cmp(move _8, move _9); - StorageDead(_9); + StorageLive(_7); + _7 = copy ((*_1).1: i16); + StorageLive(_8); + _8 = copy ((*_2).1: i16); + _9 = Cmp(move _7, move _8); StorageDead(_8); - _0 = Option::::Some(move _10); - StorageDead(_10); + StorageDead(_7); + _0 = Option::::Some(move _9); + StorageDead(_9); goto -> bb3; } bb2: { - _0 = copy _6; goto -> bb3; } diff --git a/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-abort.mir index dfe618612ab96..03a52b82b49b1 100644 --- a/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-abort.mir @@ -5,23 +5,21 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { debug end => _2; debug f => _3; let mut _0: (); - let mut _4: u32; - let mut _9: std::option::Option; - let mut _11: &impl Fn(u32); - let mut _12: (u32,); - let _13: (); + let mut _7: std::option::Option; + let mut _9: &impl Fn(u32); + let mut _10: (u32,); + let _11: (); scope 1 { - debug ((iter: std::ops::Range).0: u32) => _4; + debug ((iter: std::ops::Range).0: u32) => _1; debug ((iter: std::ops::Range).1: u32) => _2; - let _10: u32; + let _8: u32; scope 2 { - debug x => _10; + debug x => _8; } scope 4 (inlined iter::range::>::next) { scope 5 (inlined as iter::range::RangeIteratorImpl>::spec_next) { - let mut _6: bool; - let _7: u32; - let mut _8: u32; + let mut _5: bool; + let _6: u32; scope 6 { scope 8 (inlined ::forward_unchecked) { scope 9 (inlined #[track_caller] core::num::::unchecked_add) { @@ -33,7 +31,7 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { } } scope 7 (inlined std::cmp::impls::::lt) { - let mut _5: u32; + let mut _4: u32; } } } @@ -42,25 +40,22 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { } bb0: { - StorageLive(_4); - _4 = copy _1; goto -> bb1; } bb1: { - StorageLive(_9); - StorageLive(_6); + StorageLive(_7); StorageLive(_5); - _5 = copy _4; - _6 = Lt(move _5, copy _2); - StorageDead(_5); - switchInt(move _6) -> [0: bb2, otherwise: bb4]; + StorageLive(_4); + _4 = copy _1; + _5 = Lt(move _4, copy _2); + StorageDead(_4); + switchInt(move _5) -> [0: bb2, otherwise: bb4]; } bb2: { - StorageDead(_6); - StorageDead(_9); - StorageDead(_4); + StorageDead(_5); + StorageDead(_7); drop(_3) -> [return: bb3, unwind unreachable]; } @@ -69,25 +64,22 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { } bb4: { - _7 = copy _4; - StorageLive(_8); - _8 = AddUnchecked(copy _7, const 1_u32); - _4 = move _8; - StorageDead(_8); - _9 = Option::::Some(copy _7); - StorageDead(_6); - _10 = copy ((_9 as Some).0: u32); - StorageLive(_11); - _11 = &_3; - StorageLive(_12); - _12 = (copy _10,); - _13 = >::call(move _11, move _12) -> [return: bb5, unwind unreachable]; + _6 = copy _1; + _1 = AddUnchecked(copy _6, const 1_u32); + _7 = Option::::Some(copy _6); + StorageDead(_5); + _8 = copy ((_7 as Some).0: u32); + StorageLive(_9); + _9 = &_3; + StorageLive(_10); + _10 = (copy _8,); + _11 = >::call(move _9, move _10) -> [return: bb5, unwind unreachable]; } bb5: { - StorageDead(_12); - StorageDead(_11); + StorageDead(_10); StorageDead(_9); + StorageDead(_7); goto -> bb1; } } diff --git a/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-unwind.mir index e0fcfcaffc59a..3b09f33e73338 100644 --- a/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-unwind.mir @@ -5,23 +5,21 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { debug end => _2; debug f => _3; let mut _0: (); - let mut _4: u32; - let mut _9: std::option::Option; - let mut _11: &impl Fn(u32); - let mut _12: (u32,); - let _13: (); + let mut _7: std::option::Option; + let mut _9: &impl Fn(u32); + let mut _10: (u32,); + let _11: (); scope 1 { - debug ((iter: std::ops::Range).0: u32) => _4; + debug ((iter: std::ops::Range).0: u32) => _1; debug ((iter: std::ops::Range).1: u32) => _2; - let _10: u32; + let _8: u32; scope 2 { - debug x => _10; + debug x => _8; } scope 4 (inlined iter::range::>::next) { scope 5 (inlined as iter::range::RangeIteratorImpl>::spec_next) { - let mut _6: bool; - let _7: u32; - let mut _8: u32; + let mut _5: bool; + let _6: u32; scope 6 { scope 8 (inlined ::forward_unchecked) { scope 9 (inlined #[track_caller] core::num::::unchecked_add) { @@ -33,7 +31,7 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { } } scope 7 (inlined std::cmp::impls::::lt) { - let mut _5: u32; + let mut _4: u32; } } } @@ -42,25 +40,22 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { } bb0: { - StorageLive(_4); - _4 = copy _1; goto -> bb1; } bb1: { - StorageLive(_9); - StorageLive(_6); + StorageLive(_7); StorageLive(_5); - _5 = copy _4; - _6 = Lt(move _5, copy _2); - StorageDead(_5); - switchInt(move _6) -> [0: bb2, otherwise: bb4]; + StorageLive(_4); + _4 = copy _1; + _5 = Lt(move _4, copy _2); + StorageDead(_4); + switchInt(move _5) -> [0: bb2, otherwise: bb4]; } bb2: { - StorageDead(_6); - StorageDead(_9); - StorageDead(_4); + StorageDead(_5); + StorageDead(_7); drop(_3) -> [return: bb3, unwind continue]; } @@ -69,25 +64,22 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { } bb4: { - _7 = copy _4; - StorageLive(_8); - _8 = AddUnchecked(copy _7, const 1_u32); - _4 = move _8; - StorageDead(_8); - _9 = Option::::Some(copy _7); - StorageDead(_6); - _10 = copy ((_9 as Some).0: u32); - StorageLive(_11); - _11 = &_3; - StorageLive(_12); - _12 = (copy _10,); - _13 = >::call(move _11, move _12) -> [return: bb5, unwind: bb6]; + _6 = copy _1; + _1 = AddUnchecked(copy _6, const 1_u32); + _7 = Option::::Some(copy _6); + StorageDead(_5); + _8 = copy ((_7 as Some).0: u32); + StorageLive(_9); + _9 = &_3; + StorageLive(_10); + _10 = (copy _8,); + _11 = >::call(move _9, move _10) -> [return: bb5, unwind: bb6]; } bb5: { - StorageDead(_12); - StorageDead(_11); + StorageDead(_10); StorageDead(_9); + StorageDead(_7); goto -> bb1; } diff --git a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir index 72b39a7f6a87b..0a09a0fa0b2d3 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir @@ -4,30 +4,30 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { debug slice => _1; debug f => _2; let mut _0: (); - let mut _11: std::ptr::NonNull; - let mut _12: *const T; - let mut _13: usize; - let mut _32: std::option::Option<(usize, &T)>; - let mut _35: &impl Fn(usize, &T); - let mut _36: (usize, &T); - let _37: (); + let mut _6: std::ptr::NonNull; + let mut _9: *const T; + let mut _10: usize; + let mut _27: std::option::Option<(usize, &T)>; + let mut _30: &impl Fn(usize, &T); + let mut _31: (usize, &T); + let _32: (); scope 1 { - debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _11; - debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).1: *const T) => _12; + debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _6; + debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).1: *const T) => _9; debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>; - debug ((iter: Enumerate>).1: usize) => _13; - let _33: usize; - let _34: &T; + debug ((iter: Enumerate>).1: usize) => _10; + let _28: usize; + let _29: &T; scope 2 { - debug i => _33; - debug x => _34; + debug i => _28; + debug x => _29; } scope 18 (inlined > as Iterator>::next) { - let mut _27: std::option::Option<&T>; - let mut _30: (usize, bool); - let mut _31: (usize, &T); + let mut _22: std::option::Option<&T>; + let mut _25: (usize, bool); + let mut _26: (usize, &T); scope 19 { - let _29: usize; + let _24: usize; scope 24 { } } @@ -42,21 +42,19 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } } scope 25 (inlined as Try>::branch) { - let _28: &T; + let _23: &T; scope 26 { } } scope 28 (inlined as Iterator>::next) { - let _14: std::ptr::NonNull; - let _16: std::ptr::NonNull; - let mut _19: bool; - let mut _22: std::ptr::NonNull; - let mut _24: usize; - let _26: &T; + let _11: std::ptr::NonNull; + let _12: std::ptr::NonNull; + let mut _15: bool; + let mut _19: usize; + let _21: &T; scope 29 { - let _15: *const T; scope 30 { - let _23: usize; + let _18: usize; scope 31 { scope 34 (inlined #[track_caller] core::num::::unchecked_sub) { scope 35 (inlined core::ub_checks::check_language_ub) { @@ -72,21 +70,21 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } } scope 38 (inlined as PartialEq>::eq) { - let mut _17: *mut T; - let mut _18: *mut T; + let mut _13: *mut T; + let mut _14: *mut T; scope 39 (inlined NonNull::::as_ptr) { } scope 40 (inlined NonNull::::as_ptr) { } } scope 41 (inlined NonNull::::add) { - let mut _20: *const T; - let mut _21: *const T; + let mut _16: *const T; + let mut _17: *const T; scope 42 (inlined NonNull::::as_ptr) { } } scope 43 (inlined NonNull::::as_ref::<'_>) { - let _25: *const T; + let _20: *const T; scope 44 (inlined NonNull::::as_ptr) { } scope 45 (inlined std::ptr::mut_ptr::::cast_const) { @@ -102,11 +100,8 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { let _3: usize; let mut _7: *mut T; let mut _8: *mut T; - let mut _10: *const T; scope 5 { - let _6: std::ptr::NonNull; scope 6 { - let _9: *const T; scope 7 { } scope 11 (inlined std::ptr::without_provenance::) { @@ -145,7 +140,6 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { _5 = copy _4 as *const T (PtrToPtr); _6 = NonNull:: { pointer: copy _5 }; StorageDead(_5); - StorageLive(_9); switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2]; } @@ -166,97 +160,82 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } bb3: { - _10 = copy _9; - StorageDead(_9); StorageDead(_4); StorageDead(_3); - StorageLive(_11); - StorageLive(_12); - StorageLive(_13); - _11 = copy _6; - _12 = copy _10; - _13 = const 0_usize; + StorageLive(_10); + _10 = const 0_usize; goto -> bb4; } bb4: { - StorageLive(_32); - StorageLive(_29); - StorageLive(_30); StorageLive(_27); - StorageLive(_14); - StorageLive(_15); - StorageLive(_23); StorageLive(_24); - StorageLive(_16); - StorageLive(_26); - _14 = copy _11; - _15 = copy _12; + StorageLive(_25); + StorageLive(_22); + StorageLive(_11); + StorageLive(_18); + StorageLive(_19); + StorageLive(_12); + StorageLive(_21); + _11 = copy _6; switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb8]; } bb5: { - StorageLive(_19); - _16 = copy _15 as std::ptr::NonNull (Transmute); - StorageLive(_17); - _17 = copy _14 as *mut T (Transmute); - StorageLive(_18); - _18 = copy _16 as *mut T (Transmute); - _19 = Eq(copy _17, copy _18); - StorageDead(_18); - StorageDead(_17); - switchInt(move _19) -> [0: bb6, otherwise: bb7]; + StorageLive(_15); + _12 = copy _9 as std::ptr::NonNull (Transmute); + StorageLive(_13); + _13 = copy _11 as *mut T (Transmute); + StorageLive(_14); + _14 = copy _12 as *mut T (Transmute); + _15 = Eq(copy _13, copy _14); + StorageDead(_14); + StorageDead(_13); + switchInt(move _15) -> [0: bb6, otherwise: bb7]; } bb6: { - StorageDead(_19); - StorageLive(_22); - StorageLive(_21); - StorageLive(_20); - _20 = copy _14 as *const T (Transmute); - _21 = Offset(copy _20, const 1_usize); - StorageDead(_20); - _22 = NonNull:: { pointer: copy _21 }; - StorageDead(_21); - _11 = move _22; - StorageDead(_22); + StorageDead(_15); + StorageLive(_17); + StorageLive(_16); + _16 = copy _11 as *const T (Transmute); + _17 = Offset(copy _16, const 1_usize); + StorageDead(_16); + _6 = NonNull:: { pointer: copy _17 }; + StorageDead(_17); goto -> bb13; } bb7: { - StorageDead(_19); - StorageDead(_26); - StorageDead(_16); - StorageDead(_24); - StorageDead(_23); StorageDead(_15); - StorageDead(_14); + StorageDead(_21); + StorageDead(_12); + StorageDead(_19); + StorageDead(_18); + StorageDead(_11); goto -> bb10; } bb8: { - _23 = copy _15 as usize (Transmute); - switchInt(copy _23) -> [0: bb9, otherwise: bb12]; + _18 = copy _9 as usize (Transmute); + switchInt(copy _18) -> [0: bb9, otherwise: bb12]; } bb9: { - StorageDead(_26); - StorageDead(_16); - StorageDead(_24); - StorageDead(_23); - StorageDead(_15); - StorageDead(_14); + StorageDead(_21); + StorageDead(_12); + StorageDead(_19); + StorageDead(_18); + StorageDead(_11); goto -> bb10; } bb10: { + StorageDead(_22); + StorageDead(_25); + StorageDead(_24); StorageDead(_27); - StorageDead(_30); - StorageDead(_29); - StorageDead(_32); - StorageDead(_11); - StorageDead(_12); - StorageDead(_13); + StorageDead(_10); drop(_2) -> [return: bb11, unwind unreachable]; } @@ -265,51 +244,50 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } bb12: { - _24 = SubUnchecked(copy _23, const 1_usize); - _12 = copy _24 as *const T (Transmute); + _19 = SubUnchecked(copy _18, const 1_usize); + _9 = copy _19 as *const T (Transmute); goto -> bb13; } bb13: { - StorageLive(_25); - _25 = copy _14 as *const T (Transmute); - _26 = &(*_25); - StorageDead(_25); - _27 = Option::<&T>::Some(copy _26); - StorageDead(_26); - StorageDead(_16); - StorageDead(_24); - StorageDead(_23); - StorageDead(_15); - StorageDead(_14); - _28 = copy ((_27 as Some).0: &T); - StorageDead(_27); - _29 = copy _13; - _30 = AddWithOverflow(copy _13, const 1_usize); - assert(!move (_30.1: bool), "attempt to compute `{} + {}`, which would overflow", copy _13, const 1_usize) -> [success: bb14, unwind unreachable]; + StorageLive(_20); + _20 = copy _11 as *const T (Transmute); + _21 = &(*_20); + StorageDead(_20); + _22 = Option::<&T>::Some(copy _21); + StorageDead(_21); + StorageDead(_12); + StorageDead(_19); + StorageDead(_18); + StorageDead(_11); + _23 = copy ((_22 as Some).0: &T); + StorageDead(_22); + _24 = copy _10; + _25 = AddWithOverflow(copy _10, const 1_usize); + assert(!move (_25.1: bool), "attempt to compute `{} + {}`, which would overflow", copy _10, const 1_usize) -> [success: bb14, unwind unreachable]; } bb14: { - _13 = move (_30.0: usize); + _10 = move (_25.0: usize); + StorageLive(_26); + _26 = (copy _24, copy _23); + _27 = Option::<(usize, &T)>::Some(move _26); + StorageDead(_26); + StorageDead(_25); + StorageDead(_24); + _28 = copy (((_27 as Some).0: (usize, &T)).0: usize); + _29 = copy (((_27 as Some).0: (usize, &T)).1: &T); + StorageLive(_30); + _30 = &_2; StorageLive(_31); - _31 = (copy _29, copy _28); - _32 = Option::<(usize, &T)>::Some(move _31); - StorageDead(_31); - StorageDead(_30); - StorageDead(_29); - _33 = copy (((_32 as Some).0: (usize, &T)).0: usize); - _34 = copy (((_32 as Some).0: (usize, &T)).1: &T); - StorageLive(_35); - _35 = &_2; - StorageLive(_36); - _36 = (copy _33, copy _34); - _37 = >::call(move _35, move _36) -> [return: bb15, unwind unreachable]; + _31 = (copy _28, copy _29); + _32 = >::call(move _30, move _31) -> [return: bb15, unwind unreachable]; } bb15: { - StorageDead(_36); - StorageDead(_35); - StorageDead(_32); + StorageDead(_31); + StorageDead(_30); + StorageDead(_27); goto -> bb4; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-unwind.mir index 42aa152ec99d6..49efa14bb2426 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-unwind.mir @@ -4,22 +4,22 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { debug slice => _1; debug f => _2; let mut _0: (); - let mut _11: std::slice::Iter<'_, T>; + let mut _10: std::slice::Iter<'_, T>; + let mut _11: std::iter::Enumerate>; let mut _12: std::iter::Enumerate>; - let mut _13: std::iter::Enumerate>; - let mut _14: &mut std::iter::Enumerate>; - let mut _15: std::option::Option<(usize, &T)>; - let mut _16: isize; - let mut _19: &impl Fn(usize, &T); - let mut _20: (usize, &T); - let _21: (); + let mut _13: &mut std::iter::Enumerate>; + let mut _14: std::option::Option<(usize, &T)>; + let mut _15: isize; + let mut _18: &impl Fn(usize, &T); + let mut _19: (usize, &T); + let _20: (); scope 1 { - debug iter => _13; - let _17: usize; - let _18: &T; + debug iter => _12; + let _16: usize; + let _17: &T; scope 2 { - debug i => _17; - debug x => _18; + debug i => _16; + debug x => _17; } } scope 3 (inlined core::slice::::iter) { @@ -27,11 +27,10 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { let _3: usize; let mut _7: *mut T; let mut _8: *mut T; - let mut _10: *const T; + let mut _9: *const T; scope 5 { let _6: std::ptr::NonNull; scope 6 { - let _9: *const T; scope 7 { } scope 11 (inlined std::ptr::without_provenance::) { @@ -62,9 +61,10 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } bb0: { - StorageLive(_11); + StorageLive(_10); StorageLive(_3); StorageLive(_6); + StorageLive(_9); StorageLive(_4); _3 = PtrMetadata(copy _1); _4 = &raw const (*_1); @@ -72,7 +72,6 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { _5 = copy _4 as *const T (PtrToPtr); _6 = NonNull:: { pointer: copy _5 }; StorageDead(_5); - StorageLive(_9); switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2]; } @@ -93,33 +92,30 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } bb3: { - StorageLive(_10); - _10 = copy _9; - _11 = std::slice::Iter::<'_, T> { ptr: copy _6, end_or_len: copy _10, _marker: const ZeroSized: PhantomData<&T> }; - StorageDead(_10); - StorageDead(_9); + _10 = std::slice::Iter::<'_, T> { ptr: copy _6, end_or_len: copy _9, _marker: const ZeroSized: PhantomData<&T> }; StorageDead(_4); + StorageDead(_9); StorageDead(_6); StorageDead(_3); - _12 = Enumerate::> { iter: copy _11, count: const 0_usize }; - StorageDead(_11); - StorageLive(_13); - _13 = copy _12; + _11 = Enumerate::> { iter: copy _10, count: const 0_usize }; + StorageDead(_10); + StorageLive(_12); + _12 = copy _11; goto -> bb4; } bb4: { - _14 = &mut _13; - _15 = > as Iterator>::next(move _14) -> [return: bb5, unwind: bb11]; + _13 = &mut _12; + _14 = > as Iterator>::next(move _13) -> [return: bb5, unwind: bb11]; } bb5: { - _16 = discriminant(_15); - switchInt(move _16) -> [0: bb6, 1: bb8, otherwise: bb10]; + _15 = discriminant(_14); + switchInt(move _15) -> [0: bb6, 1: bb8, otherwise: bb10]; } bb6: { - StorageDead(_13); + StorageDead(_12); drop(_2) -> [return: bb7, unwind continue]; } @@ -128,18 +124,18 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } bb8: { - _17 = copy (((_15 as Some).0: (usize, &T)).0: usize); - _18 = copy (((_15 as Some).0: (usize, &T)).1: &T); + _16 = copy (((_14 as Some).0: (usize, &T)).0: usize); + _17 = copy (((_14 as Some).0: (usize, &T)).1: &T); + StorageLive(_18); + _18 = &_2; StorageLive(_19); - _19 = &_2; - StorageLive(_20); - _20 = copy ((_15 as Some).0: (usize, &T)); - _21 = >::call(move _19, move _20) -> [return: bb9, unwind: bb11]; + _19 = copy ((_14 as Some).0: (usize, &T)); + _20 = >::call(move _18, move _19) -> [return: bb9, unwind: bb11]; } bb9: { - StorageDead(_20); StorageDead(_19); + StorageDead(_18); goto -> bb4; } diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir index 0d65640ec9b14..77db75efa072e 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir @@ -4,31 +4,29 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { debug slice => _1; debug f => _2; let mut _0: (); - let mut _11: std::ptr::NonNull; - let mut _12: *const T; - let mut _26: std::option::Option<&T>; - let mut _28: &impl Fn(&T); - let mut _29: (&T,); - let _30: (); + let mut _6: std::ptr::NonNull; + let mut _9: *const T; + let mut _21: std::option::Option<&T>; + let mut _23: &impl Fn(&T); + let mut _24: (&T,); + let _25: (); scope 1 { - debug ((iter: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _11; - debug ((iter: std::slice::Iter<'_, T>).1: *const T) => _12; + debug ((iter: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _6; + debug ((iter: std::slice::Iter<'_, T>).1: *const T) => _9; debug ((iter: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>; - let _27: &T; + let _22: &T; scope 2 { - debug x => _27; + debug x => _22; } scope 16 (inlined as Iterator>::next) { - let _13: std::ptr::NonNull; - let _15: std::ptr::NonNull; - let mut _18: bool; - let mut _21: std::ptr::NonNull; - let mut _23: usize; - let _25: &T; + let _10: std::ptr::NonNull; + let _11: std::ptr::NonNull; + let mut _14: bool; + let mut _18: usize; + let _20: &T; scope 17 { - let _14: *const T; scope 18 { - let _22: usize; + let _17: usize; scope 19 { scope 22 (inlined #[track_caller] core::num::::unchecked_sub) { scope 23 (inlined core::ub_checks::check_language_ub) { @@ -44,21 +42,21 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } } scope 26 (inlined as PartialEq>::eq) { - let mut _16: *mut T; - let mut _17: *mut T; + let mut _12: *mut T; + let mut _13: *mut T; scope 27 (inlined NonNull::::as_ptr) { } scope 28 (inlined NonNull::::as_ptr) { } } scope 29 (inlined NonNull::::add) { - let mut _19: *const T; - let mut _20: *const T; + let mut _15: *const T; + let mut _16: *const T; scope 30 (inlined NonNull::::as_ptr) { } } scope 31 (inlined NonNull::::as_ref::<'_>) { - let _24: *const T; + let _19: *const T; scope 32 (inlined NonNull::::as_ptr) { } scope 33 (inlined std::ptr::mut_ptr::::cast_const) { @@ -73,11 +71,8 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { let _3: usize; let mut _7: *mut T; let mut _8: *mut T; - let mut _10: *const T; scope 5 { - let _6: std::ptr::NonNull; scope 6 { - let _9: *const T; scope 7 { } scope 11 (inlined std::ptr::without_provenance::) { @@ -112,7 +107,6 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { _5 = copy _4 as *const T (PtrToPtr); _6 = NonNull:: { pointer: copy _5 }; StorageDead(_5); - StorageLive(_9); switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2]; } @@ -133,88 +127,73 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb3: { - _10 = copy _9; - StorageDead(_9); StorageDead(_4); StorageDead(_3); - StorageLive(_11); - StorageLive(_12); - _11 = copy _6; - _12 = copy _10; goto -> bb4; } bb4: { - StorageLive(_26); - StorageLive(_13); - StorageLive(_14); - StorageLive(_22); - StorageLive(_23); - StorageLive(_15); - StorageLive(_25); - _13 = copy _11; - _14 = copy _12; + StorageLive(_21); + StorageLive(_10); + StorageLive(_17); + StorageLive(_18); + StorageLive(_11); + StorageLive(_20); + _10 = copy _6; switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb8]; } bb5: { - StorageLive(_18); - _15 = copy _14 as std::ptr::NonNull (Transmute); - StorageLive(_16); - _16 = copy _13 as *mut T (Transmute); - StorageLive(_17); - _17 = copy _15 as *mut T (Transmute); - _18 = Eq(copy _16, copy _17); - StorageDead(_17); - StorageDead(_16); - switchInt(move _18) -> [0: bb6, otherwise: bb7]; + StorageLive(_14); + _11 = copy _9 as std::ptr::NonNull (Transmute); + StorageLive(_12); + _12 = copy _10 as *mut T (Transmute); + StorageLive(_13); + _13 = copy _11 as *mut T (Transmute); + _14 = Eq(copy _12, copy _13); + StorageDead(_13); + StorageDead(_12); + switchInt(move _14) -> [0: bb6, otherwise: bb7]; } bb6: { - StorageDead(_18); - StorageLive(_21); - StorageLive(_20); - StorageLive(_19); - _19 = copy _13 as *const T (Transmute); - _20 = Offset(copy _19, const 1_usize); - StorageDead(_19); - _21 = NonNull:: { pointer: copy _20 }; - StorageDead(_20); - _11 = move _21; - StorageDead(_21); + StorageDead(_14); + StorageLive(_16); + StorageLive(_15); + _15 = copy _10 as *const T (Transmute); + _16 = Offset(copy _15, const 1_usize); + StorageDead(_15); + _6 = NonNull:: { pointer: copy _16 }; + StorageDead(_16); goto -> bb13; } bb7: { - StorageDead(_18); - StorageDead(_25); - StorageDead(_15); - StorageDead(_23); - StorageDead(_22); StorageDead(_14); - StorageDead(_13); + StorageDead(_20); + StorageDead(_11); + StorageDead(_18); + StorageDead(_17); + StorageDead(_10); goto -> bb10; } bb8: { - _22 = copy _14 as usize (Transmute); - switchInt(copy _22) -> [0: bb9, otherwise: bb12]; + _17 = copy _9 as usize (Transmute); + switchInt(copy _17) -> [0: bb9, otherwise: bb12]; } bb9: { - StorageDead(_25); - StorageDead(_15); - StorageDead(_23); - StorageDead(_22); - StorageDead(_14); - StorageDead(_13); + StorageDead(_20); + StorageDead(_11); + StorageDead(_18); + StorageDead(_17); + StorageDead(_10); goto -> bb10; } bb10: { - StorageDead(_26); - StorageDead(_11); - StorageDead(_12); + StorageDead(_21); drop(_2) -> [return: bb11, unwind unreachable]; } @@ -223,35 +202,34 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb12: { - _23 = SubUnchecked(copy _22, const 1_usize); - _12 = copy _23 as *const T (Transmute); + _18 = SubUnchecked(copy _17, const 1_usize); + _9 = copy _18 as *const T (Transmute); goto -> bb13; } bb13: { + StorageLive(_19); + _19 = copy _10 as *const T (Transmute); + _20 = &(*_19); + StorageDead(_19); + _21 = Option::<&T>::Some(copy _20); + StorageDead(_20); + StorageDead(_11); + StorageDead(_18); + StorageDead(_17); + StorageDead(_10); + _22 = copy ((_21 as Some).0: &T); + StorageLive(_23); + _23 = &_2; StorageLive(_24); - _24 = copy _13 as *const T (Transmute); - _25 = &(*_24); - StorageDead(_24); - _26 = Option::<&T>::Some(copy _25); - StorageDead(_25); - StorageDead(_15); - StorageDead(_23); - StorageDead(_22); - StorageDead(_14); - StorageDead(_13); - _27 = copy ((_26 as Some).0: &T); - StorageLive(_28); - _28 = &_2; - StorageLive(_29); - _29 = (copy _27,); - _30 = >::call(move _28, move _29) -> [return: bb14, unwind unreachable]; + _24 = (copy _22,); + _25 = >::call(move _23, move _24) -> [return: bb14, unwind unreachable]; } bb14: { - StorageDead(_29); - StorageDead(_28); - StorageDead(_26); + StorageDead(_24); + StorageDead(_23); + StorageDead(_21); goto -> bb4; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir index 02efb193474d6..b2f0efcbcd161 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir @@ -4,31 +4,29 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { debug slice => _1; debug f => _2; let mut _0: (); - let mut _11: std::ptr::NonNull; - let mut _12: *const T; - let mut _26: std::option::Option<&T>; - let mut _28: &impl Fn(&T); - let mut _29: (&T,); - let _30: (); + let mut _6: std::ptr::NonNull; + let mut _9: *const T; + let mut _21: std::option::Option<&T>; + let mut _23: &impl Fn(&T); + let mut _24: (&T,); + let _25: (); scope 1 { - debug ((iter: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _11; - debug ((iter: std::slice::Iter<'_, T>).1: *const T) => _12; + debug ((iter: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _6; + debug ((iter: std::slice::Iter<'_, T>).1: *const T) => _9; debug ((iter: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>; - let _27: &T; + let _22: &T; scope 2 { - debug x => _27; + debug x => _22; } scope 16 (inlined as Iterator>::next) { - let _13: std::ptr::NonNull; - let _15: std::ptr::NonNull; - let mut _18: bool; - let mut _21: std::ptr::NonNull; - let mut _23: usize; - let _25: &T; + let _10: std::ptr::NonNull; + let _11: std::ptr::NonNull; + let mut _14: bool; + let mut _18: usize; + let _20: &T; scope 17 { - let _14: *const T; scope 18 { - let _22: usize; + let _17: usize; scope 19 { scope 22 (inlined #[track_caller] core::num::::unchecked_sub) { scope 23 (inlined core::ub_checks::check_language_ub) { @@ -44,21 +42,21 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } } scope 26 (inlined as PartialEq>::eq) { - let mut _16: *mut T; - let mut _17: *mut T; + let mut _12: *mut T; + let mut _13: *mut T; scope 27 (inlined NonNull::::as_ptr) { } scope 28 (inlined NonNull::::as_ptr) { } } scope 29 (inlined NonNull::::add) { - let mut _19: *const T; - let mut _20: *const T; + let mut _15: *const T; + let mut _16: *const T; scope 30 (inlined NonNull::::as_ptr) { } } scope 31 (inlined NonNull::::as_ref::<'_>) { - let _24: *const T; + let _19: *const T; scope 32 (inlined NonNull::::as_ptr) { } scope 33 (inlined std::ptr::mut_ptr::::cast_const) { @@ -73,11 +71,8 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { let _3: usize; let mut _7: *mut T; let mut _8: *mut T; - let mut _10: *const T; scope 5 { - let _6: std::ptr::NonNull; scope 6 { - let _9: *const T; scope 7 { } scope 11 (inlined std::ptr::without_provenance::) { @@ -112,7 +107,6 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { _5 = copy _4 as *const T (PtrToPtr); _6 = NonNull:: { pointer: copy _5 }; StorageDead(_5); - StorageLive(_9); switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2]; } @@ -133,88 +127,73 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb3: { - _10 = copy _9; - StorageDead(_9); StorageDead(_4); StorageDead(_3); - StorageLive(_11); - StorageLive(_12); - _11 = copy _6; - _12 = copy _10; goto -> bb4; } bb4: { - StorageLive(_26); - StorageLive(_13); - StorageLive(_14); - StorageLive(_22); - StorageLive(_23); - StorageLive(_15); - StorageLive(_25); - _13 = copy _11; - _14 = copy _12; + StorageLive(_21); + StorageLive(_10); + StorageLive(_17); + StorageLive(_18); + StorageLive(_11); + StorageLive(_20); + _10 = copy _6; switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb8]; } bb5: { - StorageLive(_18); - _15 = copy _14 as std::ptr::NonNull (Transmute); - StorageLive(_16); - _16 = copy _13 as *mut T (Transmute); - StorageLive(_17); - _17 = copy _15 as *mut T (Transmute); - _18 = Eq(copy _16, copy _17); - StorageDead(_17); - StorageDead(_16); - switchInt(move _18) -> [0: bb6, otherwise: bb7]; + StorageLive(_14); + _11 = copy _9 as std::ptr::NonNull (Transmute); + StorageLive(_12); + _12 = copy _10 as *mut T (Transmute); + StorageLive(_13); + _13 = copy _11 as *mut T (Transmute); + _14 = Eq(copy _12, copy _13); + StorageDead(_13); + StorageDead(_12); + switchInt(move _14) -> [0: bb6, otherwise: bb7]; } bb6: { - StorageDead(_18); - StorageLive(_21); - StorageLive(_20); - StorageLive(_19); - _19 = copy _13 as *const T (Transmute); - _20 = Offset(copy _19, const 1_usize); - StorageDead(_19); - _21 = NonNull:: { pointer: copy _20 }; - StorageDead(_20); - _11 = move _21; - StorageDead(_21); + StorageDead(_14); + StorageLive(_16); + StorageLive(_15); + _15 = copy _10 as *const T (Transmute); + _16 = Offset(copy _15, const 1_usize); + StorageDead(_15); + _6 = NonNull:: { pointer: copy _16 }; + StorageDead(_16); goto -> bb13; } bb7: { - StorageDead(_18); - StorageDead(_25); - StorageDead(_15); - StorageDead(_23); - StorageDead(_22); StorageDead(_14); - StorageDead(_13); + StorageDead(_20); + StorageDead(_11); + StorageDead(_18); + StorageDead(_17); + StorageDead(_10); goto -> bb10; } bb8: { - _22 = copy _14 as usize (Transmute); - switchInt(copy _22) -> [0: bb9, otherwise: bb12]; + _17 = copy _9 as usize (Transmute); + switchInt(copy _17) -> [0: bb9, otherwise: bb12]; } bb9: { - StorageDead(_25); - StorageDead(_15); - StorageDead(_23); - StorageDead(_22); - StorageDead(_14); - StorageDead(_13); + StorageDead(_20); + StorageDead(_11); + StorageDead(_18); + StorageDead(_17); + StorageDead(_10); goto -> bb10; } bb10: { - StorageDead(_26); - StorageDead(_11); - StorageDead(_12); + StorageDead(_21); drop(_2) -> [return: bb11, unwind continue]; } @@ -223,35 +202,34 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb12: { - _23 = SubUnchecked(copy _22, const 1_usize); - _12 = copy _23 as *const T (Transmute); + _18 = SubUnchecked(copy _17, const 1_usize); + _9 = copy _18 as *const T (Transmute); goto -> bb13; } bb13: { + StorageLive(_19); + _19 = copy _10 as *const T (Transmute); + _20 = &(*_19); + StorageDead(_19); + _21 = Option::<&T>::Some(copy _20); + StorageDead(_20); + StorageDead(_11); + StorageDead(_18); + StorageDead(_17); + StorageDead(_10); + _22 = copy ((_21 as Some).0: &T); + StorageLive(_23); + _23 = &_2; StorageLive(_24); - _24 = copy _13 as *const T (Transmute); - _25 = &(*_24); - StorageDead(_24); - _26 = Option::<&T>::Some(copy _25); - StorageDead(_25); - StorageDead(_15); - StorageDead(_23); - StorageDead(_22); - StorageDead(_14); - StorageDead(_13); - _27 = copy ((_26 as Some).0: &T); - StorageLive(_28); - _28 = &_2; - StorageLive(_29); - _29 = (copy _27,); - _30 = >::call(move _28, move _29) -> [return: bb14, unwind: bb15]; + _24 = (copy _22,); + _25 = >::call(move _23, move _24) -> [return: bb14, unwind: bb15]; } bb14: { - StorageDead(_29); - StorageDead(_28); - StorageDead(_26); + StorageDead(_24); + StorageDead(_23); + StorageDead(_21); goto -> bb4; } diff --git a/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-abort.mir index 41e273151eca4..12ba8c446d937 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-abort.mir @@ -6,27 +6,26 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { let mut _0: (); let mut _3: usize; let mut _4: usize; - let mut _9: std::option::Option; - let mut _11: bool; - let mut _13: &impl Fn(usize, &T); - let mut _14: (usize, &T); - let _15: (); + let mut _8: std::option::Option; + let mut _10: bool; + let mut _12: &impl Fn(usize, &T); + let mut _13: (usize, &T); + let _14: (); scope 1 { debug ((iter: std::ops::Range).0: usize) => _4; debug ((iter: std::ops::Range).1: usize) => _3; - let _10: usize; + let _9: usize; scope 2 { - debug i => _10; - let _12: &T; + debug i => _9; + let _11: &T; scope 3 { - debug x => _12; + debug x => _11; } } scope 5 (inlined iter::range::>::next) { scope 6 (inlined as iter::range::RangeIteratorImpl>::spec_next) { let mut _6: bool; let _7: usize; - let mut _8: usize; scope 7 { scope 9 (inlined ::forward_unchecked) { scope 10 (inlined #[track_caller] core::num::::unchecked_add) { @@ -48,13 +47,12 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { bb0: { _3 = PtrMetadata(copy _1); - StorageLive(_4); _4 = const 0_usize; goto -> bb1; } bb1: { - StorageLive(_9); + StorageLive(_8); StorageLive(_6); StorageLive(_5); _5 = copy _4; @@ -65,8 +63,7 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { bb2: { StorageDead(_6); - StorageDead(_9); - StorageDead(_4); + StorageDead(_8); drop(_2) -> [return: bb3, unwind unreachable]; } @@ -76,30 +73,27 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { bb4: { _7 = copy _4; - StorageLive(_8); - _8 = AddUnchecked(copy _7, const 1_usize); - _4 = move _8; - StorageDead(_8); - _9 = Option::::Some(copy _7); + _4 = AddUnchecked(copy _7, const 1_usize); + _8 = Option::::Some(copy _7); StorageDead(_6); - _10 = copy ((_9 as Some).0: usize); - _11 = Lt(copy _10, copy _3); - assert(move _11, "index out of bounds: the length is {} but the index is {}", copy _3, copy _10) -> [success: bb5, unwind unreachable]; + _9 = copy ((_8 as Some).0: usize); + _10 = Lt(copy _9, copy _3); + assert(move _10, "index out of bounds: the length is {} but the index is {}", copy _3, copy _9) -> [success: bb5, unwind unreachable]; } bb5: { - _12 = &(*_1)[_10]; + _11 = &(*_1)[_9]; + StorageLive(_12); + _12 = &_2; StorageLive(_13); - _13 = &_2; - StorageLive(_14); - _14 = (copy _10, copy _12); - _15 = >::call(move _13, move _14) -> [return: bb6, unwind unreachable]; + _13 = (copy _9, copy _11); + _14 = >::call(move _12, move _13) -> [return: bb6, unwind unreachable]; } bb6: { - StorageDead(_14); StorageDead(_13); - StorageDead(_9); + StorageDead(_12); + StorageDead(_8); goto -> bb1; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-unwind.mir index ec781c1480c74..3d0d998e418a3 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-unwind.mir @@ -6,27 +6,26 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { let mut _0: (); let mut _3: usize; let mut _4: usize; - let mut _9: std::option::Option; - let mut _11: bool; - let mut _13: &impl Fn(usize, &T); - let mut _14: (usize, &T); - let _15: (); + let mut _8: std::option::Option; + let mut _10: bool; + let mut _12: &impl Fn(usize, &T); + let mut _13: (usize, &T); + let _14: (); scope 1 { debug ((iter: std::ops::Range).0: usize) => _4; debug ((iter: std::ops::Range).1: usize) => _3; - let _10: usize; + let _9: usize; scope 2 { - debug i => _10; - let _12: &T; + debug i => _9; + let _11: &T; scope 3 { - debug x => _12; + debug x => _11; } } scope 5 (inlined iter::range::>::next) { scope 6 (inlined as iter::range::RangeIteratorImpl>::spec_next) { let mut _6: bool; let _7: usize; - let mut _8: usize; scope 7 { scope 9 (inlined ::forward_unchecked) { scope 10 (inlined #[track_caller] core::num::::unchecked_add) { @@ -48,13 +47,12 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { bb0: { _3 = PtrMetadata(copy _1); - StorageLive(_4); _4 = const 0_usize; goto -> bb1; } bb1: { - StorageLive(_9); + StorageLive(_8); StorageLive(_6); StorageLive(_5); _5 = copy _4; @@ -65,8 +63,7 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { bb2: { StorageDead(_6); - StorageDead(_9); - StorageDead(_4); + StorageDead(_8); drop(_2) -> [return: bb3, unwind continue]; } @@ -76,30 +73,27 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { bb4: { _7 = copy _4; - StorageLive(_8); - _8 = AddUnchecked(copy _7, const 1_usize); - _4 = move _8; - StorageDead(_8); - _9 = Option::::Some(copy _7); + _4 = AddUnchecked(copy _7, const 1_usize); + _8 = Option::::Some(copy _7); StorageDead(_6); - _10 = copy ((_9 as Some).0: usize); - _11 = Lt(copy _10, copy _3); - assert(move _11, "index out of bounds: the length is {} but the index is {}", copy _3, copy _10) -> [success: bb5, unwind: bb7]; + _9 = copy ((_8 as Some).0: usize); + _10 = Lt(copy _9, copy _3); + assert(move _10, "index out of bounds: the length is {} but the index is {}", copy _3, copy _9) -> [success: bb5, unwind: bb7]; } bb5: { - _12 = &(*_1)[_10]; + _11 = &(*_1)[_9]; + StorageLive(_12); + _12 = &_2; StorageLive(_13); - _13 = &_2; - StorageLive(_14); - _14 = (copy _10, copy _12); - _15 = >::call(move _13, move _14) -> [return: bb6, unwind: bb7]; + _13 = (copy _9, copy _11); + _14 = >::call(move _12, move _13) -> [return: bb6, unwind: bb7]; } bb6: { - StorageDead(_14); StorageDead(_13); - StorageDead(_9); + StorageDead(_12); + StorageDead(_8); goto -> bb1; } diff --git a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir index ee638be7d7ac2..6e9ae3e15a9f7 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir @@ -4,22 +4,22 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { debug slice => _1; debug f => _2; let mut _0: (); - let mut _11: std::slice::Iter<'_, T>; + let mut _10: std::slice::Iter<'_, T>; + let mut _11: std::iter::Rev>; let mut _12: std::iter::Rev>; - let mut _13: std::iter::Rev>; - let mut _15: std::option::Option<&T>; - let mut _16: isize; - let mut _18: &impl Fn(&T); - let mut _19: (&T,); - let _20: (); + let mut _14: std::option::Option<&T>; + let mut _15: isize; + let mut _17: &impl Fn(&T); + let mut _18: (&T,); + let _19: (); scope 1 { - debug iter => _13; - let _17: &T; + debug iter => _12; + let _16: &T; scope 2 { - debug x => _17; + debug x => _16; } scope 18 (inlined > as Iterator>::next) { - let mut _14: &mut std::slice::Iter<'_, T>; + let mut _13: &mut std::slice::Iter<'_, T>; } } scope 3 (inlined core::slice::::iter) { @@ -27,11 +27,10 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { let _3: usize; let mut _7: *mut T; let mut _8: *mut T; - let mut _10: *const T; + let mut _9: *const T; scope 5 { let _6: std::ptr::NonNull; scope 6 { - let _9: *const T; scope 7 { } scope 11 (inlined std::ptr::without_provenance::) { @@ -62,9 +61,10 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb0: { - StorageLive(_11); + StorageLive(_10); StorageLive(_3); StorageLive(_6); + StorageLive(_9); StorageLive(_4); _3 = PtrMetadata(copy _1); _4 = &raw const (*_1); @@ -72,7 +72,6 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { _5 = copy _4 as *const T (PtrToPtr); _6 = NonNull:: { pointer: copy _5 }; StorageDead(_5); - StorageLive(_9); switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2]; } @@ -93,37 +92,34 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb3: { - StorageLive(_10); - _10 = copy _9; - _11 = std::slice::Iter::<'_, T> { ptr: copy _6, end_or_len: copy _10, _marker: const ZeroSized: PhantomData<&T> }; - StorageDead(_10); - StorageDead(_9); + _10 = std::slice::Iter::<'_, T> { ptr: copy _6, end_or_len: copy _9, _marker: const ZeroSized: PhantomData<&T> }; StorageDead(_4); + StorageDead(_9); StorageDead(_6); StorageDead(_3); - _12 = Rev::> { iter: copy _11 }; - StorageDead(_11); - StorageLive(_13); - _13 = copy _12; + _11 = Rev::> { iter: copy _10 }; + StorageDead(_10); + StorageLive(_12); + _12 = copy _11; goto -> bb4; } bb4: { - StorageLive(_15); StorageLive(_14); - _14 = &mut (_13.0: std::slice::Iter<'_, T>); - _15 = as DoubleEndedIterator>::next_back(move _14) -> [return: bb5, unwind unreachable]; + StorageLive(_13); + _13 = &mut (_12.0: std::slice::Iter<'_, T>); + _14 = as DoubleEndedIterator>::next_back(move _13) -> [return: bb5, unwind unreachable]; } bb5: { - StorageDead(_14); - _16 = discriminant(_15); - switchInt(move _16) -> [0: bb6, 1: bb8, otherwise: bb10]; + StorageDead(_13); + _15 = discriminant(_14); + switchInt(move _15) -> [0: bb6, 1: bb8, otherwise: bb10]; } bb6: { - StorageDead(_15); - StorageDead(_13); + StorageDead(_14); + StorageDead(_12); drop(_2) -> [return: bb7, unwind unreachable]; } @@ -132,18 +128,18 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb8: { - _17 = copy ((_15 as Some).0: &T); + _16 = copy ((_14 as Some).0: &T); + StorageLive(_17); + _17 = &_2; StorageLive(_18); - _18 = &_2; - StorageLive(_19); - _19 = (copy _17,); - _20 = >::call(move _18, move _19) -> [return: bb9, unwind unreachable]; + _18 = (copy _16,); + _19 = >::call(move _17, move _18) -> [return: bb9, unwind unreachable]; } bb9: { - StorageDead(_19); StorageDead(_18); - StorageDead(_15); + StorageDead(_17); + StorageDead(_14); goto -> bb4; } diff --git a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir index aee29d4d4fef9..128db91ffab49 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir @@ -4,22 +4,22 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { debug slice => _1; debug f => _2; let mut _0: (); - let mut _11: std::slice::Iter<'_, T>; + let mut _10: std::slice::Iter<'_, T>; + let mut _11: std::iter::Rev>; let mut _12: std::iter::Rev>; - let mut _13: std::iter::Rev>; - let mut _15: std::option::Option<&T>; - let mut _16: isize; - let mut _18: &impl Fn(&T); - let mut _19: (&T,); - let _20: (); + let mut _14: std::option::Option<&T>; + let mut _15: isize; + let mut _17: &impl Fn(&T); + let mut _18: (&T,); + let _19: (); scope 1 { - debug iter => _13; - let _17: &T; + debug iter => _12; + let _16: &T; scope 2 { - debug x => _17; + debug x => _16; } scope 18 (inlined > as Iterator>::next) { - let mut _14: &mut std::slice::Iter<'_, T>; + let mut _13: &mut std::slice::Iter<'_, T>; } } scope 3 (inlined core::slice::::iter) { @@ -27,11 +27,10 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { let _3: usize; let mut _7: *mut T; let mut _8: *mut T; - let mut _10: *const T; + let mut _9: *const T; scope 5 { let _6: std::ptr::NonNull; scope 6 { - let _9: *const T; scope 7 { } scope 11 (inlined std::ptr::without_provenance::) { @@ -62,9 +61,10 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb0: { - StorageLive(_11); + StorageLive(_10); StorageLive(_3); StorageLive(_6); + StorageLive(_9); StorageLive(_4); _3 = PtrMetadata(copy _1); _4 = &raw const (*_1); @@ -72,7 +72,6 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { _5 = copy _4 as *const T (PtrToPtr); _6 = NonNull:: { pointer: copy _5 }; StorageDead(_5); - StorageLive(_9); switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2]; } @@ -93,37 +92,34 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb3: { - StorageLive(_10); - _10 = copy _9; - _11 = std::slice::Iter::<'_, T> { ptr: copy _6, end_or_len: copy _10, _marker: const ZeroSized: PhantomData<&T> }; - StorageDead(_10); - StorageDead(_9); + _10 = std::slice::Iter::<'_, T> { ptr: copy _6, end_or_len: copy _9, _marker: const ZeroSized: PhantomData<&T> }; StorageDead(_4); + StorageDead(_9); StorageDead(_6); StorageDead(_3); - _12 = Rev::> { iter: copy _11 }; - StorageDead(_11); - StorageLive(_13); - _13 = copy _12; + _11 = Rev::> { iter: copy _10 }; + StorageDead(_10); + StorageLive(_12); + _12 = copy _11; goto -> bb4; } bb4: { - StorageLive(_15); StorageLive(_14); - _14 = &mut (_13.0: std::slice::Iter<'_, T>); - _15 = as DoubleEndedIterator>::next_back(move _14) -> [return: bb5, unwind: bb11]; + StorageLive(_13); + _13 = &mut (_12.0: std::slice::Iter<'_, T>); + _14 = as DoubleEndedIterator>::next_back(move _13) -> [return: bb5, unwind: bb11]; } bb5: { - StorageDead(_14); - _16 = discriminant(_15); - switchInt(move _16) -> [0: bb6, 1: bb8, otherwise: bb10]; + StorageDead(_13); + _15 = discriminant(_14); + switchInt(move _15) -> [0: bb6, 1: bb8, otherwise: bb10]; } bb6: { - StorageDead(_15); - StorageDead(_13); + StorageDead(_14); + StorageDead(_12); drop(_2) -> [return: bb7, unwind continue]; } @@ -132,18 +128,18 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb8: { - _17 = copy ((_15 as Some).0: &T); + _16 = copy ((_14 as Some).0: &T); + StorageLive(_17); + _17 = &_2; StorageLive(_18); - _18 = &_2; - StorageLive(_19); - _19 = (copy _17,); - _20 = >::call(move _18, move _19) -> [return: bb9, unwind: bb11]; + _18 = (copy _16,); + _19 = >::call(move _17, move _18) -> [return: bb9, unwind: bb11]; } bb9: { - StorageDead(_19); StorageDead(_18); - StorageDead(_15); + StorageDead(_17); + StorageDead(_14); goto -> bb4; } diff --git a/tests/mir-opt/pre-codegen/tuple_ord.demo_ge_partial.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/tuple_ord.demo_ge_partial.PreCodegen.after.mir index c4d0e318b58dc..29bfe962974cf 100644 --- a/tests/mir-opt/pre-codegen/tuple_ord.demo_ge_partial.PreCodegen.after.mir +++ b/tests/mir-opt/pre-codegen/tuple_ord.demo_ge_partial.PreCodegen.after.mir @@ -7,7 +7,6 @@ fn demo_ge_partial(_1: &(f32, f32), _2: &(f32, f32)) -> bool { scope 1 (inlined std::cmp::impls::::ge) { scope 2 (inlined core::tuple::::ge) { let mut _7: std::ops::ControlFlow; - let _8: bool; scope 3 { } scope 4 (inlined std::cmp::impls::::__chaining_ge) { @@ -19,8 +18,8 @@ fn demo_ge_partial(_1: &(f32, f32), _2: &(f32, f32)) -> bool { } } scope 6 (inlined std::cmp::impls::::ge) { + let mut _8: f32; let mut _9: f32; - let mut _10: f32; } } } @@ -44,10 +43,7 @@ fn demo_ge_partial(_1: &(f32, f32), _2: &(f32, f32)) -> bool { StorageDead(_5); StorageDead(_4); StorageDead(_3); - StorageLive(_8); - _8 = copy ((_7 as Break).0: bool); - _0 = copy _8; - StorageDead(_8); + _0 = copy ((_7 as Break).0: bool); goto -> bb3; } @@ -55,13 +51,13 @@ fn demo_ge_partial(_1: &(f32, f32), _2: &(f32, f32)) -> bool { StorageDead(_5); StorageDead(_4); StorageDead(_3); + StorageLive(_8); + _8 = copy ((*_1).1: f32); StorageLive(_9); - _9 = copy ((*_1).1: f32); - StorageLive(_10); - _10 = copy ((*_2).1: f32); - _0 = Ge(move _9, move _10); - StorageDead(_10); + _9 = copy ((*_2).1: f32); + _0 = Ge(move _8, move _9); StorageDead(_9); + StorageDead(_8); goto -> bb3; } diff --git a/tests/mir-opt/pre-codegen/tuple_ord.demo_le_total.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/tuple_ord.demo_le_total.PreCodegen.after.mir index 44df8b279935b..7678c92a1f0c6 100644 --- a/tests/mir-opt/pre-codegen/tuple_ord.demo_le_total.PreCodegen.after.mir +++ b/tests/mir-opt/pre-codegen/tuple_ord.demo_le_total.PreCodegen.after.mir @@ -7,7 +7,6 @@ fn demo_le_total(_1: &(u16, i16), _2: &(u16, i16)) -> bool { scope 1 (inlined std::cmp::impls::::le) { scope 2 (inlined core::tuple::::le) { let mut _7: std::ops::ControlFlow; - let _8: bool; scope 3 { } scope 4 (inlined std::cmp::impls::::__chaining_le) { @@ -19,8 +18,8 @@ fn demo_le_total(_1: &(u16, i16), _2: &(u16, i16)) -> bool { } } scope 6 (inlined std::cmp::impls::::le) { + let mut _8: i16; let mut _9: i16; - let mut _10: i16; } } } @@ -44,10 +43,7 @@ fn demo_le_total(_1: &(u16, i16), _2: &(u16, i16)) -> bool { StorageDead(_5); StorageDead(_4); StorageDead(_3); - StorageLive(_8); - _8 = copy ((_7 as Break).0: bool); - _0 = copy _8; - StorageDead(_8); + _0 = copy ((_7 as Break).0: bool); goto -> bb3; } @@ -55,13 +51,13 @@ fn demo_le_total(_1: &(u16, i16), _2: &(u16, i16)) -> bool { StorageDead(_5); StorageDead(_4); StorageDead(_3); + StorageLive(_8); + _8 = copy ((*_1).1: i16); StorageLive(_9); - _9 = copy ((*_1).1: i16); - StorageLive(_10); - _10 = copy ((*_2).1: i16); - _0 = Le(move _9, move _10); - StorageDead(_10); + _9 = copy ((*_2).1: i16); + _0 = Le(move _8, move _9); StorageDead(_9); + StorageDead(_8); goto -> bb3; } From 0dc0c42f45b2575be60b7abe4b92b2d4e32c27da Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Mon, 28 Aug 2023 18:28:43 +0000 Subject: [PATCH 2/9] Introduce fast insertion at extremities to IntervalSet. --- compiler/rustc_index/src/interval.rs | 51 ++++++++++++++++++ compiler/rustc_mir_dataflow/src/points.rs | 66 +++++++++++++---------- 2 files changed, 90 insertions(+), 27 deletions(-) diff --git a/compiler/rustc_index/src/interval.rs b/compiler/rustc_index/src/interval.rs index 0225c5c4f329a..4af5bfcaee6dc 100644 --- a/compiler/rustc_index/src/interval.rs +++ b/compiler/rustc_index/src/interval.rs @@ -140,6 +140,49 @@ impl IntervalSet { result } + /// Specialized version of `insert` when we know that the inserted point is *before* any + /// contained. + pub fn prepend(&mut self, point: I) { + let point = point.index() as u32; + + if let Some((first_start, _)) = self.map.first_mut() { + assert!(point < *first_start); + if point + 1 == *first_start { + *first_start = point; + } else { + self.map.insert(0, (point, point)); + } + } else { + // If the map is empty, push is faster than insert. + self.map.push((point, point)); + } + + debug_assert!( + self.check_invariants(), + "wrong intervals after prepend {point:?} to {self:?}" + ); + } + + /// Specialized version of `insert` when we know that the inserted point is *after* any + /// contained. + pub fn append(&mut self, point: I) { + let point = point.index() as u32; + + if let Some((_, last_end)) = self.map.last_mut() + && let _ = assert!(*last_end < point) + && point == *last_end + 1 + { + *last_end = point; + } else { + self.map.push((point, point)); + } + + debug_assert!( + self.check_invariants(), + "wrong intervals after append {point:?} to {self:?}" + ); + } + pub fn contains(&self, needle: I) -> bool { let needle = needle.index() as u32; let Some(last) = self.map.partition_point(|r| r.0 <= needle).checked_sub(1) else { @@ -325,6 +368,14 @@ impl SparseIntervalMatrix { self.ensure_row(row).insert(point) } + pub fn prepend(&mut self, row: R, point: C) { + self.ensure_row(row).prepend(point) + } + + pub fn append(&mut self, row: R, point: C) { + self.ensure_row(row).append(point) + } + pub fn contains(&self, row: R, point: C) -> bool { self.row(row).is_some_and(|r| r.contains(point)) } diff --git a/compiler/rustc_mir_dataflow/src/points.rs b/compiler/rustc_mir_dataflow/src/points.rs index 70d1a34b5fb13..55a373d4c5199 100644 --- a/compiler/rustc_mir_dataflow/src/points.rs +++ b/compiler/rustc_mir_dataflow/src/points.rs @@ -3,7 +3,7 @@ use rustc_index::interval::SparseIntervalMatrix; use rustc_index::{Idx, IndexVec}; use rustc_middle::mir::{self, BasicBlock, Body, Location}; -use crate::framework::{Analysis, Results, ResultsVisitor, visit_results}; +use crate::framework::{Analysis, Direction, Results, ResultsVisitor, visit_results}; /// Maps between a `Location` and a `PointIndex` (and vice versa). pub struct DenseLocationMap { @@ -105,27 +105,47 @@ where N: Idx, A: Analysis<'tcx, Domain = DenseBitSet>, { - let values = SparseIntervalMatrix::new(elements.num_points()); - let mut visitor = Visitor { elements, values }; - visit_results( - body, - body.basic_blocks.reverse_postorder().iter().copied(), - &mut analysis, - &results, - &mut visitor, - ); - visitor.values + let mut values = SparseIntervalMatrix::new(elements.num_points()); + let reachable_blocks = mir::traversal::reachable_as_bitset(body); + if A::Direction::IS_BACKWARD { + // Iterate blocks in decreasing order, to visit locations in decreasing order. This + // allows to use the more efficient `prepend` method to interval sets. + let callback = |state: &DenseBitSet, location| { + let point = elements.point_from_location(location); + // Use internal iterator manually as it is much more efficient. + state.iter().for_each(|node| values.prepend(node, point)); + }; + let mut visitor = Visitor { callback }; + visit_results( + body, + // Note the `.rev()`. + body.basic_blocks.indices().filter(|&bb| reachable_blocks.contains(bb)).rev(), + &mut analysis, + &results, + &mut visitor, + ); + } else { + // Iterate blocks in increasing order, to visit locations in increasing order. This + // allows to use the more efficient `append` method to interval sets. + let callback = |state: &DenseBitSet, location| { + let point = elements.point_from_location(location); + // Use internal iterator manually as it is much more efficient. + state.iter().for_each(|node| values.append(node, point)); + }; + let mut visitor = Visitor { callback }; + visit_results(body, reachable_blocks.iter(), &mut analysis, &results, &mut visitor); + } + values } -struct Visitor<'a, N: Idx> { - elements: &'a DenseLocationMap, - values: SparseIntervalMatrix, +struct Visitor { + callback: F, } -impl<'tcx, A, N> ResultsVisitor<'tcx, A> for Visitor<'_, N> +impl<'tcx, A, F> ResultsVisitor<'tcx, A> for Visitor where - A: Analysis<'tcx, Domain = DenseBitSet>, - N: Idx, + A: Analysis<'tcx>, + F: FnMut(&A::Domain, Location), { fn visit_after_primary_statement_effect<'mir>( &mut self, @@ -134,11 +154,7 @@ where _statement: &'mir mir::Statement<'tcx>, location: Location, ) { - let point = self.elements.point_from_location(location); - // Use internal iterator manually as it is much more efficient. - state.iter().for_each(|node| { - self.values.insert(node, point); - }); + (self.callback)(state, location); } fn visit_after_primary_terminator_effect<'mir>( @@ -148,10 +164,6 @@ where _terminator: &'mir mir::Terminator<'tcx>, location: Location, ) { - let point = self.elements.point_from_location(location); - // Use internal iterator manually as it is much more efficient. - state.iter().for_each(|node| { - self.values.insert(node, point); - }); + (self.callback)(state, location); } } From 87f97b0e822e7390e300529b7a8385b24cd2c9f0 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Tue, 1 Jul 2025 12:36:08 +0000 Subject: [PATCH 3/9] Remove obsolete comment. --- compiler/rustc_mir_transform/src/dest_prop.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index 6cc0a9c01d940..b1ebcee17c040 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -150,14 +150,6 @@ pub(super) struct DestinationPropagation; impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - // For now, only run at MIR opt level 3. Two things need to be changed before this can be - // turned on by default: - // 1. Because of the overeager removal of storage statements, this can cause stack space - // regressions. This opt is not the place to fix this though, it's a more general - // problem in MIR. - // 2. Despite being an overall perf improvement, this still causes a 30% regression in - // keccak. We can temporarily fix this by bounding function size, but in the long term - // we should fix this by being smarter about invalidating analysis results. sess.mir_opt_level() >= 2 } From f116249264c2b4f0367c5ce9945360ca98b33195 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Wed, 2 Jul 2025 08:36:22 +0000 Subject: [PATCH 4/9] Reimplement DestinationPropagation according to live ranges. --- compiler/rustc_index/src/interval.rs | 26 + .../rustc_mir_dataflow/src/impls/liveness.rs | 37 +- compiler/rustc_mir_dataflow/src/impls/mod.rs | 3 +- compiler/rustc_mir_dataflow/src/points.rs | 80 +- compiler/rustc_mir_transform/src/dest_prop.rs | 843 ++++++++---------- ...in.DestinationPropagation.panic-abort.diff | 16 +- ...n.DestinationPropagation.panic-unwind.diff | 16 +- ...e_111005.wrong.DestinationPropagation.diff | 4 +- ...ated_loop.PreCodegen.after.panic-abort.mir | 166 ++-- ...ward_loop.PreCodegen.after.panic-abort.mir | 122 +-- ...ard_loop.PreCodegen.after.panic-unwind.mir | 122 +-- 11 files changed, 665 insertions(+), 770 deletions(-) diff --git a/compiler/rustc_index/src/interval.rs b/compiler/rustc_index/src/interval.rs index 4af5bfcaee6dc..69a7a69610e04 100644 --- a/compiler/rustc_index/src/interval.rs +++ b/compiler/rustc_index/src/interval.rs @@ -219,6 +219,32 @@ impl IntervalSet { }) } + pub fn disjoint(&self, other: &IntervalSet) -> bool + where + I: Step, + { + let helper = move || { + let mut self_iter = self.iter_intervals(); + let mut other_iter = other.iter_intervals(); + + let mut self_current = self_iter.next()?; + let mut other_current = other_iter.next()?; + + loop { + if self_current.end <= other_current.start { + self_current = self_iter.next()?; + continue; + } + if other_current.end <= self_current.start { + other_current = other_iter.next()?; + continue; + } + return Some(false); + } + }; + helper().unwrap_or(true) + } + pub fn is_empty(&self) -> bool { self.map.is_empty() } diff --git a/compiler/rustc_mir_dataflow/src/impls/liveness.rs b/compiler/rustc_mir_dataflow/src/impls/liveness.rs index 6ec1b03a34e68..5eba474a60c7a 100644 --- a/compiler/rustc_mir_dataflow/src/impls/liveness.rs +++ b/compiler/rustc_mir_dataflow/src/impls/liveness.rs @@ -92,7 +92,7 @@ impl<'tcx> Visitor<'tcx> for TransferFunction<'_> { } match DefUse::for_place(*place, context) { - Some(DefUse::Def) => { + DefUse::Def => { if let PlaceContext::MutatingUse( MutatingUseContext::Call | MutatingUseContext::AsmOutput, ) = context @@ -105,8 +105,8 @@ impl<'tcx> Visitor<'tcx> for TransferFunction<'_> { self.0.kill(place.local); } } - Some(DefUse::Use) => self.0.gen_(place.local), - None => {} + DefUse::Use => self.0.gen_(place.local), + DefUse::PartialWrite | DefUse::NonUse => {} } self.visit_projection(place.as_ref(), context, location); @@ -131,23 +131,29 @@ impl<'tcx> Visitor<'tcx> for YieldResumeEffect<'_> { } #[derive(Eq, PartialEq, Clone)] -enum DefUse { +pub enum DefUse { + /// Full write to the local. Def, + /// Read of any part of the local. Use, + /// Partial write to the local. + PartialWrite, + /// Non-use, like debuginfo. + NonUse, } impl DefUse { fn apply(state: &mut DenseBitSet, place: Place<'_>, context: PlaceContext) { match DefUse::for_place(place, context) { - Some(DefUse::Def) => state.kill(place.local), - Some(DefUse::Use) => state.gen_(place.local), - None => {} + DefUse::Def => state.kill(place.local), + DefUse::Use => state.gen_(place.local), + DefUse::PartialWrite | DefUse::NonUse => {} } } - fn for_place(place: Place<'_>, context: PlaceContext) -> Option { + pub fn for_place(place: Place<'_>, context: PlaceContext) -> DefUse { match context { - PlaceContext::NonUse(_) => None, + PlaceContext::NonUse(_) => DefUse::NonUse, PlaceContext::MutatingUse( MutatingUseContext::Call @@ -156,21 +162,20 @@ impl DefUse { | MutatingUseContext::Store | MutatingUseContext::Deinit, ) => { + // Treat derefs as a use of the base local. `*p = 4` is not a def of `p` but a use. if place.is_indirect() { - // Treat derefs as a use of the base local. `*p = 4` is not a def of `p` but a - // use. - Some(DefUse::Use) + DefUse::Use } else if place.projection.is_empty() { - Some(DefUse::Def) + DefUse::Def } else { - None + DefUse::PartialWrite } } // Setting the discriminant is not a use because it does no reading, but it is also not // a def because it does not overwrite the whole place PlaceContext::MutatingUse(MutatingUseContext::SetDiscriminant) => { - place.is_indirect().then_some(DefUse::Use) + if place.is_indirect() { DefUse::Use } else { DefUse::PartialWrite } } // All other contexts are uses... @@ -188,7 +193,7 @@ impl DefUse { | NonMutatingUseContext::PlaceMention | NonMutatingUseContext::FakeBorrow | NonMutatingUseContext::SharedBorrow, - ) => Some(DefUse::Use), + ) => DefUse::Use, PlaceContext::MutatingUse(MutatingUseContext::Projection) | PlaceContext::NonMutatingUse(NonMutatingUseContext::Projection) => { diff --git a/compiler/rustc_mir_dataflow/src/impls/mod.rs b/compiler/rustc_mir_dataflow/src/impls/mod.rs index 3f29b819a6d18..6d573e1c00e1c 100644 --- a/compiler/rustc_mir_dataflow/src/impls/mod.rs +++ b/compiler/rustc_mir_dataflow/src/impls/mod.rs @@ -9,7 +9,8 @@ pub use self::initialized::{ MaybeUninitializedPlaces, MaybeUninitializedPlacesDomain, }; pub use self::liveness::{ - MaybeLiveLocals, MaybeTransitiveLiveLocals, TransferFunction as LivenessTransferFunction, + DefUse, MaybeLiveLocals, MaybeTransitiveLiveLocals, + TransferFunction as LivenessTransferFunction, }; pub use self::storage_liveness::{ MaybeRequiresStorage, MaybeStorageDead, MaybeStorageLive, always_storage_live_locals, diff --git a/compiler/rustc_mir_dataflow/src/points.rs b/compiler/rustc_mir_dataflow/src/points.rs index 55a373d4c5199..e3d1e04a319ba 100644 --- a/compiler/rustc_mir_dataflow/src/points.rs +++ b/compiler/rustc_mir_dataflow/src/points.rs @@ -1,9 +1,5 @@ -use rustc_index::bit_set::DenseBitSet; -use rustc_index::interval::SparseIntervalMatrix; use rustc_index::{Idx, IndexVec}; -use rustc_middle::mir::{self, BasicBlock, Body, Location}; - -use crate::framework::{Analysis, Direction, Results, ResultsVisitor, visit_results}; +use rustc_middle::mir::{BasicBlock, Body, Location}; /// Maps between a `Location` and a `PointIndex` (and vice versa). pub struct DenseLocationMap { @@ -93,77 +89,3 @@ rustc_index::newtype_index! { #[debug_format = "PointIndex({})"] pub struct PointIndex {} } - -/// Add points depending on the result of the given dataflow analysis. -pub fn save_as_intervals<'tcx, N, A>( - elements: &DenseLocationMap, - body: &mir::Body<'tcx>, - mut analysis: A, - results: Results, -) -> SparseIntervalMatrix -where - N: Idx, - A: Analysis<'tcx, Domain = DenseBitSet>, -{ - let mut values = SparseIntervalMatrix::new(elements.num_points()); - let reachable_blocks = mir::traversal::reachable_as_bitset(body); - if A::Direction::IS_BACKWARD { - // Iterate blocks in decreasing order, to visit locations in decreasing order. This - // allows to use the more efficient `prepend` method to interval sets. - let callback = |state: &DenseBitSet, location| { - let point = elements.point_from_location(location); - // Use internal iterator manually as it is much more efficient. - state.iter().for_each(|node| values.prepend(node, point)); - }; - let mut visitor = Visitor { callback }; - visit_results( - body, - // Note the `.rev()`. - body.basic_blocks.indices().filter(|&bb| reachable_blocks.contains(bb)).rev(), - &mut analysis, - &results, - &mut visitor, - ); - } else { - // Iterate blocks in increasing order, to visit locations in increasing order. This - // allows to use the more efficient `append` method to interval sets. - let callback = |state: &DenseBitSet, location| { - let point = elements.point_from_location(location); - // Use internal iterator manually as it is much more efficient. - state.iter().for_each(|node| values.append(node, point)); - }; - let mut visitor = Visitor { callback }; - visit_results(body, reachable_blocks.iter(), &mut analysis, &results, &mut visitor); - } - values -} - -struct Visitor { - callback: F, -} - -impl<'tcx, A, F> ResultsVisitor<'tcx, A> for Visitor -where - A: Analysis<'tcx>, - F: FnMut(&A::Domain, Location), -{ - fn visit_after_primary_statement_effect<'mir>( - &mut self, - _analysis: &mut A, - state: &A::Domain, - _statement: &'mir mir::Statement<'tcx>, - location: Location, - ) { - (self.callback)(state, location); - } - - fn visit_after_primary_terminator_effect<'mir>( - &mut self, - _analysis: &mut A, - state: &A::Domain, - _terminator: &'mir mir::Terminator<'tcx>, - location: Location, - ) { - (self.callback)(state, location); - } -} diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index b1ebcee17c040..ca5801d3b7dcd 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -59,6 +59,12 @@ //! The first two conditions are simple structural requirements on the `Assign` statements that can //! be trivially checked. The third requirement however is more difficult and costly to check. //! +//! ## Current implementation +//! +//! The current implementation relies on live range computation to check for conflicts. We only +//! allow to merge locals that have disjoint live ranges. The live range are defined with +//! half-statement granularity, so as to make all writes be live for at least a half statement. +//! //! ## Future Improvements //! //! There are a number of ways in which this pass could be improved in the future: @@ -117,9 +123,8 @@ //! - Layout optimizations for coroutines have been added to improve code generation for //! async/await, which are very similar in spirit to what this optimization does. //! -//! Also, rustc now has a simple NRVO pass (see `nrvo.rs`), which handles a subset of the cases that -//! this destination propagation pass handles, proving that similar optimizations can be performed -//! on MIR. +//! [The next approach][attempt 4] computes a conflict matrix between locals by forbidding merging +//! locals with competing writes or with one write while the other is live. //! //! ## Pre/Post Optimization //! @@ -130,20 +135,18 @@ //! [attempt 1]: https://github.com/rust-lang/rust/pull/47954 //! [attempt 2]: https://github.com/rust-lang/rust/pull/71003 //! [attempt 3]: https://github.com/rust-lang/rust/pull/72632 +//! [attempt 4]: https://github.com/rust-lang/rust/pull/96451 -use rustc_data_structures::fx::{FxIndexMap, IndexEntry, IndexOccupiedEntry}; +use rustc_data_structures::fx::FxIndexMap; use rustc_index::bit_set::DenseBitSet; use rustc_index::interval::SparseIntervalMatrix; -use rustc_middle::bug; -use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor}; -use rustc_middle::mir::{ - Body, HasLocalDecls, InlineAsmOperand, Local, LocalKind, Location, Operand, PassWhere, Place, - Rvalue, Statement, StatementKind, TerminatorKind, dump_mir, traversal, -}; +use rustc_index::{IndexSlice, IndexVec, newtype_index}; +use rustc_middle::mir::visit::{MutVisitor, NonMutatingUseContext, PlaceContext, Visitor}; +use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; -use rustc_mir_dataflow::Analysis; -use rustc_mir_dataflow::impls::MaybeLiveLocals; -use rustc_mir_dataflow::points::{DenseLocationMap, PointIndex, save_as_intervals}; +use rustc_mir_dataflow::impls::DefUse; +use rustc_mir_dataflow::points::{DenseLocationMap, PointIndex}; +use rustc_mir_dataflow::{Analysis, Backward, Results}; use tracing::{debug, trace}; pub(super) struct DestinationPropagation; @@ -153,84 +156,62 @@ impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation { sess.mir_opt_level() >= 2 } + #[tracing::instrument(level = "trace", skip(self, tcx, body))] fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let def_id = body.source.def_id(); - let mut candidates = Candidates::default(); - let mut write_info = WriteInfo::default(); - trace!(func = ?tcx.def_path_str(def_id)); + trace!(?def_id); let borrowed = rustc_mir_dataflow::impls::borrowed_locals(body); - let live = MaybeLiveLocals.iterate_to_fixpoint(tcx, body, Some("MaybeLiveLocals-DestProp")); + let candidates = Candidates::find(body, &borrowed); + trace!(?candidates); + if candidates.c.is_empty() { + return; + } + + let live = + MaybeTwoStepLiveLocals.iterate_to_fixpoint(tcx, body, Some("MaybeLiveLocals-DestProp")); + let points = DenseLocationMap::new(body); - let mut live = save_as_intervals(&points, body, live.analysis, live.results); - - // In order to avoid having to collect data for every single pair of locals in the body, we - // do not allow doing more than one merge for places that are derived from the same local at - // once. To avoid missed opportunities, we instead iterate to a fixed point - we'll refer to - // each of these iterations as a "round." - // - // Reaching a fixed point could in theory take up to `min(l, s)` rounds - however, we do not - // expect to see MIR like that. To verify this, a test was run against `[rust-lang/regex]` - - // the average MIR body saw 1.32 full iterations of this loop. The most that was hit were 30 - // for a single function. Only 80/2801 (2.9%) of functions saw at least 5. - // - // [rust-lang/regex]: - // https://github.com/rust-lang/regex/tree/b5372864e2df6a2f5e543a556a62197f50ca3650 - let mut round_count = 0; - loop { - // PERF: Can we do something smarter than recalculating the candidates and liveness - // results? - candidates.reset_and_find(body, &borrowed); - trace!(?candidates); - dest_prop_mir_dump(tcx, body, &points, &live, round_count); - - FilterInformation::filter_liveness( - &mut candidates, - &points, - &live, - &mut write_info, - body, - ); - - // Because we only filter once per round, it is unsound to use a local for more than - // one merge operation within a single round of optimizations. We store here which ones - // we have already used. - let mut merged_locals: DenseBitSet = - DenseBitSet::new_empty(body.local_decls.len()); - - // This is the set of merges we will apply this round. It is a subset of the candidates. - let mut merges = FxIndexMap::default(); - - for (src, candidates) in candidates.c.iter() { - if merged_locals.contains(*src) { - continue; - } - let Some(dest) = candidates.iter().find(|dest| !merged_locals.contains(**dest)) - else { - continue; - }; - - // Replace `src` by `dest` everywhere. - merges.insert(*src, *dest); - merged_locals.insert(*src); - merged_locals.insert(*dest); - - // Update liveness information based on the merge we just performed. - // Every location where `src` was live, `dest` will be live. - live.union_rows(*src, *dest); - } - trace!(merging = ?merges); + let mut relevant = RelevantLocals::compute(&candidates, body.local_decls.len()); + let mut live = save_as_intervals(&points, body, &relevant.original, live.results); - if merges.is_empty() { - break; - } - round_count += 1; + dest_prop_mir_dump(tcx, body, &points, &live, &relevant); + + let mut merged_locals = DenseBitSet::new_empty(body.local_decls.len()); + + for (src, candidates) in candidates.c.into_iter() { + trace!(?src, ?candidates); + + let Some(src) = relevant.find(src) else { continue }; + let Some(src_live_ranges) = &live.row(src) else { continue }; + trace!(?src, ?src_live_ranges); + + let dst = candidates.into_iter().find_map(|dst| { + let dst = relevant.find(dst)?; + let dst_live_ranges = &live.row(dst)?; + trace!(?dst, ?dst_live_ranges); + + let disjoint = src_live_ranges.disjoint(dst_live_ranges); + disjoint.then_some(dst) + }); + let Some(dst) = dst else { continue }; + + merged_locals.insert(relevant.original[src]); + merged_locals.insert(relevant.original[dst]); - apply_merges(body, tcx, merges, merged_locals); + relevant.union(src, dst); + live.union_rows(src, dst); } + trace!(?merged_locals); - trace!(round_count); + relevant.make_idempotent(); + + if merged_locals.is_empty() { + return; + } + + apply_merges(body, tcx, relevant, merged_locals); } fn is_required(&self) -> bool { @@ -238,30 +219,6 @@ impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation { } } -#[derive(Debug, Default)] -struct Candidates { - /// The set of candidates we are considering in this optimization. - /// - /// We will always merge the key into at most one of its values. - /// - /// Whether a place ends up in the key or the value does not correspond to whether it appears as - /// the lhs or rhs of any assignment. As a matter of fact, the places in here might never appear - /// in an assignment at all. This happens because if we see an assignment like this: - /// - /// ```ignore (syntax-highlighting-only) - /// _1.0 = _2.0 - /// ``` - /// - /// We will still report that we would like to merge `_1` and `_2` in an attempt to allow us to - /// remove that assignment. - c: FxIndexMap>, - - /// A reverse index of the `c` set; if the `c` set contains `a => Place { local: b, proj }`, - /// then this contains `b => a`. - // PERF: Possibly these should be `SmallVec`s? - reverse: FxIndexMap>, -} - ////////////////////////////////////////////////////////// // Merging // @@ -270,16 +227,16 @@ struct Candidates { fn apply_merges<'tcx>( body: &mut Body<'tcx>, tcx: TyCtxt<'tcx>, - merges: FxIndexMap, + relevant: RelevantLocals, merged_locals: DenseBitSet, ) { - let mut merger = Merger { tcx, merges, merged_locals }; + let mut merger = Merger { tcx, relevant, merged_locals }; merger.visit_body_preserves_cfg(body); } struct Merger<'tcx> { tcx: TyCtxt<'tcx>, - merges: FxIndexMap, + relevant: RelevantLocals, merged_locals: DenseBitSet, } @@ -289,8 +246,8 @@ impl<'tcx> MutVisitor<'tcx> for Merger<'tcx> { } fn visit_local(&mut self, local: &mut Local, _: PlaceContext, _location: Location) { - if let Some(dest) = self.merges.get(local) { - *local = *dest; + if let Some(relevant) = self.relevant.find(*local) { + *local = self.relevant.original[relevant]; } } @@ -328,376 +285,116 @@ impl<'tcx> MutVisitor<'tcx> for Merger<'tcx> { } ////////////////////////////////////////////////////////// -// Liveness filtering +// Relevant locals // -// This section enforces bullet point 2 +// Small utility to reduce size of the conflict matrix by only considering locals that appear in +// the candidates -struct FilterInformation<'a, 'tcx> { - body: &'a Body<'tcx>, - points: &'a DenseLocationMap, - live: &'a SparseIntervalMatrix, - candidates: &'a mut Candidates, - write_info: &'a mut WriteInfo, - at: Location, +newtype_index! { + /// Represent a subset of locals which appear in candidates. + struct RelevantLocal {} } -// We first implement some utility functions which we will expose removing candidates according to -// different needs. Throughout the liveness filtering, the `candidates` are only ever accessed -// through these methods, and not directly. -impl Candidates { - /// Collects the candidates for merging. - /// - /// This is responsible for enforcing the first and third bullet point. - fn reset_and_find<'tcx>(&mut self, body: &Body<'tcx>, borrowed: &DenseBitSet) { - self.c.clear(); - self.reverse.clear(); - let mut visitor = FindAssignments { body, candidates: &mut self.c, borrowed }; - visitor.visit_body(body); - // Deduplicate candidates. - for (_, cands) in self.c.iter_mut() { - cands.sort(); - cands.dedup(); - } - // Generate the reverse map. - for (src, cands) in self.c.iter() { - for dest in cands.iter().copied() { - self.reverse.entry(dest).or_default().push(*src); - } - } - } - - /// Just `Vec::retain`, but the condition is inverted and we add debugging output - fn vec_filter_candidates( - src: Local, - v: &mut Vec, - mut f: impl FnMut(Local) -> CandidateFilter, - at: Location, - ) { - v.retain(|dest| { - let remove = f(*dest); - if remove == CandidateFilter::Remove { - trace!("eliminating {:?} => {:?} due to conflict at {:?}", src, dest, at); - } - remove == CandidateFilter::Keep - }); - } - - /// `vec_filter_candidates` but for an `Entry` - fn entry_filter_candidates( - mut entry: IndexOccupiedEntry<'_, Local, Vec>, - p: Local, - f: impl FnMut(Local) -> CandidateFilter, - at: Location, - ) { - let candidates = entry.get_mut(); - Self::vec_filter_candidates(p, candidates, f, at); - if candidates.len() == 0 { - // FIXME(#120456) - is `swap_remove` correct? - entry.swap_remove(); - } - } - - /// For all candidates `(p, q)` or `(q, p)` removes the candidate if `f(q)` says to do so - fn filter_candidates_by( - &mut self, - p: Local, - mut f: impl FnMut(Local) -> CandidateFilter, - at: Location, - ) { - // Cover the cases where `p` appears as a `src` - if let IndexEntry::Occupied(entry) = self.c.entry(p) { - Self::entry_filter_candidates(entry, p, &mut f, at); - } - // And the cases where `p` appears as a `dest` - let Some(srcs) = self.reverse.get_mut(&p) else { - return; - }; - // We use `retain` here to remove the elements from the reverse set if we've removed the - // matching candidate in the forward set. - srcs.retain(|src| { - if f(*src) == CandidateFilter::Keep { - return true; - } - let IndexEntry::Occupied(entry) = self.c.entry(*src) else { - return false; - }; - Self::entry_filter_candidates( - entry, - *src, - |dest| { - if dest == p { CandidateFilter::Remove } else { CandidateFilter::Keep } - }, - at, - ); - false - }); - } +#[derive(Debug)] +struct RelevantLocals { + original: IndexVec, + shrink: IndexVec>, + renames: IndexVec, } -#[derive(Copy, Clone, PartialEq, Eq)] -enum CandidateFilter { - Keep, - Remove, -} +impl RelevantLocals { + #[tracing::instrument(level = "trace", skip(candidates, num_locals), ret)] + fn compute(candidates: &Candidates, num_locals: usize) -> RelevantLocals { + let mut original = IndexVec::with_capacity(candidates.c.len()); + let mut shrink = IndexVec::from_elem_n(None, num_locals); -impl<'a, 'tcx> FilterInformation<'a, 'tcx> { - /// Filters the set of candidates to remove those that conflict. - /// - /// The steps we take are exactly those that are outlined at the top of the file. For each - /// statement/terminator, we collect the set of locals that are written to in that - /// statement/terminator, and then we remove all pairs of candidates that contain one such local - /// and another one that is live. - /// - /// We need to be careful about the ordering of operations within each statement/terminator - /// here. Many statements might write and read from more than one place, and we need to consider - /// them all. The strategy for doing this is as follows: We first gather all the places that are - /// written to within the statement/terminator via `WriteInfo`. Then, we use the liveness - /// analysis from *before* the statement/terminator (in the control flow sense) to eliminate - /// candidates - this is because we want to conservatively treat a pair of locals that is both - /// read and written in the statement/terminator to be conflicting, and the liveness analysis - /// before the statement/terminator will correctly report locals that are read in the - /// statement/terminator to be live. We are additionally conservative by treating all written to - /// locals as also being read from. - fn filter_liveness( - candidates: &mut Candidates, - points: &DenseLocationMap, - live: &SparseIntervalMatrix, - write_info: &mut WriteInfo, - body: &Body<'tcx>, - ) { - let mut this = FilterInformation { - body, - points, - live, - candidates, - // We don't actually store anything at this scope, we just keep things here to be able - // to reuse the allocation. - write_info, - // Doesn't matter what we put here, will be overwritten before being used - at: Location::START, + // Mark a local as relevant and record it into the maps. + let mut declare = |local| { + shrink.get_or_insert_with(local, || original.push(local)); }; - this.internal_filter_liveness(); - } - - fn internal_filter_liveness(&mut self) { - for (block, data) in traversal::preorder(self.body) { - self.at = Location { block, statement_index: data.statements.len() }; - self.write_info.for_terminator(&data.terminator().kind); - self.apply_conflicts(); - for (i, statement) in data.statements.iter().enumerate().rev() { - self.at = Location { block, statement_index: i }; - self.write_info.for_statement(&statement.kind, self.body); - self.apply_conflicts(); + for (&src, destinations) in candidates.c.iter() { + declare(src); + for &dest in destinations { + declare(dest) } } - } - fn apply_conflicts(&mut self) { - let writes = &self.write_info.writes; - for p in writes { - let other_skip = self.write_info.skip_pair.and_then(|(a, b)| { - if a == *p { - Some(b) - } else if b == *p { - Some(a) - } else { - None - } - }); - let at = self.points.point_from_location(self.at); - self.candidates.filter_candidates_by( - *p, - |q| { - if Some(q) == other_skip { - return CandidateFilter::Keep; - } - // It is possible that a local may be live for less than the - // duration of a statement This happens in the case of function - // calls or inline asm. Because of this, we also mark locals as - // conflicting when both of them are written to in the same - // statement. - if self.live.contains(q, at) || writes.contains(&q) { - CandidateFilter::Remove - } else { - CandidateFilter::Keep - } - }, - self.at, - ); - } + let renames = IndexVec::from_fn_n(|l| l, original.len()); + RelevantLocals { original, shrink, renames } } -} -/// Describes where a statement/terminator writes to -#[derive(Default, Debug)] -struct WriteInfo { - writes: Vec, - /// If this pair of locals is a candidate pair, completely skip processing it during this - /// statement. All other candidates are unaffected. - skip_pair: Option<(Local, Local)>, -} - -impl WriteInfo { - fn for_statement<'tcx>(&mut self, statement: &StatementKind<'tcx>, body: &Body<'tcx>) { - self.reset(); - match statement { - StatementKind::Assign(box (lhs, rhs)) => { - self.add_place(*lhs); - match rhs { - Rvalue::Use(op) => { - self.add_operand(op); - self.consider_skipping_for_assign_use(*lhs, op, body); - } - Rvalue::Repeat(op, _) => { - self.add_operand(op); - } - Rvalue::Cast(_, op, _) - | Rvalue::UnaryOp(_, op) - | Rvalue::ShallowInitBox(op, _) => { - self.add_operand(op); - } - Rvalue::BinaryOp(_, ops) => { - for op in [&ops.0, &ops.1] { - self.add_operand(op); - } - } - Rvalue::Aggregate(_, ops) => { - for op in ops { - self.add_operand(op); - } - } - Rvalue::WrapUnsafeBinder(op, _) => { - self.add_operand(op); - } - Rvalue::ThreadLocalRef(_) - | Rvalue::NullaryOp(_, _) - | Rvalue::Ref(_, _, _) - | Rvalue::RawPtr(_, _) - | Rvalue::Len(_) - | Rvalue::Discriminant(_) - | Rvalue::CopyForDeref(_) => {} - } - } - // Retags are technically also reads, but reporting them as a write suffices - StatementKind::SetDiscriminant { place, .. } - | StatementKind::Deinit(place) - | StatementKind::Retag(_, place) => { - self.add_place(**place); - } - StatementKind::Intrinsic(_) - | StatementKind::ConstEvalCounter - | StatementKind::Nop - | StatementKind::Coverage(_) - | StatementKind::StorageLive(_) - | StatementKind::StorageDead(_) - | StatementKind::BackwardIncompatibleDropHint { .. } - | StatementKind::PlaceMention(_) => {} - StatementKind::FakeRead(_) | StatementKind::AscribeUserType(_, _) => { - bug!("{:?} not found in this MIR phase", statement) - } + fn find(&self, src: Local) -> Option { + let mut src = self.shrink[src]?; + while let s2 = self.renames[src] + && src != s2 + { + src = s2 } + Some(src) } - fn consider_skipping_for_assign_use<'tcx>( - &mut self, - lhs: Place<'tcx>, - rhs: &Operand<'tcx>, - body: &Body<'tcx>, - ) { - let Some(rhs) = rhs.place() else { return }; - if let Some(pair) = places_to_candidate_pair(lhs, rhs, body) { - self.skip_pair = Some(pair); - } + fn union(&mut self, lhs: RelevantLocal, rhs: RelevantLocal) { + self.renames[lhs] = rhs; } - fn for_terminator<'tcx>(&mut self, terminator: &TerminatorKind<'tcx>) { - self.reset(); - match terminator { - TerminatorKind::SwitchInt { discr: op, .. } - | TerminatorKind::Assert { cond: op, .. } => { - self.add_operand(op); - } - TerminatorKind::Call { destination, func, args, .. } => { - self.add_place(*destination); - self.add_operand(func); - for arg in args { - self.add_operand(&arg.node); - } - } - TerminatorKind::TailCall { func, args, .. } => { - self.add_operand(func); - for arg in args { - self.add_operand(&arg.node); - } - } - TerminatorKind::InlineAsm { operands, .. } => { - for asm_operand in operands { - match asm_operand { - InlineAsmOperand::In { value, .. } => { - self.add_operand(value); - } - InlineAsmOperand::Out { place, .. } => { - if let Some(place) = place { - self.add_place(*place); - } - } - // Note that the `late` field in `InOut` is about whether the registers used - // for these things overlap, and is of absolutely no interest to us. - InlineAsmOperand::InOut { in_value, out_place, .. } => { - if let Some(place) = out_place { - self.add_place(*place); - } - self.add_operand(in_value); - } - InlineAsmOperand::Const { .. } - | InlineAsmOperand::SymFn { .. } - | InlineAsmOperand::SymStatic { .. } - | InlineAsmOperand::Label { .. } => {} - } - } - } - TerminatorKind::Goto { .. } - | TerminatorKind::UnwindResume - | TerminatorKind::UnwindTerminate(_) - | TerminatorKind::Return - | TerminatorKind::Unreachable { .. } => (), - TerminatorKind::Drop { .. } => { - // `Drop`s create a `&mut` and so are not considered - } - TerminatorKind::Yield { .. } - | TerminatorKind::CoroutineDrop - | TerminatorKind::FalseEdge { .. } - | TerminatorKind::FalseUnwind { .. } => { - bug!("{:?} not found in this MIR phase", terminator) + fn make_idempotent(&mut self) { + for l in self.renames.indices() { + let mut h = self.renames[l]; + while let h2 = self.renames[h] + && h != h2 + { + h = h2 } + self.renames[l] = h; + debug_assert_eq!(h, self.renames[h], "non-idempotent for {l:?}"); } } +} - fn add_place(&mut self, place: Place<'_>) { - self.writes.push(place.local); - } +///////////////////////////////////////////////////// +// Candidate accumulation + +#[derive(Debug, Default)] +struct Candidates { + /// The set of candidates we are considering in this optimization. + /// + /// We will always merge the key into at most one of its values. + /// + /// Whether a place ends up in the key or the value does not correspond to whether it appears as + /// the lhs or rhs of any assignment. As a matter of fact, the places in here might never appear + /// in an assignment at all. This happens because if we see an assignment like this: + /// + /// ```ignore (syntax-highlighting-only) + /// _1.0 = _2.0 + /// ``` + /// + /// We will still report that we would like to merge `_1` and `_2` in an attempt to allow us to + /// remove that assignment. + c: FxIndexMap>, +} + +// We first implement some utility functions which we will expose removing candidates according to +// different needs. Throughout the liveness filtering, the `candidates` are only ever accessed +// through these methods, and not directly. +impl Candidates { + /// Collects the candidates for merging. + /// + /// This is responsible for enforcing the first and third bullet point. + fn find(body: &Body<'_>, borrowed: &DenseBitSet) -> Candidates { + let mut visitor = FindAssignments { body, candidates: Default::default(), borrowed }; + visitor.visit_body(body); - fn add_operand<'tcx>(&mut self, op: &Operand<'tcx>) { - match op { - // FIXME(JakobDegen): In a previous version, the `Move` case was incorrectly treated as - // being a read only. This was unsound, however we cannot add a regression test because - // it is not possible to set this off with current MIR. Once we have that ability, a - // regression test should be added. - Operand::Move(p) => self.add_place(*p), - Operand::Copy(_) | Operand::Constant(_) => (), + // Deduplicate candidates. + for (_, cands) in visitor.candidates.iter_mut() { + cands.sort(); + cands.dedup(); } - } - fn reset(&mut self) { - self.writes.clear(); - self.skip_pair = None; + Candidates { c: visitor.candidates } } } -///////////////////////////////////////////////////// -// Candidate accumulation - /// If the pair of places is being considered for merging, returns the candidate which would be /// merged in order to accomplish this. /// @@ -735,7 +432,7 @@ fn places_to_candidate_pair<'tcx>( struct FindAssignments<'a, 'tcx> { body: &'a Body<'tcx>, - candidates: &'a mut FxIndexMap>, + candidates: FxIndexMap>, borrowed: &'a DenseBitSet, } @@ -795,18 +492,248 @@ fn dest_prop_mir_dump<'tcx>( tcx: TyCtxt<'tcx>, body: &Body<'tcx>, points: &DenseLocationMap, - live: &SparseIntervalMatrix, - round: usize, + live: &SparseIntervalMatrix, + relevant: &RelevantLocals, ) { - let locals_live_at = |location| { + let locals_live_at = |location, effect| { let location = points.point_from_location(location); - live.rows().filter(|&r| live.contains(r, location)).collect::>() + let location = TwoStepIndex::new(location, effect); + live.rows() + .filter(|&r| live.contains(r, location)) + .map(|rl| relevant.original[rl]) + .collect::>() }; - dump_mir(tcx, false, "DestinationPropagation-dataflow", &round, body, |pass_where, w| { + dump_mir(tcx, false, "DestinationPropagation-dataflow", &0, body, |pass_where, w| { if let PassWhere::BeforeLocation(loc) = pass_where { - writeln!(w, " // live: {:?}", locals_live_at(loc))?; + writeln!(w, " // before: {:?}", locals_live_at(loc, Effect::Before))?; + } + if let PassWhere::AfterLocation(loc) = pass_where { + writeln!(w, " // after: {:?}", locals_live_at(loc, Effect::After))?; } Ok(()) }); } + +struct MaybeTwoStepLiveLocals; + +#[derive(Copy, Clone, Debug)] +enum Effect { + Before, + After, +} + +rustc_index::newtype_index! { + /// A `PointIndex` but with the lower bit encoding early/late inside the statement. + #[orderable] + #[debug_format = "TwoStepIndex({})"] + struct TwoStepIndex {} +} + +impl TwoStepIndex { + fn new(point: PointIndex, effect: Effect) -> TwoStepIndex { + let effect = match effect { + Effect::Before => 0, + Effect::After => 1, + }; + TwoStepIndex::from_u32(2 * point.as_u32() + (effect as u32)) + } +} + +struct VisitPlacesWith(F); + +impl<'tcx, F> Visitor<'tcx> for VisitPlacesWith +where + F: FnMut(Place<'tcx>, PlaceContext), +{ + fn visit_place(&mut self, place: &Place<'tcx>, ctxt: PlaceContext, _: Location) { + (self.0)(*place, ctxt); + for proj in place.projection.iter() { + match proj { + ProjectionElem::Index(index) => (self.0)( + index.into(), + PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy), + ), + ProjectionElem::Deref + | ProjectionElem::Field(..) + | ProjectionElem::Downcast(..) + | ProjectionElem::ConstantIndex { .. } + | ProjectionElem::Subslice { .. } + | ProjectionElem::OpaqueCast(..) + | ProjectionElem::UnwrapUnsafeBinder(..) + | ProjectionElem::Subtype(..) => {} + }; + } + } +} + +impl<'tcx> Analysis<'tcx> for MaybeTwoStepLiveLocals { + type Domain = DenseBitSet; + type Direction = Backward; + + const NAME: &'static str = "transitive liveness"; + + fn bottom_value(&self, body: &Body<'tcx>) -> DenseBitSet { + // bottom = not live + DenseBitSet::new_empty(body.local_decls.len()) + } + + fn initialize_start_block(&self, _: &Body<'tcx>, _: &mut DenseBitSet) { + // No variables are live until we observe a use + } + + // This happens between the previous statement and this one. + #[tracing::instrument(level = "trace", skip(self, statement))] + fn apply_primary_statement_effect( + &mut self, + state: &mut DenseBitSet, + statement: &Statement<'tcx>, + location: Location, + ) { + VisitPlacesWith(|place, ctxt| match DefUse::for_place(place, ctxt) { + DefUse::Def => { + state.remove(place.local); + } + DefUse::Use => { + state.insert(place.local); + } + DefUse::PartialWrite | DefUse::NonUse => {} + }) + .visit_statement(statement, location); + } + + // This happens between this statement and the next one. + #[tracing::instrument(level = "trace", skip(self, statement))] + fn apply_early_statement_effect( + &mut self, + state: &mut DenseBitSet, + statement: &Statement<'tcx>, + location: Location, + ) { + // We need to ensure we have a non-zero live range even for dead stores. This is done by + // marking all the writes locals as live in the second half of the statement. + VisitPlacesWith(|place: Place<'tcx>, ctxt| match DefUse::for_place(place, ctxt) { + DefUse::Def | DefUse::PartialWrite => { + state.insert(place.local); + } + // We already perform the reads in the first part of the statement. As statements are + // not splittable, we do not need to re-read the same values. + DefUse::Use | DefUse::NonUse => {} + }) + .visit_statement(statement, location); + } + + // We model terminator as a special case in this two-step analysis. Consider the terminator + // `destination = func(arg0...)`. + // + // -- state at (location, Effect::Before) + // read(arg0)... + // write(destination) + // -- state at (location, Effect::After) + // read(arg0)... + + // This happens between the last statement and the terminator. + #[tracing::instrument(level = "trace", skip(self, terminator))] + fn apply_primary_terminator_effect<'mir>( + &mut self, + state: &mut DenseBitSet, + terminator: &'mir Terminator<'tcx>, + location: Location, + ) -> TerminatorEdges<'mir, 'tcx> { + // Consider that all writes in this terminator happen at the start of the execution of the + // terminator. For instance if we pass a return-pointer to a `Call` terminator. + VisitPlacesWith(|place: Place<'tcx>, ctxt| match DefUse::for_place(place, ctxt) { + DefUse::Def => { + state.remove(place.local); + } + DefUse::Use => { + state.insert(place.local); + } + DefUse::PartialWrite | DefUse::NonUse => {} + }) + .visit_terminator(terminator, location); + terminator.edges() + } + + // This happens between the terminator and the end of the block. + #[tracing::instrument(level = "trace", skip(self, terminator))] + fn apply_early_terminator_effect<'mir>( + &mut self, + state: &mut DenseBitSet, + terminator: &'mir Terminator<'tcx>, + location: Location, + ) { + // Consider that all reads in this terminator happen at the end of the execution of the + // terminator, even after it may have written to the destination local. For instance if we + // pass arguments as pointers to a `Call` terminator. + VisitPlacesWith(|place: Place<'tcx>, ctxt| match DefUse::for_place(place, ctxt) { + DefUse::Def | DefUse::Use | DefUse::PartialWrite => { + state.insert(place.local); + } + DefUse::NonUse => {} + }) + .visit_terminator(terminator, location); + } +} + +/// Add points depending on the result of the given dataflow analysis. +fn save_as_intervals<'tcx>( + elements: &DenseLocationMap, + body: &Body<'tcx>, + relevant: &IndexSlice, + results: Results>, +) -> SparseIntervalMatrix { + let mut values = SparseIntervalMatrix::new(2 * elements.num_points()); + let mut state = MaybeTwoStepLiveLocals.bottom_value(body); + let reachable_blocks = traversal::reachable_as_bitset(body); + + let two_step_loc = |location, effect| { + let point = elements.point_from_location(location); + TwoStepIndex::new(point, effect) + }; + let mut prepend_at = |state: &mut DenseBitSet, twostep| { + for (relevant, &original) in relevant.iter_enumerated() { + if state.contains(original) { + values.prepend(relevant, twostep); + } + } + }; + + // Iterate blocks in decreasing order, to visit locations in decreasing order. This + // allows to use the more efficient `prepend` method to interval sets. + for block in body.basic_blocks.indices().rev() { + if !reachable_blocks.contains(block) { + continue; + } + + state.clone_from(&results[block]); + + let block_data = &body.basic_blocks[block]; + let loc = Location { block, statement_index: block_data.statements.len() }; + + let term = block_data.terminator(); + let mut twostep = two_step_loc(loc, Effect::After); + MaybeTwoStepLiveLocals.apply_early_terminator_effect(&mut state, term, loc); + prepend_at(&mut state, twostep); + + twostep = TwoStepIndex::from_u32(twostep.as_u32() - 1); + debug_assert_eq!(twostep, two_step_loc(loc, Effect::Before)); + MaybeTwoStepLiveLocals.apply_primary_terminator_effect(&mut state, term, loc); + prepend_at(&mut state, twostep); + + for (statement_index, stmt) in block_data.statements.iter().enumerate().rev() { + let loc = Location { block, statement_index }; + twostep = TwoStepIndex::from_u32(twostep.as_u32() - 1); + debug_assert_eq!(twostep, two_step_loc(loc, Effect::After)); + MaybeTwoStepLiveLocals.apply_early_statement_effect(&mut state, stmt, loc); + prepend_at(&mut state, twostep); + + twostep = TwoStepIndex::from_u32(twostep.as_u32() - 1); + debug_assert_eq!(twostep, two_step_loc(loc, Effect::Before)); + MaybeTwoStepLiveLocals.apply_primary_statement_effect(&mut state, stmt, loc); + prepend_at(&mut state, twostep); + } + } + + values +} diff --git a/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-abort.diff b/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-abort.diff index 5d8aaedae3742..f9b8881ae3b15 100644 --- a/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-abort.diff +++ b/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-abort.diff @@ -9,15 +9,15 @@ let mut _6: i32; scope 1 { - debug x => _1; -+ debug x => _6; ++ debug x => _4; let _2: i32; scope 2 { - debug y => _2; -+ debug y => _6; ++ debug y => _4; let _3: i32; scope 3 { - debug z => _3; -+ debug z => _6; ++ debug z => _4; } } } @@ -26,7 +26,7 @@ - StorageLive(_1); - _1 = val() -> [return: bb1, unwind unreachable]; + nop; -+ _6 = val() -> [return: bb1, unwind unreachable]; ++ _4 = val() -> [return: bb1, unwind unreachable]; } bb1: { @@ -47,16 +47,14 @@ + nop; + nop; StorageLive(_5); -- StorageLive(_6); + StorageLive(_6); - _6 = copy _1; -+ nop; -+ nop; ++ _6 = copy _4; _5 = std::mem::drop::(move _6) -> [return: bb2, unwind unreachable]; } bb2: { -- StorageDead(_6); -+ nop; + StorageDead(_6); StorageDead(_5); _0 = const (); - StorageDead(_3); diff --git a/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-unwind.diff b/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-unwind.diff index 05c9bcc1d7371..013e40ff462de 100644 --- a/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-unwind.diff +++ b/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-unwind.diff @@ -9,15 +9,15 @@ let mut _6: i32; scope 1 { - debug x => _1; -+ debug x => _6; ++ debug x => _4; let _2: i32; scope 2 { - debug y => _2; -+ debug y => _6; ++ debug y => _4; let _3: i32; scope 3 { - debug z => _3; -+ debug z => _6; ++ debug z => _4; } } } @@ -26,7 +26,7 @@ - StorageLive(_1); - _1 = val() -> [return: bb1, unwind continue]; + nop; -+ _6 = val() -> [return: bb1, unwind continue]; ++ _4 = val() -> [return: bb1, unwind continue]; } bb1: { @@ -47,16 +47,14 @@ + nop; + nop; StorageLive(_5); -- StorageLive(_6); + StorageLive(_6); - _6 = copy _1; -+ nop; -+ nop; ++ _6 = copy _4; _5 = std::mem::drop::(move _6) -> [return: bb2, unwind continue]; } bb2: { -- StorageDead(_6); -+ nop; + StorageDead(_6); StorageDead(_5); _0 = const (); - StorageDead(_3); diff --git a/tests/mir-opt/dest-prop/nrvo_miscompile_111005.wrong.DestinationPropagation.diff b/tests/mir-opt/dest-prop/nrvo_miscompile_111005.wrong.DestinationPropagation.diff index 60cf9236f6c2a..cd8cb13c0ff96 100644 --- a/tests/mir-opt/dest-prop/nrvo_miscompile_111005.wrong.DestinationPropagation.diff +++ b/tests/mir-opt/dest-prop/nrvo_miscompile_111005.wrong.DestinationPropagation.diff @@ -9,9 +9,9 @@ - _2 = copy _1; - _0 = copy _2; - _2 = const 'b'; -+ nop; + _0 = copy _1; -+ _1 = const 'b'; ++ nop; ++ _0 = const 'b'; return; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir index 0a09a0fa0b2d3..e6a184640a8c0 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir @@ -7,27 +7,27 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { let mut _6: std::ptr::NonNull; let mut _9: *const T; let mut _10: usize; - let mut _27: std::option::Option<(usize, &T)>; - let mut _30: &impl Fn(usize, &T); - let mut _31: (usize, &T); - let _32: (); + let mut _28: std::option::Option<(usize, &T)>; + let mut _31: &impl Fn(usize, &T); + let mut _32: (usize, &T); + let _33: (); scope 1 { debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _6; debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).1: *const T) => _9; debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>; debug ((iter: Enumerate>).1: usize) => _10; - let _28: usize; - let _29: &T; + let _29: usize; + let _30: &T; scope 2 { - debug i => _28; - debug x => _29; + debug i => _29; + debug x => _30; } scope 18 (inlined > as Iterator>::next) { - let mut _22: std::option::Option<&T>; - let mut _25: (usize, bool); - let mut _26: (usize, &T); + let mut _23: std::option::Option<&T>; + let mut _26: (usize, bool); + let mut _27: (usize, &T); scope 19 { - let _24: usize; + let _25: usize; scope 24 { } } @@ -42,19 +42,20 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } } scope 25 (inlined as Try>::branch) { - let _23: &T; + let _24: &T; scope 26 { } } scope 28 (inlined as Iterator>::next) { let _11: std::ptr::NonNull; - let _12: std::ptr::NonNull; - let mut _15: bool; - let mut _19: usize; - let _21: &T; + let _13: std::ptr::NonNull; + let mut _16: bool; + let mut _20: usize; + let _22: &T; scope 29 { + let _12: *const T; scope 30 { - let _18: usize; + let _19: usize; scope 31 { scope 34 (inlined #[track_caller] core::num::::unchecked_sub) { scope 35 (inlined core::ub_checks::check_language_ub) { @@ -70,21 +71,21 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } } scope 38 (inlined as PartialEq>::eq) { - let mut _13: *mut T; let mut _14: *mut T; + let mut _15: *mut T; scope 39 (inlined NonNull::::as_ptr) { } scope 40 (inlined NonNull::::as_ptr) { } } scope 41 (inlined NonNull::::add) { - let mut _16: *const T; let mut _17: *const T; + let mut _18: *const T; scope 42 (inlined NonNull::::as_ptr) { } } scope 43 (inlined NonNull::::as_ref::<'_>) { - let _20: *const T; + let _21: *const T; scope 44 (inlined NonNull::::as_ptr) { } scope 45 (inlined std::ptr::mut_ptr::::cast_const) { @@ -168,73 +169,77 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } bb4: { - StorageLive(_27); - StorageLive(_24); + StorageLive(_28); StorageLive(_25); - StorageLive(_22); + StorageLive(_26); + StorageLive(_23); StorageLive(_11); - StorageLive(_18); - StorageLive(_19); StorageLive(_12); - StorageLive(_21); + StorageLive(_19); + StorageLive(_20); + StorageLive(_13); + StorageLive(_22); _11 = copy _6; + _12 = copy _9; switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb8]; } bb5: { - StorageLive(_15); - _12 = copy _9 as std::ptr::NonNull (Transmute); - StorageLive(_13); - _13 = copy _11 as *mut T (Transmute); + StorageLive(_16); + _13 = copy _12 as std::ptr::NonNull (Transmute); StorageLive(_14); - _14 = copy _12 as *mut T (Transmute); - _15 = Eq(copy _13, copy _14); + _14 = copy _11 as *mut T (Transmute); + StorageLive(_15); + _15 = copy _13 as *mut T (Transmute); + _16 = Eq(copy _14, copy _15); + StorageDead(_15); StorageDead(_14); - StorageDead(_13); - switchInt(move _15) -> [0: bb6, otherwise: bb7]; + switchInt(move _16) -> [0: bb6, otherwise: bb7]; } bb6: { - StorageDead(_15); - StorageLive(_17); - StorageLive(_16); - _16 = copy _11 as *const T (Transmute); - _17 = Offset(copy _16, const 1_usize); StorageDead(_16); - _6 = NonNull:: { pointer: copy _17 }; + StorageLive(_18); + StorageLive(_17); + _17 = copy _11 as *const T (Transmute); + _18 = Offset(copy _17, const 1_usize); StorageDead(_17); + _6 = NonNull:: { pointer: copy _18 }; + StorageDead(_18); goto -> bb13; } bb7: { - StorageDead(_15); - StorageDead(_21); - StorageDead(_12); + StorageDead(_16); + StorageDead(_22); + StorageDead(_13); + StorageDead(_20); StorageDead(_19); - StorageDead(_18); + StorageDead(_12); StorageDead(_11); goto -> bb10; } bb8: { - _18 = copy _9 as usize (Transmute); - switchInt(copy _18) -> [0: bb9, otherwise: bb12]; + _19 = copy _12 as usize (Transmute); + switchInt(copy _19) -> [0: bb9, otherwise: bb12]; } bb9: { - StorageDead(_21); - StorageDead(_12); + StorageDead(_22); + StorageDead(_13); + StorageDead(_20); StorageDead(_19); - StorageDead(_18); + StorageDead(_12); StorageDead(_11); goto -> bb10; } bb10: { - StorageDead(_22); + StorageDead(_23); + StorageDead(_26); StorageDead(_25); - StorageDead(_24); - StorageDead(_27); + StorageDead(_28); StorageDead(_10); drop(_2) -> [return: bb11, unwind unreachable]; } @@ -244,50 +249,51 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } bb12: { - _19 = SubUnchecked(copy _18, const 1_usize); - _9 = copy _19 as *const T (Transmute); + _20 = SubUnchecked(copy _19, const 1_usize); + _9 = copy _20 as *const T (Transmute); goto -> bb13; } bb13: { - StorageLive(_20); - _20 = copy _11 as *const T (Transmute); - _21 = &(*_20); - StorageDead(_20); - _22 = Option::<&T>::Some(copy _21); + StorageLive(_21); + _21 = copy _11 as *const T (Transmute); + _22 = &(*_21); StorageDead(_21); - StorageDead(_12); + _23 = Option::<&T>::Some(copy _22); + StorageDead(_22); + StorageDead(_13); + StorageDead(_20); StorageDead(_19); - StorageDead(_18); + StorageDead(_12); StorageDead(_11); - _23 = copy ((_22 as Some).0: &T); - StorageDead(_22); - _24 = copy _10; - _25 = AddWithOverflow(copy _10, const 1_usize); - assert(!move (_25.1: bool), "attempt to compute `{} + {}`, which would overflow", copy _10, const 1_usize) -> [success: bb14, unwind unreachable]; + _24 = copy ((_23 as Some).0: &T); + StorageDead(_23); + _25 = copy _10; + _26 = AddWithOverflow(copy _10, const 1_usize); + assert(!move (_26.1: bool), "attempt to compute `{} + {}`, which would overflow", copy _10, const 1_usize) -> [success: bb14, unwind unreachable]; } bb14: { - _10 = move (_25.0: usize); - StorageLive(_26); - _26 = (copy _24, copy _23); - _27 = Option::<(usize, &T)>::Some(move _26); + _10 = move (_26.0: usize); + StorageLive(_27); + _27 = (copy _25, copy _24); + _28 = Option::<(usize, &T)>::Some(move _27); + StorageDead(_27); StorageDead(_26); StorageDead(_25); - StorageDead(_24); - _28 = copy (((_27 as Some).0: (usize, &T)).0: usize); - _29 = copy (((_27 as Some).0: (usize, &T)).1: &T); - StorageLive(_30); - _30 = &_2; + _29 = copy (((_28 as Some).0: (usize, &T)).0: usize); + _30 = copy (((_28 as Some).0: (usize, &T)).1: &T); StorageLive(_31); - _31 = (copy _28, copy _29); - _32 = >::call(move _30, move _31) -> [return: bb15, unwind unreachable]; + _31 = &_2; + StorageLive(_32); + _32 = (copy _29, copy _30); + _33 = >::call(move _31, move _32) -> [return: bb15, unwind unreachable]; } bb15: { + StorageDead(_32); StorageDead(_31); - StorageDead(_30); - StorageDead(_27); + StorageDead(_28); goto -> bb4; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir index 77db75efa072e..540b7ac3d5520 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir @@ -6,27 +6,28 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { let mut _0: (); let mut _6: std::ptr::NonNull; let mut _9: *const T; - let mut _21: std::option::Option<&T>; - let mut _23: &impl Fn(&T); - let mut _24: (&T,); - let _25: (); + let mut _22: std::option::Option<&T>; + let mut _24: &impl Fn(&T); + let mut _25: (&T,); + let _26: (); scope 1 { debug ((iter: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _6; debug ((iter: std::slice::Iter<'_, T>).1: *const T) => _9; debug ((iter: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>; - let _22: &T; + let _23: &T; scope 2 { - debug x => _22; + debug x => _23; } scope 16 (inlined as Iterator>::next) { let _10: std::ptr::NonNull; - let _11: std::ptr::NonNull; - let mut _14: bool; - let mut _18: usize; - let _20: &T; + let _12: std::ptr::NonNull; + let mut _15: bool; + let mut _19: usize; + let _21: &T; scope 17 { + let _11: *const T; scope 18 { - let _17: usize; + let _18: usize; scope 19 { scope 22 (inlined #[track_caller] core::num::::unchecked_sub) { scope 23 (inlined core::ub_checks::check_language_ub) { @@ -42,21 +43,21 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } } scope 26 (inlined as PartialEq>::eq) { - let mut _12: *mut T; let mut _13: *mut T; + let mut _14: *mut T; scope 27 (inlined NonNull::::as_ptr) { } scope 28 (inlined NonNull::::as_ptr) { } } scope 29 (inlined NonNull::::add) { - let mut _15: *const T; let mut _16: *const T; + let mut _17: *const T; scope 30 (inlined NonNull::::as_ptr) { } } scope 31 (inlined NonNull::::as_ref::<'_>) { - let _19: *const T; + let _20: *const T; scope 32 (inlined NonNull::::as_ptr) { } scope 33 (inlined std::ptr::mut_ptr::::cast_const) { @@ -133,67 +134,71 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb4: { - StorageLive(_21); + StorageLive(_22); StorageLive(_10); - StorageLive(_17); - StorageLive(_18); StorageLive(_11); - StorageLive(_20); + StorageLive(_18); + StorageLive(_19); + StorageLive(_12); + StorageLive(_21); _10 = copy _6; + _11 = copy _9; switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb8]; } bb5: { - StorageLive(_14); - _11 = copy _9 as std::ptr::NonNull (Transmute); - StorageLive(_12); - _12 = copy _10 as *mut T (Transmute); + StorageLive(_15); + _12 = copy _11 as std::ptr::NonNull (Transmute); StorageLive(_13); - _13 = copy _11 as *mut T (Transmute); - _14 = Eq(copy _12, copy _13); + _13 = copy _10 as *mut T (Transmute); + StorageLive(_14); + _14 = copy _12 as *mut T (Transmute); + _15 = Eq(copy _13, copy _14); + StorageDead(_14); StorageDead(_13); - StorageDead(_12); - switchInt(move _14) -> [0: bb6, otherwise: bb7]; + switchInt(move _15) -> [0: bb6, otherwise: bb7]; } bb6: { - StorageDead(_14); - StorageLive(_16); - StorageLive(_15); - _15 = copy _10 as *const T (Transmute); - _16 = Offset(copy _15, const 1_usize); StorageDead(_15); - _6 = NonNull:: { pointer: copy _16 }; + StorageLive(_17); + StorageLive(_16); + _16 = copy _10 as *const T (Transmute); + _17 = Offset(copy _16, const 1_usize); StorageDead(_16); + _6 = NonNull:: { pointer: copy _17 }; + StorageDead(_17); goto -> bb13; } bb7: { - StorageDead(_14); - StorageDead(_20); - StorageDead(_11); + StorageDead(_15); + StorageDead(_21); + StorageDead(_12); + StorageDead(_19); StorageDead(_18); - StorageDead(_17); + StorageDead(_11); StorageDead(_10); goto -> bb10; } bb8: { - _17 = copy _9 as usize (Transmute); - switchInt(copy _17) -> [0: bb9, otherwise: bb12]; + _18 = copy _11 as usize (Transmute); + switchInt(copy _18) -> [0: bb9, otherwise: bb12]; } bb9: { - StorageDead(_20); - StorageDead(_11); + StorageDead(_21); + StorageDead(_12); + StorageDead(_19); StorageDead(_18); - StorageDead(_17); + StorageDead(_11); StorageDead(_10); goto -> bb10; } bb10: { - StorageDead(_21); + StorageDead(_22); drop(_2) -> [return: bb11, unwind unreachable]; } @@ -202,34 +207,35 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb12: { - _18 = SubUnchecked(copy _17, const 1_usize); - _9 = copy _18 as *const T (Transmute); + _19 = SubUnchecked(copy _18, const 1_usize); + _9 = copy _19 as *const T (Transmute); goto -> bb13; } bb13: { - StorageLive(_19); - _19 = copy _10 as *const T (Transmute); - _20 = &(*_19); - StorageDead(_19); - _21 = Option::<&T>::Some(copy _20); + StorageLive(_20); + _20 = copy _10 as *const T (Transmute); + _21 = &(*_20); StorageDead(_20); - StorageDead(_11); + _22 = Option::<&T>::Some(copy _21); + StorageDead(_21); + StorageDead(_12); + StorageDead(_19); StorageDead(_18); - StorageDead(_17); + StorageDead(_11); StorageDead(_10); - _22 = copy ((_21 as Some).0: &T); - StorageLive(_23); - _23 = &_2; + _23 = copy ((_22 as Some).0: &T); StorageLive(_24); - _24 = (copy _22,); - _25 = >::call(move _23, move _24) -> [return: bb14, unwind unreachable]; + _24 = &_2; + StorageLive(_25); + _25 = (copy _23,); + _26 = >::call(move _24, move _25) -> [return: bb14, unwind unreachable]; } bb14: { + StorageDead(_25); StorageDead(_24); - StorageDead(_23); - StorageDead(_21); + StorageDead(_22); goto -> bb4; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir index b2f0efcbcd161..e45bb76f33e68 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir @@ -6,27 +6,28 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { let mut _0: (); let mut _6: std::ptr::NonNull; let mut _9: *const T; - let mut _21: std::option::Option<&T>; - let mut _23: &impl Fn(&T); - let mut _24: (&T,); - let _25: (); + let mut _22: std::option::Option<&T>; + let mut _24: &impl Fn(&T); + let mut _25: (&T,); + let _26: (); scope 1 { debug ((iter: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _6; debug ((iter: std::slice::Iter<'_, T>).1: *const T) => _9; debug ((iter: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>; - let _22: &T; + let _23: &T; scope 2 { - debug x => _22; + debug x => _23; } scope 16 (inlined as Iterator>::next) { let _10: std::ptr::NonNull; - let _11: std::ptr::NonNull; - let mut _14: bool; - let mut _18: usize; - let _20: &T; + let _12: std::ptr::NonNull; + let mut _15: bool; + let mut _19: usize; + let _21: &T; scope 17 { + let _11: *const T; scope 18 { - let _17: usize; + let _18: usize; scope 19 { scope 22 (inlined #[track_caller] core::num::::unchecked_sub) { scope 23 (inlined core::ub_checks::check_language_ub) { @@ -42,21 +43,21 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } } scope 26 (inlined as PartialEq>::eq) { - let mut _12: *mut T; let mut _13: *mut T; + let mut _14: *mut T; scope 27 (inlined NonNull::::as_ptr) { } scope 28 (inlined NonNull::::as_ptr) { } } scope 29 (inlined NonNull::::add) { - let mut _15: *const T; let mut _16: *const T; + let mut _17: *const T; scope 30 (inlined NonNull::::as_ptr) { } } scope 31 (inlined NonNull::::as_ref::<'_>) { - let _19: *const T; + let _20: *const T; scope 32 (inlined NonNull::::as_ptr) { } scope 33 (inlined std::ptr::mut_ptr::::cast_const) { @@ -133,67 +134,71 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb4: { - StorageLive(_21); + StorageLive(_22); StorageLive(_10); - StorageLive(_17); - StorageLive(_18); StorageLive(_11); - StorageLive(_20); + StorageLive(_18); + StorageLive(_19); + StorageLive(_12); + StorageLive(_21); _10 = copy _6; + _11 = copy _9; switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb8]; } bb5: { - StorageLive(_14); - _11 = copy _9 as std::ptr::NonNull (Transmute); - StorageLive(_12); - _12 = copy _10 as *mut T (Transmute); + StorageLive(_15); + _12 = copy _11 as std::ptr::NonNull (Transmute); StorageLive(_13); - _13 = copy _11 as *mut T (Transmute); - _14 = Eq(copy _12, copy _13); + _13 = copy _10 as *mut T (Transmute); + StorageLive(_14); + _14 = copy _12 as *mut T (Transmute); + _15 = Eq(copy _13, copy _14); + StorageDead(_14); StorageDead(_13); - StorageDead(_12); - switchInt(move _14) -> [0: bb6, otherwise: bb7]; + switchInt(move _15) -> [0: bb6, otherwise: bb7]; } bb6: { - StorageDead(_14); - StorageLive(_16); - StorageLive(_15); - _15 = copy _10 as *const T (Transmute); - _16 = Offset(copy _15, const 1_usize); StorageDead(_15); - _6 = NonNull:: { pointer: copy _16 }; + StorageLive(_17); + StorageLive(_16); + _16 = copy _10 as *const T (Transmute); + _17 = Offset(copy _16, const 1_usize); StorageDead(_16); + _6 = NonNull:: { pointer: copy _17 }; + StorageDead(_17); goto -> bb13; } bb7: { - StorageDead(_14); - StorageDead(_20); - StorageDead(_11); + StorageDead(_15); + StorageDead(_21); + StorageDead(_12); + StorageDead(_19); StorageDead(_18); - StorageDead(_17); + StorageDead(_11); StorageDead(_10); goto -> bb10; } bb8: { - _17 = copy _9 as usize (Transmute); - switchInt(copy _17) -> [0: bb9, otherwise: bb12]; + _18 = copy _11 as usize (Transmute); + switchInt(copy _18) -> [0: bb9, otherwise: bb12]; } bb9: { - StorageDead(_20); - StorageDead(_11); + StorageDead(_21); + StorageDead(_12); + StorageDead(_19); StorageDead(_18); - StorageDead(_17); + StorageDead(_11); StorageDead(_10); goto -> bb10; } bb10: { - StorageDead(_21); + StorageDead(_22); drop(_2) -> [return: bb11, unwind continue]; } @@ -202,34 +207,35 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb12: { - _18 = SubUnchecked(copy _17, const 1_usize); - _9 = copy _18 as *const T (Transmute); + _19 = SubUnchecked(copy _18, const 1_usize); + _9 = copy _19 as *const T (Transmute); goto -> bb13; } bb13: { - StorageLive(_19); - _19 = copy _10 as *const T (Transmute); - _20 = &(*_19); - StorageDead(_19); - _21 = Option::<&T>::Some(copy _20); + StorageLive(_20); + _20 = copy _10 as *const T (Transmute); + _21 = &(*_20); StorageDead(_20); - StorageDead(_11); + _22 = Option::<&T>::Some(copy _21); + StorageDead(_21); + StorageDead(_12); + StorageDead(_19); StorageDead(_18); - StorageDead(_17); + StorageDead(_11); StorageDead(_10); - _22 = copy ((_21 as Some).0: &T); - StorageLive(_23); - _23 = &_2; + _23 = copy ((_22 as Some).0: &T); StorageLive(_24); - _24 = (copy _22,); - _25 = >::call(move _23, move _24) -> [return: bb14, unwind: bb15]; + _24 = &_2; + StorageLive(_25); + _25 = (copy _23,); + _26 = >::call(move _24, move _25) -> [return: bb14, unwind: bb15]; } bb14: { + StorageDead(_25); StorageDead(_24); - StorageDead(_23); - StorageDead(_21); + StorageDead(_22); goto -> bb4; } From 9a096d767e579ea36759f4d2ad81d37bcd2b7079 Mon Sep 17 00:00:00 2001 From: Camille Gillot Date: Mon, 18 Aug 2025 22:53:14 +0000 Subject: [PATCH 5/9] Use regular MaybeLiveLocals. --- compiler/rustc_index/src/interval.rs | 7 +- compiler/rustc_mir_transform/src/dest_prop.rs | 181 +++++------------- .../dest-prop/nrvo_miscompile_111005.rs | 3 +- ...e_111005.wrong.DestinationPropagation.diff | 4 +- 4 files changed, 57 insertions(+), 138 deletions(-) diff --git a/compiler/rustc_index/src/interval.rs b/compiler/rustc_index/src/interval.rs index 69a7a69610e04..e81e7e2bc442a 100644 --- a/compiler/rustc_index/src/interval.rs +++ b/compiler/rustc_index/src/interval.rs @@ -146,8 +146,11 @@ impl IntervalSet { let point = point.index() as u32; if let Some((first_start, _)) = self.map.first_mut() { - assert!(point < *first_start); - if point + 1 == *first_start { + assert!(point <= *first_start); + if point == *first_start { + // The point is already present in the set. + } else if point + 1 == *first_start { + // Just extend the first range. *first_start = point; } else { self.map.insert(0, (point, point)); diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index ca5801d3b7dcd..c4875e6adb665 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -140,13 +140,13 @@ use rustc_data_structures::fx::FxIndexMap; use rustc_index::bit_set::DenseBitSet; use rustc_index::interval::SparseIntervalMatrix; -use rustc_index::{IndexSlice, IndexVec, newtype_index}; +use rustc_index::{IndexVec, newtype_index}; use rustc_middle::mir::visit::{MutVisitor, NonMutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; -use rustc_mir_dataflow::impls::DefUse; +use rustc_mir_dataflow::impls::{DefUse, MaybeLiveLocals}; use rustc_mir_dataflow::points::{DenseLocationMap, PointIndex}; -use rustc_mir_dataflow::{Analysis, Backward, Results}; +use rustc_mir_dataflow::{Analysis, Results}; use tracing::{debug, trace}; pub(super) struct DestinationPropagation; @@ -169,12 +169,11 @@ impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation { return; } - let live = - MaybeTwoStepLiveLocals.iterate_to_fixpoint(tcx, body, Some("MaybeLiveLocals-DestProp")); + let live = MaybeLiveLocals.iterate_to_fixpoint(tcx, body, Some("MaybeLiveLocals-DestProp")); let points = DenseLocationMap::new(body); let mut relevant = RelevantLocals::compute(&candidates, body.local_decls.len()); - let mut live = save_as_intervals(&points, body, &relevant.original, live.results); + let mut live = save_as_intervals(&points, body, &relevant, live.results); dest_prop_mir_dump(tcx, body, &points, &live, &relevant); @@ -515,8 +514,6 @@ fn dest_prop_mir_dump<'tcx>( }); } -struct MaybeTwoStepLiveLocals; - #[derive(Copy, Clone, Debug)] enum Effect { Before, @@ -567,137 +564,29 @@ where } } -impl<'tcx> Analysis<'tcx> for MaybeTwoStepLiveLocals { - type Domain = DenseBitSet; - type Direction = Backward; - - const NAME: &'static str = "transitive liveness"; - - fn bottom_value(&self, body: &Body<'tcx>) -> DenseBitSet { - // bottom = not live - DenseBitSet::new_empty(body.local_decls.len()) - } - - fn initialize_start_block(&self, _: &Body<'tcx>, _: &mut DenseBitSet) { - // No variables are live until we observe a use - } - - // This happens between the previous statement and this one. - #[tracing::instrument(level = "trace", skip(self, statement))] - fn apply_primary_statement_effect( - &mut self, - state: &mut DenseBitSet, - statement: &Statement<'tcx>, - location: Location, - ) { - VisitPlacesWith(|place, ctxt| match DefUse::for_place(place, ctxt) { - DefUse::Def => { - state.remove(place.local); - } - DefUse::Use => { - state.insert(place.local); - } - DefUse::PartialWrite | DefUse::NonUse => {} - }) - .visit_statement(statement, location); - } - - // This happens between this statement and the next one. - #[tracing::instrument(level = "trace", skip(self, statement))] - fn apply_early_statement_effect( - &mut self, - state: &mut DenseBitSet, - statement: &Statement<'tcx>, - location: Location, - ) { - // We need to ensure we have a non-zero live range even for dead stores. This is done by - // marking all the writes locals as live in the second half of the statement. - VisitPlacesWith(|place: Place<'tcx>, ctxt| match DefUse::for_place(place, ctxt) { - DefUse::Def | DefUse::PartialWrite => { - state.insert(place.local); - } - // We already perform the reads in the first part of the statement. As statements are - // not splittable, we do not need to re-read the same values. - DefUse::Use | DefUse::NonUse => {} - }) - .visit_statement(statement, location); - } - - // We model terminator as a special case in this two-step analysis. Consider the terminator - // `destination = func(arg0...)`. - // - // -- state at (location, Effect::Before) - // read(arg0)... - // write(destination) - // -- state at (location, Effect::After) - // read(arg0)... - - // This happens between the last statement and the terminator. - #[tracing::instrument(level = "trace", skip(self, terminator))] - fn apply_primary_terminator_effect<'mir>( - &mut self, - state: &mut DenseBitSet, - terminator: &'mir Terminator<'tcx>, - location: Location, - ) -> TerminatorEdges<'mir, 'tcx> { - // Consider that all writes in this terminator happen at the start of the execution of the - // terminator. For instance if we pass a return-pointer to a `Call` terminator. - VisitPlacesWith(|place: Place<'tcx>, ctxt| match DefUse::for_place(place, ctxt) { - DefUse::Def => { - state.remove(place.local); - } - DefUse::Use => { - state.insert(place.local); - } - DefUse::PartialWrite | DefUse::NonUse => {} - }) - .visit_terminator(terminator, location); - terminator.edges() - } - - // This happens between the terminator and the end of the block. - #[tracing::instrument(level = "trace", skip(self, terminator))] - fn apply_early_terminator_effect<'mir>( - &mut self, - state: &mut DenseBitSet, - terminator: &'mir Terminator<'tcx>, - location: Location, - ) { - // Consider that all reads in this terminator happen at the end of the execution of the - // terminator, even after it may have written to the destination local. For instance if we - // pass arguments as pointers to a `Call` terminator. - VisitPlacesWith(|place: Place<'tcx>, ctxt| match DefUse::for_place(place, ctxt) { - DefUse::Def | DefUse::Use | DefUse::PartialWrite => { - state.insert(place.local); - } - DefUse::NonUse => {} - }) - .visit_terminator(terminator, location); - } -} - /// Add points depending on the result of the given dataflow analysis. fn save_as_intervals<'tcx>( elements: &DenseLocationMap, body: &Body<'tcx>, - relevant: &IndexSlice, + relevant: &RelevantLocals, results: Results>, ) -> SparseIntervalMatrix { let mut values = SparseIntervalMatrix::new(2 * elements.num_points()); - let mut state = MaybeTwoStepLiveLocals.bottom_value(body); + let mut state = MaybeLiveLocals.bottom_value(body); let reachable_blocks = traversal::reachable_as_bitset(body); let two_step_loc = |location, effect| { let point = elements.point_from_location(location); TwoStepIndex::new(point, effect) }; - let mut prepend_at = |state: &mut DenseBitSet, twostep| { - for (relevant, &original) in relevant.iter_enumerated() { - if state.contains(original) { - values.prepend(relevant, twostep); + let prepend_at = + |values: &mut SparseIntervalMatrix<_, _>, state: &DenseBitSet, twostep| { + for (relevant, &original) in relevant.original.iter_enumerated() { + if state.contains(original) { + values.prepend(relevant, twostep); + } } - } - }; + }; // Iterate blocks in decreasing order, to visit locations in decreasing order. This // allows to use the more efficient `prepend` method to interval sets. @@ -713,25 +602,51 @@ fn save_as_intervals<'tcx>( let term = block_data.terminator(); let mut twostep = two_step_loc(loc, Effect::After); - MaybeTwoStepLiveLocals.apply_early_terminator_effect(&mut state, term, loc); - prepend_at(&mut state, twostep); + prepend_at(&mut values, &state, twostep); + // Ensure we have a non-zero live range even for dead stores. This is done by marking all + // the written-to locals as live in the second half of the statement. + // We also ensure that operands read by terminators conflict with writes by that terminator. + // For instance a function call may read args after having written to the destination. + VisitPlacesWith(|place, ctxt| match DefUse::for_place(place, ctxt) { + DefUse::Def | DefUse::Use | DefUse::PartialWrite => { + if let Some(relevant) = relevant.shrink[place.local] { + values.insert(relevant, twostep); + } + } + DefUse::NonUse => {} + }) + .visit_terminator(term, loc); twostep = TwoStepIndex::from_u32(twostep.as_u32() - 1); debug_assert_eq!(twostep, two_step_loc(loc, Effect::Before)); - MaybeTwoStepLiveLocals.apply_primary_terminator_effect(&mut state, term, loc); - prepend_at(&mut state, twostep); + MaybeLiveLocals.apply_early_terminator_effect(&mut state, term, loc); + MaybeLiveLocals.apply_primary_terminator_effect(&mut state, term, loc); + prepend_at(&mut values, &state, twostep); for (statement_index, stmt) in block_data.statements.iter().enumerate().rev() { let loc = Location { block, statement_index }; twostep = TwoStepIndex::from_u32(twostep.as_u32() - 1); debug_assert_eq!(twostep, two_step_loc(loc, Effect::After)); - MaybeTwoStepLiveLocals.apply_early_statement_effect(&mut state, stmt, loc); - prepend_at(&mut state, twostep); + prepend_at(&mut values, &state, twostep); + // Ensure we have a non-zero live range even for dead stores. This is done by marking + // all the written-to locals as live in the second half of the statement. + VisitPlacesWith(|place, ctxt| match DefUse::for_place(place, ctxt) { + DefUse::Def | DefUse::PartialWrite => { + if let Some(relevant) = relevant.shrink[place.local] { + values.insert(relevant, twostep); + } + } + DefUse::Use | DefUse::NonUse => {} + }) + .visit_statement(stmt, loc); twostep = TwoStepIndex::from_u32(twostep.as_u32() - 1); debug_assert_eq!(twostep, two_step_loc(loc, Effect::Before)); - MaybeTwoStepLiveLocals.apply_primary_statement_effect(&mut state, stmt, loc); - prepend_at(&mut state, twostep); + MaybeLiveLocals.apply_early_statement_effect(&mut state, stmt, loc); + MaybeLiveLocals.apply_primary_statement_effect(&mut state, stmt, loc); + // ... but reads from operands are marked as live here so they do not conflict with + // the all the writes we manually marked as live in the second half of the statement. + prepend_at(&mut values, &state, twostep); } } diff --git a/tests/mir-opt/dest-prop/nrvo_miscompile_111005.rs b/tests/mir-opt/dest-prop/nrvo_miscompile_111005.rs index 22d4a8b59c5ce..5d425e84a24ed 100644 --- a/tests/mir-opt/dest-prop/nrvo_miscompile_111005.rs +++ b/tests/mir-opt/dest-prop/nrvo_miscompile_111005.rs @@ -1,4 +1,3 @@ -// skip-filecheck // This is a miscompilation, #111005 to track //@ test-mir-pass: DestinationPropagation @@ -10,6 +9,8 @@ use core::intrinsics::mir::*; // EMIT_MIR nrvo_miscompile_111005.wrong.DestinationPropagation.diff #[custom_mir(dialect = "runtime", phase = "initial")] pub fn wrong(arg: char) -> char { + // CHECK-LABEL: fn wrong( + // CHECK-NOT: _0 = const 'b'; mir! { { let temp = arg; diff --git a/tests/mir-opt/dest-prop/nrvo_miscompile_111005.wrong.DestinationPropagation.diff b/tests/mir-opt/dest-prop/nrvo_miscompile_111005.wrong.DestinationPropagation.diff index cd8cb13c0ff96..60cf9236f6c2a 100644 --- a/tests/mir-opt/dest-prop/nrvo_miscompile_111005.wrong.DestinationPropagation.diff +++ b/tests/mir-opt/dest-prop/nrvo_miscompile_111005.wrong.DestinationPropagation.diff @@ -9,9 +9,9 @@ - _2 = copy _1; - _0 = copy _2; - _2 = const 'b'; -+ _0 = copy _1; + nop; -+ _0 = const 'b'; ++ _0 = copy _1; ++ _1 = const 'b'; return; } } From 4997476c3774cdb9261867a48be08a4154ce157f Mon Sep 17 00:00:00 2001 From: Camille Gillot Date: Tue, 19 Aug 2025 23:27:06 +0000 Subject: [PATCH 6/9] Simplify VisitPlacesWith. --- compiler/rustc_mir_transform/src/dest_prop.rs | 25 ++++++------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index c4875e6adb665..a8ff6cc6880d8 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -141,7 +141,7 @@ use rustc_data_structures::fx::FxIndexMap; use rustc_index::bit_set::DenseBitSet; use rustc_index::interval::SparseIntervalMatrix; use rustc_index::{IndexVec, newtype_index}; -use rustc_middle::mir::visit::{MutVisitor, NonMutatingUseContext, PlaceContext, Visitor}; +use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; use rustc_mir_dataflow::impls::{DefUse, MaybeLiveLocals}; @@ -543,24 +543,13 @@ impl<'tcx, F> Visitor<'tcx> for VisitPlacesWith where F: FnMut(Place<'tcx>, PlaceContext), { - fn visit_place(&mut self, place: &Place<'tcx>, ctxt: PlaceContext, _: Location) { + fn visit_local(&mut self, local: Local, ctxt: PlaceContext, _: Location) { + (self.0)(local.into(), ctxt); + } + + fn visit_place(&mut self, place: &Place<'tcx>, ctxt: PlaceContext, location: Location) { (self.0)(*place, ctxt); - for proj in place.projection.iter() { - match proj { - ProjectionElem::Index(index) => (self.0)( - index.into(), - PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy), - ), - ProjectionElem::Deref - | ProjectionElem::Field(..) - | ProjectionElem::Downcast(..) - | ProjectionElem::ConstantIndex { .. } - | ProjectionElem::Subslice { .. } - | ProjectionElem::OpaqueCast(..) - | ProjectionElem::UnwrapUnsafeBinder(..) - | ProjectionElem::Subtype(..) => {} - }; - } + self.visit_projection(place.as_ref(), ctxt, location); } } From 808ce08d75b7e2e53aaea471dbd0350ccfb1ddb3 Mon Sep 17 00:00:00 2001 From: Camille Gillot Date: Fri, 22 Aug 2025 20:51:46 +0000 Subject: [PATCH 7/9] Do not use prepend to avoid quadratic behaviour. --- compiler/rustc_index/src/interval.rs | 44 ++++------------ compiler/rustc_mir_transform/src/dest_prop.rs | 51 ++++++++++--------- 2 files changed, 37 insertions(+), 58 deletions(-) diff --git a/compiler/rustc_index/src/interval.rs b/compiler/rustc_index/src/interval.rs index e81e7e2bc442a..dda5253e7c547 100644 --- a/compiler/rustc_index/src/interval.rs +++ b/compiler/rustc_index/src/interval.rs @@ -140,42 +140,20 @@ impl IntervalSet { result } - /// Specialized version of `insert` when we know that the inserted point is *before* any - /// contained. - pub fn prepend(&mut self, point: I) { - let point = point.index() as u32; - - if let Some((first_start, _)) = self.map.first_mut() { - assert!(point <= *first_start); - if point == *first_start { - // The point is already present in the set. - } else if point + 1 == *first_start { - // Just extend the first range. - *first_start = point; - } else { - self.map.insert(0, (point, point)); - } - } else { - // If the map is empty, push is faster than insert. - self.map.push((point, point)); - } - - debug_assert!( - self.check_invariants(), - "wrong intervals after prepend {point:?} to {self:?}" - ); - } - /// Specialized version of `insert` when we know that the inserted point is *after* any /// contained. pub fn append(&mut self, point: I) { let point = point.index() as u32; - if let Some((_, last_end)) = self.map.last_mut() - && let _ = assert!(*last_end < point) - && point == *last_end + 1 - { - *last_end = point; + if let Some((_, last_end)) = self.map.last_mut() { + assert!(*last_end <= point); + if point == *last_end { + // The point is already in the set. + } else if point == *last_end + 1 { + *last_end = point; + } else { + self.map.push((point, point)); + } } else { self.map.push((point, point)); } @@ -397,10 +375,6 @@ impl SparseIntervalMatrix { self.ensure_row(row).insert(point) } - pub fn prepend(&mut self, row: R, point: C) { - self.ensure_row(row).prepend(point) - } - pub fn append(&mut self, row: R, point: C) { self.ensure_row(row).append(point) } diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index a8ff6cc6880d8..c6204c231c261 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -145,7 +145,7 @@ use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; use rustc_mir_dataflow::impls::{DefUse, MaybeLiveLocals}; -use rustc_mir_dataflow::points::{DenseLocationMap, PointIndex}; +use rustc_mir_dataflow::points::DenseLocationMap; use rustc_mir_dataflow::{Analysis, Results}; use tracing::{debug, trace}; @@ -494,9 +494,7 @@ fn dest_prop_mir_dump<'tcx>( live: &SparseIntervalMatrix, relevant: &RelevantLocals, ) { - let locals_live_at = |location, effect| { - let location = points.point_from_location(location); - let location = TwoStepIndex::new(location, effect); + let locals_live_at = |location| { live.rows() .filter(|&r| live.contains(r, location)) .map(|rl| relevant.original[rl]) @@ -504,10 +502,14 @@ fn dest_prop_mir_dump<'tcx>( }; dump_mir(tcx, false, "DestinationPropagation-dataflow", &0, body, |pass_where, w| { if let PassWhere::BeforeLocation(loc) = pass_where { - writeln!(w, " // before: {:?}", locals_live_at(loc, Effect::Before))?; + let location = TwoStepIndex::new(points, loc, Effect::Before); + let live = locals_live_at(location); + writeln!(w, " // before: {:?} => {:?}", location, live)?; } if let PassWhere::AfterLocation(loc) = pass_where { - writeln!(w, " // after: {:?}", locals_live_at(loc, Effect::After))?; + let location = TwoStepIndex::new(points, loc, Effect::After); + let live = locals_live_at(location); + writeln!(w, " // after: {:?} => {:?}", location, live)?; } Ok(()) @@ -521,19 +523,25 @@ enum Effect { } rustc_index::newtype_index! { - /// A `PointIndex` but with the lower bit encoding early/late inside the statement. + /// A reversed `PointIndex` but with the lower bit encoding early/late inside the statement. + /// The reversed order allows to use the more efficient `IntervalSet::append` method while we + /// iterate on the statements in reverse order. #[orderable] #[debug_format = "TwoStepIndex({})"] struct TwoStepIndex {} } impl TwoStepIndex { - fn new(point: PointIndex, effect: Effect) -> TwoStepIndex { + fn new(elements: &DenseLocationMap, location: Location, effect: Effect) -> TwoStepIndex { + let point = elements.point_from_location(location); let effect = match effect { Effect::Before => 0, Effect::After => 1, }; - TwoStepIndex::from_u32(2 * point.as_u32() + (effect as u32)) + let max_index = 2 * elements.num_points() as u32 - 1; + let index = 2 * point.as_u32() + (effect as u32); + // Reverse the indexing to use more efficient `IntervalSet::append`. + TwoStepIndex::from_u32(max_index - index) } } @@ -564,21 +572,18 @@ fn save_as_intervals<'tcx>( let mut state = MaybeLiveLocals.bottom_value(body); let reachable_blocks = traversal::reachable_as_bitset(body); - let two_step_loc = |location, effect| { - let point = elements.point_from_location(location); - TwoStepIndex::new(point, effect) - }; - let prepend_at = + let two_step_loc = |location, effect| TwoStepIndex::new(elements, location, effect); + let append_at = |values: &mut SparseIntervalMatrix<_, _>, state: &DenseBitSet, twostep| { for (relevant, &original) in relevant.original.iter_enumerated() { if state.contains(original) { - values.prepend(relevant, twostep); + values.append(relevant, twostep); } } }; // Iterate blocks in decreasing order, to visit locations in decreasing order. This - // allows to use the more efficient `prepend` method to interval sets. + // allows to use the more efficient `append` method to interval sets. for block in body.basic_blocks.indices().rev() { if !reachable_blocks.contains(block) { continue; @@ -591,7 +596,7 @@ fn save_as_intervals<'tcx>( let term = block_data.terminator(); let mut twostep = two_step_loc(loc, Effect::After); - prepend_at(&mut values, &state, twostep); + append_at(&mut values, &state, twostep); // Ensure we have a non-zero live range even for dead stores. This is done by marking all // the written-to locals as live in the second half of the statement. // We also ensure that operands read by terminators conflict with writes by that terminator. @@ -606,17 +611,17 @@ fn save_as_intervals<'tcx>( }) .visit_terminator(term, loc); - twostep = TwoStepIndex::from_u32(twostep.as_u32() - 1); + twostep = TwoStepIndex::from_u32(twostep.as_u32() + 1); debug_assert_eq!(twostep, two_step_loc(loc, Effect::Before)); MaybeLiveLocals.apply_early_terminator_effect(&mut state, term, loc); MaybeLiveLocals.apply_primary_terminator_effect(&mut state, term, loc); - prepend_at(&mut values, &state, twostep); + append_at(&mut values, &state, twostep); for (statement_index, stmt) in block_data.statements.iter().enumerate().rev() { let loc = Location { block, statement_index }; - twostep = TwoStepIndex::from_u32(twostep.as_u32() - 1); + twostep = TwoStepIndex::from_u32(twostep.as_u32() + 1); debug_assert_eq!(twostep, two_step_loc(loc, Effect::After)); - prepend_at(&mut values, &state, twostep); + append_at(&mut values, &state, twostep); // Ensure we have a non-zero live range even for dead stores. This is done by marking // all the written-to locals as live in the second half of the statement. VisitPlacesWith(|place, ctxt| match DefUse::for_place(place, ctxt) { @@ -629,13 +634,13 @@ fn save_as_intervals<'tcx>( }) .visit_statement(stmt, loc); - twostep = TwoStepIndex::from_u32(twostep.as_u32() - 1); + twostep = TwoStepIndex::from_u32(twostep.as_u32() + 1); debug_assert_eq!(twostep, two_step_loc(loc, Effect::Before)); MaybeLiveLocals.apply_early_statement_effect(&mut state, stmt, loc); MaybeLiveLocals.apply_primary_statement_effect(&mut state, stmt, loc); // ... but reads from operands are marked as live here so they do not conflict with // the all the writes we manually marked as live in the second half of the statement. - prepend_at(&mut values, &state, twostep); + append_at(&mut values, &state, twostep); } } From 33b86872af7ec40db252191be4c3d37e186ab405 Mon Sep 17 00:00:00 2001 From: Camille Gillot Date: Tue, 26 Aug 2025 23:22:59 +0000 Subject: [PATCH 8/9] Unify a source with all possible destinations. --- compiler/rustc_mir_transform/src/dest_prop.rs | 98 ++++++++----------- ...in.DestinationPropagation.panic-abort.diff | 21 ++-- ...n.DestinationPropagation.panic-unwind.diff | 21 ++-- ...stinationPropagation.after.panic-abort.mir | 6 +- ...tinationPropagation.after.panic-unwind.mir | 6 +- ...stinationPropagation.after.panic-abort.mir | 6 +- ...tinationPropagation.after.panic-unwind.mir | 6 +- ...ated_loop.PreCodegen.after.panic-abort.mir | 4 +- ...ted_loop.PreCodegen.after.panic-unwind.mir | 2 +- ...ward_loop.PreCodegen.after.panic-abort.mir | 4 +- ...ard_loop.PreCodegen.after.panic-unwind.mir | 4 +- ...ange_loop.PreCodegen.after.panic-abort.mir | 2 +- ...nge_loop.PreCodegen.after.panic-unwind.mir | 2 +- ...erse_loop.PreCodegen.after.panic-abort.mir | 2 +- ...rse_loop.PreCodegen.after.panic-unwind.mir | 2 +- 15 files changed, 84 insertions(+), 102 deletions(-) diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index c6204c231c261..1eccb43756372 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -182,29 +182,51 @@ impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation { for (src, candidates) in candidates.c.into_iter() { trace!(?src, ?candidates); - let Some(src) = relevant.find(src) else { continue }; - let Some(src_live_ranges) = &live.row(src) else { continue }; - trace!(?src, ?src_live_ranges); - - let dst = candidates.into_iter().find_map(|dst| { - let dst = relevant.find(dst)?; - let dst_live_ranges = &live.row(dst)?; + for dst in candidates { + // We call `relevant.find(src)` inside the loop, as a previous iteration may have + // renamed `src` to one of the locals in `dst`. + let Some(mut src) = relevant.find(src) else { continue }; + let Some(src_live_ranges) = live.row(src) else { continue }; + trace!(?src, ?src_live_ranges); + + let Some(mut dst) = relevant.find(dst) else { continue }; + let Some(dst_live_ranges) = live.row(dst) else { continue }; trace!(?dst, ?dst_live_ranges); - let disjoint = src_live_ranges.disjoint(dst_live_ranges); - disjoint.then_some(dst) - }); - let Some(dst) = dst else { continue }; + if src_live_ranges.disjoint(dst_live_ranges) { + // We want to replace `src` by `dst`. + let mut orig_src = relevant.original[src]; + let mut orig_dst = relevant.original[dst]; + + // The return place and function arguments are required and cannot be renamed. + // This check cannot be made during candidate collection, as we may want to + // unify the same non-required local with several required locals. + match (is_local_required(orig_src, body), is_local_required(orig_dst, body)) { + // Renaming `src` is ok. + (false, _) => {} + // Renaming `src` is wrong, but renaming `dst` is ok. + (true, false) => { + std::mem::swap(&mut src, &mut dst); + std::mem::swap(&mut orig_src, &mut orig_dst); + } + // Neither local can be renamed, so skip this case. + (true, true) => continue, + } - merged_locals.insert(relevant.original[src]); - merged_locals.insert(relevant.original[dst]); + trace!(?src, ?dst, "merge"); + merged_locals.insert(orig_src); + merged_locals.insert(orig_dst); - relevant.union(src, dst); - live.union_rows(src, dst); + // Replace `src` by `dst`. + relevant.union(src, dst); + live.union_rows(/* read */ src, /* write */ dst); + } + } } trace!(?merged_locals); relevant.make_idempotent(); + trace!(?relevant.renames); if merged_locals.is_empty() { return; @@ -394,41 +416,6 @@ impl Candidates { } } -/// If the pair of places is being considered for merging, returns the candidate which would be -/// merged in order to accomplish this. -/// -/// The contract here is in one direction - there is a guarantee that merging the locals that are -/// outputted by this function would result in an assignment between the inputs becoming a -/// self-assignment. However, there is no guarantee that the returned pair is actually suitable for -/// merging - candidate collection must still check this independently. -/// -/// This output is unique for each unordered pair of input places. -fn places_to_candidate_pair<'tcx>( - a: Place<'tcx>, - b: Place<'tcx>, - body: &Body<'tcx>, -) -> Option<(Local, Local)> { - let (mut a, mut b) = if a.projection.len() == 0 && b.projection.len() == 0 { - (a.local, b.local) - } else { - return None; - }; - - // By sorting, we make sure we're input order independent - if a > b { - std::mem::swap(&mut a, &mut b); - } - - // We could now return `(a, b)`, but then we miss some candidates in the case where `a` can't be - // used as a `src`. - if is_local_required(a, body) { - std::mem::swap(&mut a, &mut b); - } - // We could check `is_local_required` again here, but there's no need - after all, we make no - // promise that the candidate pair is actually valid - Some((a, b)) -} - struct FindAssignments<'a, 'tcx> { body: &'a Body<'tcx>, candidates: FxIndexMap>, @@ -441,11 +428,9 @@ impl<'tcx> Visitor<'tcx> for FindAssignments<'_, 'tcx> { lhs, Rvalue::CopyForDeref(rhs) | Rvalue::Use(Operand::Copy(rhs) | Operand::Move(rhs)), )) = &statement.kind + && let Some(src) = lhs.as_local() + && let Some(dest) = rhs.as_local() { - let Some((src, dest)) = places_to_candidate_pair(*lhs, *rhs, self.body) else { - return; - }; - // As described at the top of the file, we do not go near things that have // their address taken. if self.borrowed.contains(src) || self.borrowed.contains(dest) { @@ -462,11 +447,6 @@ impl<'tcx> Visitor<'tcx> for FindAssignments<'_, 'tcx> { return; } - // Also, we need to make sure that MIR actually allows the `src` to be removed - if is_local_required(src, self.body) { - return; - } - // We may insert duplicates here, but that's fine self.candidates.entry(src).or_default().push(dest); } diff --git a/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-abort.diff b/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-abort.diff index f9b8881ae3b15..a2f10be31a9e8 100644 --- a/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-abort.diff +++ b/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-abort.diff @@ -8,25 +8,23 @@ let _5: (); let mut _6: i32; scope 1 { -- debug x => _1; -+ debug x => _4; + debug x => _1; let _2: i32; scope 2 { - debug y => _2; -+ debug y => _4; ++ debug y => _1; let _3: i32; scope 3 { - debug z => _3; -+ debug z => _4; ++ debug z => _1; } } } bb0: { - StorageLive(_1); -- _1 = val() -> [return: bb1, unwind unreachable]; + nop; -+ _4 = val() -> [return: bb1, unwind unreachable]; + _1 = val() -> [return: bb1, unwind unreachable]; } bb1: { @@ -47,14 +45,17 @@ + nop; + nop; StorageLive(_5); - StorageLive(_6); +- StorageLive(_6); - _6 = copy _1; -+ _6 = copy _4; - _5 = std::mem::drop::(move _6) -> [return: bb2, unwind unreachable]; +- _5 = std::mem::drop::(move _6) -> [return: bb2, unwind unreachable]; ++ nop; ++ nop; ++ _5 = std::mem::drop::(move _1) -> [return: bb2, unwind unreachable]; } bb2: { - StorageDead(_6); +- StorageDead(_6); ++ nop; StorageDead(_5); _0 = const (); - StorageDead(_3); diff --git a/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-unwind.diff b/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-unwind.diff index 013e40ff462de..a08488615b15a 100644 --- a/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-unwind.diff +++ b/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-unwind.diff @@ -8,25 +8,23 @@ let _5: (); let mut _6: i32; scope 1 { -- debug x => _1; -+ debug x => _4; + debug x => _1; let _2: i32; scope 2 { - debug y => _2; -+ debug y => _4; ++ debug y => _1; let _3: i32; scope 3 { - debug z => _3; -+ debug z => _4; ++ debug z => _1; } } } bb0: { - StorageLive(_1); -- _1 = val() -> [return: bb1, unwind continue]; + nop; -+ _4 = val() -> [return: bb1, unwind continue]; + _1 = val() -> [return: bb1, unwind continue]; } bb1: { @@ -47,14 +45,17 @@ + nop; + nop; StorageLive(_5); - StorageLive(_6); +- StorageLive(_6); - _6 = copy _1; -+ _6 = copy _4; - _5 = std::mem::drop::(move _6) -> [return: bb2, unwind continue]; +- _5 = std::mem::drop::(move _6) -> [return: bb2, unwind continue]; ++ nop; ++ nop; ++ _5 = std::mem::drop::(move _1) -> [return: bb2, unwind continue]; } bb2: { - StorageDead(_6); +- StorageDead(_6); ++ nop; StorageDead(_5); _0 = const (); - StorageDead(_3); diff --git a/tests/mir-opt/dest-prop/dead_stores_79191.f.DestinationPropagation.after.panic-abort.mir b/tests/mir-opt/dest-prop/dead_stores_79191.f.DestinationPropagation.after.panic-abort.mir index eb4209731c62f..15061da812063 100644 --- a/tests/mir-opt/dest-prop/dead_stores_79191.f.DestinationPropagation.after.panic-abort.mir +++ b/tests/mir-opt/dest-prop/dead_stores_79191.f.DestinationPropagation.after.panic-abort.mir @@ -7,16 +7,16 @@ fn f(_1: usize) -> usize { let mut _3: usize; let mut _4: usize; scope 1 { - debug b => _3; + debug b => _2; } bb0: { nop; - _3 = copy _1; + _2 = copy _1; _1 = const 5_usize; nop; nop; - _1 = move _3; + _1 = move _2; nop; nop; nop; diff --git a/tests/mir-opt/dest-prop/dead_stores_79191.f.DestinationPropagation.after.panic-unwind.mir b/tests/mir-opt/dest-prop/dead_stores_79191.f.DestinationPropagation.after.panic-unwind.mir index fe9a7376a5856..ddfe4dc5b3ebf 100644 --- a/tests/mir-opt/dest-prop/dead_stores_79191.f.DestinationPropagation.after.panic-unwind.mir +++ b/tests/mir-opt/dest-prop/dead_stores_79191.f.DestinationPropagation.after.panic-unwind.mir @@ -7,16 +7,16 @@ fn f(_1: usize) -> usize { let mut _3: usize; let mut _4: usize; scope 1 { - debug b => _3; + debug b => _2; } bb0: { nop; - _3 = copy _1; + _2 = copy _1; _1 = const 5_usize; nop; nop; - _1 = move _3; + _1 = move _2; nop; nop; nop; diff --git a/tests/mir-opt/dest-prop/dead_stores_better.f.DestinationPropagation.after.panic-abort.mir b/tests/mir-opt/dest-prop/dead_stores_better.f.DestinationPropagation.after.panic-abort.mir index eb4209731c62f..15061da812063 100644 --- a/tests/mir-opt/dest-prop/dead_stores_better.f.DestinationPropagation.after.panic-abort.mir +++ b/tests/mir-opt/dest-prop/dead_stores_better.f.DestinationPropagation.after.panic-abort.mir @@ -7,16 +7,16 @@ fn f(_1: usize) -> usize { let mut _3: usize; let mut _4: usize; scope 1 { - debug b => _3; + debug b => _2; } bb0: { nop; - _3 = copy _1; + _2 = copy _1; _1 = const 5_usize; nop; nop; - _1 = move _3; + _1 = move _2; nop; nop; nop; diff --git a/tests/mir-opt/dest-prop/dead_stores_better.f.DestinationPropagation.after.panic-unwind.mir b/tests/mir-opt/dest-prop/dead_stores_better.f.DestinationPropagation.after.panic-unwind.mir index fe9a7376a5856..ddfe4dc5b3ebf 100644 --- a/tests/mir-opt/dest-prop/dead_stores_better.f.DestinationPropagation.after.panic-unwind.mir +++ b/tests/mir-opt/dest-prop/dead_stores_better.f.DestinationPropagation.after.panic-unwind.mir @@ -7,16 +7,16 @@ fn f(_1: usize) -> usize { let mut _3: usize; let mut _4: usize; scope 1 { - debug b => _3; + debug b => _2; } bb0: { nop; - _3 = copy _1; + _2 = copy _1; _1 = const 5_usize; nop; nop; - _1 = move _3; + _1 = move _2; nop; nop; nop; diff --git a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir index e6a184640a8c0..104987b0fdda9 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir @@ -4,8 +4,6 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { debug slice => _1; debug f => _2; let mut _0: (); - let mut _6: std::ptr::NonNull; - let mut _9: *const T; let mut _10: usize; let mut _28: std::option::Option<(usize, &T)>; let mut _31: &impl Fn(usize, &T); @@ -47,6 +45,7 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } } scope 28 (inlined as Iterator>::next) { + let mut _6: std::ptr::NonNull; let _11: std::ptr::NonNull; let _13: std::ptr::NonNull; let mut _16: bool; @@ -103,6 +102,7 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { let mut _8: *mut T; scope 5 { scope 6 { + let _9: *const T; scope 7 { } scope 11 (inlined std::ptr::without_provenance::) { diff --git a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-unwind.mir index 49efa14bb2426..28b12cdf36755 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-unwind.mir @@ -27,10 +27,10 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { let _3: usize; let mut _7: *mut T; let mut _8: *mut T; - let mut _9: *const T; scope 5 { let _6: std::ptr::NonNull; scope 6 { + let _9: *const T; scope 7 { } scope 11 (inlined std::ptr::without_provenance::) { diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir index 540b7ac3d5520..4d0e3548e7d6b 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir @@ -4,8 +4,6 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { debug slice => _1; debug f => _2; let mut _0: (); - let mut _6: std::ptr::NonNull; - let mut _9: *const T; let mut _22: std::option::Option<&T>; let mut _24: &impl Fn(&T); let mut _25: (&T,); @@ -19,6 +17,7 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { debug x => _23; } scope 16 (inlined as Iterator>::next) { + let mut _6: std::ptr::NonNull; let _10: std::ptr::NonNull; let _12: std::ptr::NonNull; let mut _15: bool; @@ -74,6 +73,7 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { let mut _8: *mut T; scope 5 { scope 6 { + let _9: *const T; scope 7 { } scope 11 (inlined std::ptr::without_provenance::) { diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir index e45bb76f33e68..2b5d8c27d7109 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir @@ -4,8 +4,6 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { debug slice => _1; debug f => _2; let mut _0: (); - let mut _6: std::ptr::NonNull; - let mut _9: *const T; let mut _22: std::option::Option<&T>; let mut _24: &impl Fn(&T); let mut _25: (&T,); @@ -19,6 +17,7 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { debug x => _23; } scope 16 (inlined as Iterator>::next) { + let mut _6: std::ptr::NonNull; let _10: std::ptr::NonNull; let _12: std::ptr::NonNull; let mut _15: bool; @@ -74,6 +73,7 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { let mut _8: *mut T; scope 5 { scope 6 { + let _9: *const T; scope 7 { } scope 11 (inlined std::ptr::without_provenance::) { diff --git a/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-abort.mir index 12ba8c446d937..145375990710b 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-abort.mir @@ -5,7 +5,6 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { debug f => _2; let mut _0: (); let mut _3: usize; - let mut _4: usize; let mut _8: std::option::Option; let mut _10: bool; let mut _12: &impl Fn(usize, &T); @@ -24,6 +23,7 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } scope 5 (inlined iter::range::>::next) { scope 6 (inlined as iter::range::RangeIteratorImpl>::spec_next) { + let mut _4: usize; let mut _6: bool; let _7: usize; scope 7 { diff --git a/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-unwind.mir index 3d0d998e418a3..8e573ef488fce 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-unwind.mir @@ -5,7 +5,6 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { debug f => _2; let mut _0: (); let mut _3: usize; - let mut _4: usize; let mut _8: std::option::Option; let mut _10: bool; let mut _12: &impl Fn(usize, &T); @@ -24,6 +23,7 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } scope 5 (inlined iter::range::>::next) { scope 6 (inlined as iter::range::RangeIteratorImpl>::spec_next) { + let mut _4: usize; let mut _6: bool; let _7: usize; scope 7 { diff --git a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir index 6e9ae3e15a9f7..0adf268d766dd 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir @@ -27,10 +27,10 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { let _3: usize; let mut _7: *mut T; let mut _8: *mut T; - let mut _9: *const T; scope 5 { let _6: std::ptr::NonNull; scope 6 { + let _9: *const T; scope 7 { } scope 11 (inlined std::ptr::without_provenance::) { diff --git a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir index 128db91ffab49..cb0d640d445bb 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir @@ -27,10 +27,10 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { let _3: usize; let mut _7: *mut T; let mut _8: *mut T; - let mut _9: *const T; scope 5 { let _6: std::ptr::NonNull; scope 6 { + let _9: *const T; scope 7 { } scope 11 (inlined std::ptr::without_provenance::) { From 843e6f8f362e5a7bf6778ca70b6c58e5df10a982 Mon Sep 17 00:00:00 2001 From: Camille Gillot Date: Tue, 26 Aug 2025 23:38:45 +0000 Subject: [PATCH 9/9] Simplify candidate collection. --- compiler/rustc_mir_transform/src/dest_prop.rs | 98 ++++++++----------- 1 file changed, 43 insertions(+), 55 deletions(-) diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index 1eccb43756372..9d6688d6a7fc1 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -137,7 +137,6 @@ //! [attempt 3]: https://github.com/rust-lang/rust/pull/72632 //! [attempt 4]: https://github.com/rust-lang/rust/pull/96451 -use rustc_data_structures::fx::FxIndexMap; use rustc_index::bit_set::DenseBitSet; use rustc_index::interval::SparseIntervalMatrix; use rustc_index::{IndexVec, newtype_index}; @@ -179,48 +178,47 @@ impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation { let mut merged_locals = DenseBitSet::new_empty(body.local_decls.len()); - for (src, candidates) in candidates.c.into_iter() { - trace!(?src, ?candidates); - - for dst in candidates { - // We call `relevant.find(src)` inside the loop, as a previous iteration may have - // renamed `src` to one of the locals in `dst`. - let Some(mut src) = relevant.find(src) else { continue }; - let Some(src_live_ranges) = live.row(src) else { continue }; - trace!(?src, ?src_live_ranges); - - let Some(mut dst) = relevant.find(dst) else { continue }; - let Some(dst_live_ranges) = live.row(dst) else { continue }; - trace!(?dst, ?dst_live_ranges); - - if src_live_ranges.disjoint(dst_live_ranges) { - // We want to replace `src` by `dst`. - let mut orig_src = relevant.original[src]; - let mut orig_dst = relevant.original[dst]; - - // The return place and function arguments are required and cannot be renamed. - // This check cannot be made during candidate collection, as we may want to - // unify the same non-required local with several required locals. - match (is_local_required(orig_src, body), is_local_required(orig_dst, body)) { - // Renaming `src` is ok. - (false, _) => {} - // Renaming `src` is wrong, but renaming `dst` is ok. - (true, false) => { - std::mem::swap(&mut src, &mut dst); - std::mem::swap(&mut orig_src, &mut orig_dst); - } - // Neither local can be renamed, so skip this case. - (true, true) => continue, - } + for (src, dst) in candidates.c.into_iter() { + trace!(?src, ?dst); - trace!(?src, ?dst, "merge"); - merged_locals.insert(orig_src); - merged_locals.insert(orig_dst); + let Some(mut src) = relevant.find(src) else { continue }; + let Some(mut dst) = relevant.find(dst) else { continue }; + if src == dst { + continue; + } - // Replace `src` by `dst`. - relevant.union(src, dst); - live.union_rows(/* read */ src, /* write */ dst); + let Some(src_live_ranges) = live.row(src) else { continue }; + let Some(dst_live_ranges) = live.row(dst) else { continue }; + trace!(?src, ?src_live_ranges); + trace!(?dst, ?dst_live_ranges); + + if src_live_ranges.disjoint(dst_live_ranges) { + // We want to replace `src` by `dst`. + let mut orig_src = relevant.original[src]; + let mut orig_dst = relevant.original[dst]; + + // The return place and function arguments are required and cannot be renamed. + // This check cannot be made during candidate collection, as we may want to + // unify the same non-required local with several required locals. + match (is_local_required(orig_src, body), is_local_required(orig_dst, body)) { + // Renaming `src` is ok. + (false, _) => {} + // Renaming `src` is wrong, but renaming `dst` is ok. + (true, false) => { + std::mem::swap(&mut src, &mut dst); + std::mem::swap(&mut orig_src, &mut orig_dst); + } + // Neither local can be renamed, so skip this case. + (true, true) => continue, } + + trace!(?src, ?dst, "merge"); + merged_locals.insert(orig_src); + merged_locals.insert(orig_dst); + + // Replace `src` by `dst`. + relevant.union(src, dst); + live.union_rows(/* read */ src, /* write */ dst); } } trace!(?merged_locals); @@ -334,11 +332,9 @@ impl RelevantLocals { shrink.get_or_insert_with(local, || original.push(local)); }; - for (&src, destinations) in candidates.c.iter() { + for &(src, dest) in candidates.c.iter() { declare(src); - for &dest in destinations { - declare(dest) - } + declare(dest) } let renames = IndexVec::from_fn_n(|l| l, original.len()); @@ -380,8 +376,6 @@ impl RelevantLocals { struct Candidates { /// The set of candidates we are considering in this optimization. /// - /// We will always merge the key into at most one of its values. - /// /// Whether a place ends up in the key or the value does not correspond to whether it appears as /// the lhs or rhs of any assignment. As a matter of fact, the places in here might never appear /// in an assignment at all. This happens because if we see an assignment like this: @@ -392,7 +386,7 @@ struct Candidates { /// /// We will still report that we would like to merge `_1` and `_2` in an attempt to allow us to /// remove that assignment. - c: FxIndexMap>, + c: Vec<(Local, Local)>, } // We first implement some utility functions which we will expose removing candidates according to @@ -406,19 +400,13 @@ impl Candidates { let mut visitor = FindAssignments { body, candidates: Default::default(), borrowed }; visitor.visit_body(body); - // Deduplicate candidates. - for (_, cands) in visitor.candidates.iter_mut() { - cands.sort(); - cands.dedup(); - } - Candidates { c: visitor.candidates } } } struct FindAssignments<'a, 'tcx> { body: &'a Body<'tcx>, - candidates: FxIndexMap>, + candidates: Vec<(Local, Local)>, borrowed: &'a DenseBitSet, } @@ -448,7 +436,7 @@ impl<'tcx> Visitor<'tcx> for FindAssignments<'_, 'tcx> { } // We may insert duplicates here, but that's fine - self.candidates.entry(src).or_default().push(dest); + self.candidates.push((src, dest)); } } }