Auto merge of #113166 - moulins:ref-niches-initial, r=oli-obk

Prototype: Add unstable `-Z reference-niches` option

MCP: rust-lang/compiler-team#641
Relevant RFC: rust-lang/rfcs#3204

This prototype adds a new `-Z reference-niches` option, controlling the range of valid bit-patterns for reference types (`&T` and `&mut T`), thereby enabling new enum niching opportunities. Like `-Z randomize-layout`, this setting is crate-local; as such, references to built-in types (primitives, tuples, ...) are not affected.

The possible settings are (here, `MAX` denotes the all-1 bit-pattern):
| `-Z reference-niches=` | Valid range |
|:---:|:---:|
| `null` (the default) | `1..=MAX` |
| `size` | `1..=(MAX- size)` |
| `align` | `align..=MAX.align_down_to(align)` |
| `size,align` | `align..=(MAX-size).align_down_to(align)` |

------

This is very WIP, and I'm not sure the approach I've taken here is the best one, but stage 1 tests pass locally; I believe this is in a good enough state to unleash this upon unsuspecting 3rd-party code, and see what breaks.

Author: bors

src/intptrcast.rs

@@ -207,7 +207,7 @@ impl<'mir, 'tcx> GlobalStateInner {
                     .checked_add(max(size.bytes(), 1))
                     .ok_or_else(|| err_exhaust!(AddressSpaceFull))?;
                 // Even if `Size` didn't overflow, we might still have filled up the address space.
-                if global_state.next_base_addr > ecx.target_usize_max() {
+                if global_state.next_base_addr > ecx.data_layout().target_usize_max() {
                     throw_exhaust!(AddressSpaceFull);
                 }
                 // Given that `next_base_addr` increases in each allocation, pushing the

src/shims/mod.rs

@@ -21,6 +21,7 @@ use log::trace;
 
 use rustc_middle::{mir, ty};
 use rustc_target::spec::abi::Abi;
+use rustc_target::abi::HasDataLayout as _;
 
 use crate::*;
 use helpers::check_arg_count;
@@ -108,7 +109,8 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
         }
 
         // Return error result (usize::MAX), and jump to caller.
-        this.write_scalar(Scalar::from_target_usize(this.target_usize_max(), this), dest)?;
+        let usize_max = this.data_layout().target_usize_max();
+        this.write_scalar(Scalar::from_target_usize(usize_max, this), dest)?;
         this.go_to_block(ret);
         Ok(true)
     }

src/shims/unix/fs.rs

@@ -12,7 +12,7 @@ use log::trace;
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::ty::TyCtxt;
-use rustc_target::abi::{Align, Size};
+use rustc_target::abi::{Align, Size, HasDataLayout as _};
 
 use crate::shims::os_str::bytes_to_os_str;
 use crate::*;
@@ -753,7 +753,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
         // We cap the number of read bytes to the largest value that we are able to fit in both the
         // host's and target's `isize`. This saves us from having to handle overflows later.
         let count = count
-            .min(u64::try_from(this.target_isize_max()).unwrap())
+            .min(u64::try_from(this.data_layout().target_isize_max()).unwrap())
             .min(u64::try_from(isize::MAX).unwrap());
         let communicate = this.machine.communicate();
 
@@ -807,7 +807,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
         // We cap the number of written bytes to the largest value that we are able to fit in both the
         // host's and target's `isize`. This saves us from having to handle overflows later.
         let count = count
-            .min(u64::try_from(this.target_isize_max()).unwrap())
+            .min(u64::try_from(this.data_layout().target_isize_max()).unwrap())
             .min(u64::try_from(isize::MAX).unwrap());
         let communicate = this.machine.communicate();
 

tests/fail/layout_cycle.rs

@@ -1,5 +1,5 @@
 //@error-in-other-file: a cycle occurred during layout computation
-//~^ ERROR: cycle detected when computing layout of
+//~^ ERROR: cycle detected when computing layout (naive) of
 
 use std::mem;
 

tests/fail/layout_cycle.stderr

@@ -1,7 +1,8 @@
-error[E0391]: cycle detected when computing layout of `S<S<()>>`
+error[E0391]: cycle detected when computing layout (naive) of `S<S<()>>`
    |
-   = note: ...which requires computing layout of `<S<()> as Tr>::I`...
-   = note: ...which again requires computing layout of `S<S<()>>`, completing the cycle
+   = note: ...which requires computing layout (naive) of `<S<()> as Tr>::I`...
+   = note: ...which again requires computing layout (naive) of `S<S<()>>`, completing the cycle
+   = note: cycle used when computing layout of `S<S<()>>`
    = note: see https://rustc-dev-guide.rust-lang.org/overview.html#queries and https://rustc-dev-guide.rust-lang.org/query.html for more information
 
 error: post-monomorphization error: a cycle occurred during layout computation