Auto merge of #14137 - Xaeroxe:checksum-freshness, r=<try>

initial version of checksum based freshness

Implementation for #14136 and resolves #6529

This PR implements the use of checksums in cargo fingerprints as an alternative to using mtimes. This is most useful on systems with poor mtime implementations.

This has a dependency on rust-lang/rust#126930. It's expected this will increase the time it takes to declare a build to be fresh. Still this loss in performance may be preferable to the issues the ecosystem has had with the use of mtimes for determining freshness.

Author: bors

Cargo.lock

@@ -23,6 +23,7 @@ anstream = "0.6.15"
 anstyle = "1.0.8"
 anyhow = "1.0.86"
 base64 = "0.22.1"
+blake3 = "1.5.2"
 bytesize = "1.3"
 cargo = { path = "" }
 cargo-credential = { version = "0.4.2", path = "credential/cargo-credential" }
@@ -148,6 +149,7 @@ anstream.workspace = true
 anstyle.workspace = true
 anyhow.workspace = true
 base64.workspace = true
+blake3.workspace = true
 bytesize.workspace = true
 cargo-credential.workspace = true
 cargo-platform.workspace = true
@@ -197,6 +199,7 @@ shell-escape.workspace = true
 supports-hyperlinks.workspace = true
 tar.workspace = true
 tempfile.workspace = true
+thiserror.workspace = true
 time.workspace = true
 toml.workspace = true
 toml_edit.workspace = true

Cargo.toml

@@ -94,6 +94,7 @@ allow = [
     "MIT",
     "MIT-0",
     "Apache-2.0",
+    "BSD-2-Clause",
     "BSD-3-Clause",
     "MPL-2.0",
     "Unicode-DFS-2016",

deny.toml

@@ -16,7 +16,7 @@ use jobserver::Client;
 
 use super::build_plan::BuildPlan;
 use super::custom_build::{self, BuildDeps, BuildScriptOutputs, BuildScripts};
-use super::fingerprint::Fingerprint;
+use super::fingerprint::{Checksum, Fingerprint};
 use super::job_queue::JobQueue;
 use super::layout::Layout;
 use super::lto::Lto;
@@ -50,6 +50,8 @@ pub struct BuildRunner<'a, 'gctx> {
     pub fingerprints: HashMap<Unit, Arc<Fingerprint>>,
     /// Cache of file mtimes to reduce filesystem hits.
     pub mtime_cache: HashMap<PathBuf, FileTime>,
+    /// Cache of file checksums to reduce filesystem reads.
+    pub checksum_cache: HashMap<PathBuf, Checksum>,
     /// A set used to track which units have been compiled.
     /// A unit may appear in the job graph multiple times as a dependency of
     /// multiple packages, but it only needs to run once.
@@ -113,6 +115,7 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
             build_script_outputs: Arc::new(Mutex::new(BuildScriptOutputs::default())),
             fingerprints: HashMap::new(),
             mtime_cache: HashMap::new(),
+            checksum_cache: HashMap::new(),
             compiled: HashSet::new(),
             build_scripts: HashMap::new(),
             build_explicit_deps: HashMap::new(),

src/cargo/core/compiler/build_runner/mod.rs

@@ -34,6 +34,9 @@ pub enum DirtyReason {
         old: String,
         new: String,
     },
+    ChecksumUseChanged {
+        old: bool,
+    },
     DepInfoOutputChanged {
         old: PathBuf,
         new: PathBuf,
@@ -183,6 +186,16 @@ impl DirtyReason {
             DirtyReason::PrecalculatedComponentsChanged { .. } => {
                 s.dirty_because(unit, "the precalculated components changed")
             }
+            DirtyReason::ChecksumUseChanged { old } => {
+                if *old {
+                    s.dirty_because(
+                        unit,
+                        "the prior compilation used checksum freshness and this one does not",
+                    )
+                } else {
+                    s.dirty_because(unit, "checksum freshness requested, prior compilation did not use checksum freshness")
+                }
+            }
             DirtyReason::DepInfoOutputChanged { .. } => {
                 s.dirty_because(unit, "the dependency info output changed")
             }
@@ -222,6 +235,20 @@ impl DirtyReason {
                             format_args!("the file `{}` is missing", file.display()),
                         )
                     }
+                    StaleItem::UnableToReadFile(file) => {
+                        let file = file.strip_prefix(root).unwrap_or(&file);
+                        s.dirty_because(
+                            unit,
+                            format_args!("the file `{}` could not be read", file.display()),
+                        )
+                    }
+                    StaleItem::FailedToReadMetadata(file) => {
+                        let file = file.strip_prefix(root).unwrap_or(&file);
+                        s.dirty_because(
+                            unit,
+                            format_args!("couldn't read metadata for file `{}`", file.display()),
+                        )
+                    }
                     StaleItem::ChangedFile {
                         stale,
                         stale_mtime,
@@ -235,6 +262,41 @@ impl DirtyReason {
                             format_args!("the file `{}` has changed ({after})", file.display()),
                         )
                     }
+                    StaleItem::ChangedChecksum {
+                        source,
+                        stored_checksum,
+                        new_checksum,
+                    } => {
+                        let file = source.strip_prefix(root).unwrap_or(&source);
+                        s.dirty_because(
+                            unit,
+                            format_args!(
+                                "the file `{}` has changed (checksum didn't match, {stored_checksum} != {new_checksum})",
+                                file.display(),
+                            ),
+                        )
+                    }
+                    StaleItem::FileSizeChanged {
+                        path,
+                        old_size,
+                        new_size,
+                    } => {
+                        let file = path.strip_prefix(root).unwrap_or(&path);
+                        s.dirty_because(
+                            unit,
+                            format_args!(
+                                "file size changed ({old_size} != {new_size}) for `{}`",
+                                file.display()
+                            ),
+                        )
+                    }
+                    StaleItem::MissingChecksum(path) => {
+                        let file = path.strip_prefix(root).unwrap_or(&path);
+                        s.dirty_because(
+                            unit,
+                            format_args!("the checksum for file `{}` is missing", file.display()),
+                        )
+                    }
                     StaleItem::ChangedEnv { var, .. } => s.dirty_because(
                         unit,
                         format_args!("the environment variable {var} changed"),