WIP: Builtin/opaque dependencies

crates/cargo-util-schemas/src/core/source_kind.rs

@@ -15,6 +15,8 @@ pub enum SourceKind {
     LocalRegistry,
     /// A directory-based registry.
     Directory,
+    /// Package sources distributed with the rust toolchain
+    Builtin,
 }
 
 // The hash here is important for what folder packages get downloaded into.
@@ -40,6 +42,7 @@ impl SourceKind {
             SourceKind::SparseRegistry => None,
             SourceKind::LocalRegistry => Some("local-registry"),
             SourceKind::Directory => Some("directory"),
+            SourceKind::Builtin => Some("builtin"),
         }
     }
 }
@@ -71,6 +74,10 @@ impl Ord for SourceKind {
             (_, SourceKind::Directory) => Ordering::Greater,
 
             (SourceKind::Git(a), SourceKind::Git(b)) => a.cmp(b),
+            (SourceKind::Git(_), _) => Ordering::Less,
+            (_, SourceKind::Git(_)) => Ordering::Greater,
+
+            (SourceKind::Builtin, SourceKind::Builtin) => Ordering::Equal,
         }
     }
 }

src/cargo/core/compiler/standard_lib.rs

@@ -15,7 +15,11 @@ use std::path::PathBuf;
 
 use super::BuildConfig;
 
-fn std_crates<'a>(crates: &'a [String], default: &'static str, units: &[Unit]) -> HashSet<&'a str> {
+pub fn std_crates<'a>(
+    crates: &'a [String],
+    default: &'static str,
+    units: &[Unit],
+) -> HashSet<&'a str> {
     let mut crates = HashSet::from_iter(crates.iter().map(|s| s.as_str()));
     // This is a temporary hack until there is a more principled way to
     // declare dependencies in Cargo.toml.
@@ -99,6 +103,7 @@ pub fn resolve_std<'gctx>(
         HasDevUnits::No,
         crate::core::resolver::features::ForceAllTargets::No,
         dry_run,
+        false,
     )?;
     debug_assert_eq!(resolve.specs_and_features.len(), 1);
     Ok((
@@ -216,7 +221,7 @@ fn generate_roots(
     Ok(())
 }
 
-fn detect_sysroot_src_path(target_data: &RustcTargetData<'_>) -> CargoResult<PathBuf> {
+pub fn detect_sysroot_src_path(target_data: &RustcTargetData<'_>) -> CargoResult<PathBuf> {
     if let Some(s) = target_data.gctx.get_env_os("__CARGO_TESTS_ONLY_SRC_ROOT") {
         return Ok(s.into());
     }

src/cargo/core/compiler/unit_dependencies.rs

@@ -52,6 +52,8 @@ struct State<'a, 'gctx> {
     std_resolve: Option<&'a Resolve>,
     /// Like `usr_features` but for building standard library (`-Zbuild-std`).
     std_features: Option<&'a ResolvedFeatures>,
+    // The root units of any opaque dependencies present in the user resolve
+    opaque_roots: &'a HashMap<CompileKind, Vec<Unit>>,
     /// `true` while generating the dependencies for the standard library.
     is_std: bool,
     /// The high-level operation requested by the user.
@@ -92,7 +94,7 @@ pub fn build_unit_dependencies<'a, 'gctx>(
     resolve: &'a Resolve,
     features: &'a ResolvedFeatures,
     std_resolve: Option<&'a (Resolve, ResolvedFeatures)>,
-    roots: &[Unit],
+    roots: &[Unit], //TODO: builtins can be roots if requested on the command line
     scrape_units: &[Unit],
     std_roots: &HashMap<CompileKind, Vec<Unit>>,
     intent: UserIntent,
@@ -119,6 +121,7 @@ pub fn build_unit_dependencies<'a, 'gctx>(
         usr_features: features,
         std_resolve,
         std_features,
+        opaque_roots: std_roots,
         is_std: false,
         intent,
         target_data,
@@ -129,15 +132,14 @@ pub fn build_unit_dependencies<'a, 'gctx>(
     };
 
     let std_unit_deps = calc_deps_of_std(&mut state, std_roots)?;
+    if let Some(std_unit_deps) = std_unit_deps {
+        attach_std_deps(&mut state, std_unit_deps);
+    }
 
     deps_of_roots(roots, &mut state)?;
     super::links::validate_links(state.resolve(), &state.unit_dependencies)?;
     // Hopefully there aren't any links conflicts with the standard library?
 
-    if let Some(std_unit_deps) = std_unit_deps {
-        attach_std_deps(&mut state, std_roots, std_unit_deps);
-    }
-
     connect_run_custom_build_deps(&mut state);
 
     // Dependencies are used in tons of places throughout the backend, many of
@@ -188,36 +190,14 @@ fn calc_deps_of_std(
     Ok(Some(std::mem::take(&mut state.unit_dependencies)))
 }
 
-/// Add the standard library units to the `unit_dependencies`.
-fn attach_std_deps(
-    state: &mut State<'_, '_>,
-    std_roots: &HashMap<CompileKind, Vec<Unit>>,
-    std_unit_deps: UnitGraph,
-) {
-    // Attach the standard library as a dependency of every target unit.
-    let mut found = false;
-    for (unit, deps) in state.unit_dependencies.iter_mut() {
-        if !unit.kind.is_host() && !unit.mode.is_run_custom_build() {
-            deps.extend(std_roots[&unit.kind].iter().map(|unit| UnitDep {
-                unit: unit.clone(),
-                unit_for: UnitFor::new_normal(unit.kind),
-                extern_crate_name: unit.pkg.name(),
-                dep_name: None,
-                // TODO: Does this `public` make sense?
-                public: true,
-                noprelude: true,
-                nounused: true,
-            }));
-            found = true;
+/// Add the dependencies of standard library units to the `unit_dependencies`.
+fn attach_std_deps(state: &mut State<'_, '_>, std_unit_deps: UnitGraph) {
+    for (unit, deps) in std_unit_deps.into_iter() {
+        if unit.pkg.package_id().name() == "sysroot" {
+            continue;
         }
-    }
-    // And also include the dependencies of the standard library itself. Don't
-    // include these if no units actually needed the standard library.
-    if found {
-        for (unit, deps) in std_unit_deps.into_iter() {
-            if let Some(other_unit) = state.unit_dependencies.insert(unit, deps) {
-                panic!("std unit collision with existing unit: {:?}", other_unit);
-            }
+        if let Some(other_unit) = state.unit_dependencies.insert(unit, deps) {
+            panic!("std unit collision with existing unit: {:?}", other_unit);
         }
     }
 }
@@ -334,16 +314,38 @@ fn compute_deps(
             )?;
             ret.push(unit_dep);
         } else {
-            let unit_dep = new_unit_dep(
-                state,
-                unit,
-                dep_pkg,
-                dep_lib,
-                dep_unit_for,
-                unit.kind.for_target(dep_lib),
-                mode,
-                IS_NO_ARTIFACT_DEP,
-            )?;
+            // if builtin, return from state.opaque_roots
+            let unit_dep = if dep_pkg_id.source_id().is_builtin() {
+                let unit: Vec<_> = state.opaque_roots[&unit.kind.for_target(dep_lib)]
+                    .iter()
+                    .filter(|&u| u.pkg.name() == dep_pkg_id.name())
+                    .collect();
+                assert!(
+                    unit.len() == 1,
+                    "libstd was resolved with all possible builtin deps as roots"
+                );
+                let unit = unit[0];
+                UnitDep {
+                    unit: unit.clone(),
+                    unit_for: UnitFor::new_normal(unit.kind),
+                    extern_crate_name: unit.pkg.name(),
+                    dep_name: None,
+                    public: true,
+                    noprelude: true,
+                    nounused: true,
+                }
+            } else {
+                new_unit_dep(
+                    state,
+                    unit,
+                    dep_pkg,
+                    dep_lib,
+                    dep_unit_for,
+                    unit.kind.for_target(dep_lib),
+                    mode,
+                    IS_NO_ARTIFACT_DEP,
+                )?
+            };
             ret.push(unit_dep);
         }
 
@@ -359,6 +361,28 @@ fn compute_deps(
     }
     state.dev_dependency_edges.extend(dev_deps);
 
+    if unit.mode.is_rustc_test() && unit.target.harness() {
+        let unit: Vec<_> = state.opaque_roots[&unit.kind]
+            .iter()
+            .filter(|&u| u.pkg.name() == "test")
+            .collect();
+        assert!(
+            unit.len() == 1,
+            "libstd was resolved with test crate as root"
+        );
+        let unit = unit[0];
+        let unitdep = UnitDep {
+            unit: unit.clone(),
+            unit_for: UnitFor::new_normal(unit.kind),
+            extern_crate_name: unit.pkg.name(),
+            dep_name: None,
+            public: true,
+            noprelude: true,
+            nounused: true,
+        };
+        ret.push(unitdep);
+    }
+
     // If this target is a build script, then what we've collected so far is
     // all we need. If this isn't a build script, then it depends on the
     // build script if there is one.

src/cargo/core/dependency.rs

@@ -51,6 +51,10 @@ struct Inner {
     // This dependency should be used only for this platform.
     // `None` means *all platforms*.
     platform: Option<Platform>,
+
+    // Opaque dependencies should be resolved with a separate resolver run, and handled
+    // by unit generation.
+    opaque: bool,
 }
 
 #[derive(Serialize)]
@@ -162,6 +166,30 @@ impl Dependency {
                 platform: None,
                 explicit_name_in_toml: None,
                 artifact: None,
+                opaque: false,
+            }),
+        }
+    }
+
+    pub fn new_injected_builtin(name: InternedString, root: &PathBuf) -> Dependency {
+        assert!(!name.is_empty());
+        Dependency {
+            inner: Arc::new(Inner {
+                name,
+                source_id: SourceId::new_builtin(&name, root).expect("package name is valid url"),
+                registry_id: None,
+                req: OptVersionReq::Any,
+                kind: DepKind::Normal,
+                only_match_name: true,
+                optional: false,
+                public: true,
+                features: Vec::new(),
+                default_features: true,
+                specified_req: false,
+                platform: None,
+                explicit_name_in_toml: None,
+                artifact: None,
+                opaque: true,
             }),
         }
     }
@@ -455,6 +483,10 @@ impl Dependency {
     pub(crate) fn maybe_lib(&self) -> bool {
         self.artifact().map(|a| a.is_lib).unwrap_or(true)
     }
+
+    pub fn is_opaque(&self) -> bool {
+        self.inner.opaque
+    }
 }
 
 /// The presence of an artifact turns an ordinary dependency into an Artifact dependency.

src/cargo/core/registry.rs

@@ -9,7 +9,7 @@
 //! The former is just one kind of source,
 //! while the latter involves operations on the registry Web API.
 
-use std::collections::{HashMap, HashSet};
+use std::collections::{BTreeMap, HashMap, HashSet};
 use std::task::{Poll, ready};
 
 use crate::core::{Dependency, PackageId, PackageSet, Patch, SourceId, Summary};
@@ -24,6 +24,7 @@ use crate::util::{CanonicalUrl, GlobalContext};
 use annotate_snippets::Level;
 use anyhow::Context as _;
 use itertools::Itertools;
+use semver::Version;
 use tracing::{debug, trace};
 use url::Url;
 
@@ -724,6 +725,32 @@ impl<'gctx> Registry for PackageRegistry<'gctx> {
                 )
             })?;
 
+        if dep.is_opaque() {
+            // Currently, all opaque dependencies are builtins.
+            // Create a dummy Summary that can be replaced with a real package during
+            // unit generation
+            trace!(
+                "Injecting package to satisfy builtin dependency on {}",
+                dep.package_name()
+            );
+            let ver = if dep.package_name() == "compiler_builtins" {
+                //TODO: hack
+                Version::new(0, 1, 160)
+            } else {
+                Version::new(0, 0, 0)
+            };
+            let pkg_id = PackageId::new(dep.package_name(), ver, dep.source_id());
+
+            let summary = Summary::new(
+                pkg_id,
+                vec![],
+                &BTreeMap::new(),
+                Option::<String>::None,
+                None,
+            )?;
+            f(IndexSummary::Candidate(summary));
+            return Poll::Ready(Ok(()));
+        }
         let source = self.sources.get_mut(dep.source_id());
         match (override_summary, source) {
             (Some(_), None) => {

src/cargo/core/resolver/context.rs

@@ -25,6 +25,10 @@ pub struct ResolverContext {
     /// a way to look up for a package in activations what packages required it
     /// and all of the exact deps that it fulfilled.
     pub parents: Graph<PackageId, im_rc::HashSet<Dependency, rustc_hash::FxBuildHasher>>,
+    // Opaque dependencies require a separate resolver run as they allow for multiple
+    // different semver-compatible versions of crates in the final resolve. This is the
+    // (unactivated) set of Summaries that need handling in a future invocation
+    //pub promises: HashSet<Dependency>,
 }
 
 /// When backtracking it can be useful to know how far back to go.