Merge pull request rust-lang#1225 from bjorn3/build_system_rework

bjorn3 · web-flow · commit 9152ded3bfd1 · 2022-04-30T13:34:40.000+02:00
Use -Zcodegen-backend instead of a custom rustc driver
diff --git a/build_system/build_backend.rs b/build_system/build_backend.rs
@@ -34,18 +34,6 @@ pub(crate) fn build_backend(
         _ => unreachable!(),
     }
 
-    // Set the rpath to make the cg_clif executable find librustc_codegen_cranelift without changing
-    // LD_LIBRARY_PATH
-    if cfg!(unix) {
-        if cfg!(target_os = "macos") {
-            rustflags += " -Csplit-debuginfo=unpacked \
-                -Clink-arg=-Wl,-rpath,@loader_path/../lib \
-                -Zosx-rpath-install-name";
-        } else {
-            rustflags += " -Clink-arg=-Wl,-rpath=$ORIGIN/../lib ";
-        }
-    }
-
     cmd.env("RUSTFLAGS", rustflags);
 
     eprintln!("[BUILD] rustc_codegen_cranelift");
diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs
@@ -1,4 +1,3 @@
-use std::env;
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::process::{self, Command};
@@ -22,35 +21,28 @@ pub(crate) fn build_sysroot(
     fs::create_dir_all(target_dir.join("lib")).unwrap();
 
     // Copy the backend
-    for file in ["cg_clif", "cg_clif_build_sysroot"] {
-        try_hard_link(
-            cg_clif_build_dir.join(get_file_name(file, "bin")),
-            target_dir.join("bin").join(get_file_name(file, "bin")),
-        );
-    }
-
     let cg_clif_dylib = get_file_name("rustc_codegen_cranelift", "dylib");
-    try_hard_link(
-        cg_clif_build_dir.join(&cg_clif_dylib),
-        target_dir
-            .join(if cfg!(windows) {
-                // Windows doesn't have rpath support, so the cg_clif dylib needs to be next to the
-                // binaries.
-                "bin"
-            } else {
-                "lib"
-            })
-            .join(cg_clif_dylib),
-    );
-
-    // Build and copy cargo wrapper
-    let mut build_cargo_wrapper_cmd = Command::new("rustc");
-    build_cargo_wrapper_cmd
-        .arg("scripts/cargo-clif.rs")
-        .arg("-o")
-        .arg(target_dir.join("cargo-clif"))
-        .arg("-g");
-    spawn_and_wait(build_cargo_wrapper_cmd);
+    let cg_clif_dylib_path = target_dir
+        .join(if cfg!(windows) {
+            // Windows doesn't have rpath support, so the cg_clif dylib needs to be next to the
+            // binaries.
+            "bin"
+        } else {
+            "lib"
+        })
+        .join(&cg_clif_dylib);
+    try_hard_link(cg_clif_build_dir.join(cg_clif_dylib), &cg_clif_dylib_path);
+
+    // Build and copy rustc and cargo wrappers
+    for wrapper in ["rustc-clif", "cargo-clif"] {
+        let mut build_cargo_wrapper_cmd = Command::new("rustc");
+        build_cargo_wrapper_cmd
+            .arg(PathBuf::from("scripts").join(format!("{wrapper}.rs")))
+            .arg("-o")
+            .arg(target_dir.join(wrapper))
+            .arg("-g");
+        spawn_and_wait(build_cargo_wrapper_cmd);
+    }
 
     let default_sysroot = super::rustc_info::get_default_sysroot();
 
@@ -117,7 +109,13 @@ pub(crate) fn build_sysroot(
             }
         }
         SysrootKind::Clif => {
-            build_clif_sysroot_for_triple(channel, target_dir, host_triple, None);
+            build_clif_sysroot_for_triple(
+                channel,
+                target_dir,
+                host_triple,
+                &cg_clif_dylib_path,
+                None,
+            );
 
             if host_triple != target_triple {
                 // When cross-compiling it is often necessary to manually pick the right linker
@@ -126,14 +124,21 @@ pub(crate) fn build_sysroot(
                 } else {
                     None
                 };
-                build_clif_sysroot_for_triple(channel, target_dir, target_triple, linker);
+                build_clif_sysroot_for_triple(
+                    channel,
+                    target_dir,
+                    target_triple,
+                    &cg_clif_dylib_path,
+                    linker,
+                );
             }
 
             // Copy std for the host to the lib dir. This is necessary for the jit mode to find
             // libstd.
             for file in fs::read_dir(host_rustlib_lib).unwrap() {
                 let file = file.unwrap().path();
-                if file.file_name().unwrap().to_str().unwrap().contains("std-") {
+                let filename = file.file_name().unwrap().to_str().unwrap();
+                if filename.contains("std-") && !filename.contains(".rlib") {
                     try_hard_link(&file, target_dir.join("lib").join(file.file_name().unwrap()));
                 }
             }
@@ -145,6 +150,7 @@ fn build_clif_sysroot_for_triple(
     channel: &str,
     target_dir: &Path,
     triple: &str,
+    cg_clif_dylib_path: &Path,
     linker: Option<&str>,
 ) {
     match fs::read_to_string(Path::new("build_sysroot").join("rustc_version")) {
@@ -178,7 +184,8 @@ fn build_clif_sysroot_for_triple(
     // Build sysroot
     let mut build_cmd = Command::new("cargo");
     build_cmd.arg("build").arg("--target").arg(triple).current_dir("build_sysroot");
-    let mut rustflags = "--clif -Zforce-unstable-if-unmarked".to_string();
+    let mut rustflags = "-Zforce-unstable-if-unmarked -Cpanic=abort".to_string();
+    rustflags.push_str(&format!(" -Zcodegen-backend={}", cg_clif_dylib_path.to_str().unwrap()));
     if channel == "release" {
         build_cmd.arg("--release");
         rustflags.push_str(" -Zmir-opt-level=3");
@@ -188,10 +195,6 @@ fn build_clif_sysroot_for_triple(
         write!(rustflags, " -Clinker={}", linker).unwrap();
     }
     build_cmd.env("RUSTFLAGS", rustflags);
-    build_cmd.env(
-        "RUSTC",
-        env::current_dir().unwrap().join(target_dir).join("bin").join("cg_clif_build_sysroot"),
-    );
     build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif");
     spawn_and_wait(build_cmd);
 
diff --git a/build_system/mod.rs b/build_system/mod.rs
@@ -86,6 +86,7 @@ pub fn main() {
             arg => arg_error!("Unexpected argument {}", arg),
         }
     }
+    target_dir = std::env::current_dir().unwrap().join(target_dir);
 
     let host_triple = if let Ok(host_triple) = std::env::var("HOST_TRIPLE") {
         host_triple
diff --git a/docs/usage.md b/docs/usage.md
@@ -19,7 +19,7 @@ This will build your project with rustc_codegen_cranelift instead of the usual L
 > You should prefer using the Cargo method.
 
 ```bash
-$ $cg_clif_dir/build/bin/cg_clif my_crate.rs
+$ $cg_clif_dir/build/rustc-clif my_crate.rs
 ```
 
 ## Jit mode
@@ -38,7 +38,7 @@ $ $cg_clif_dir/build/cargo-clif jit
 or
 
 ```bash
-$ $cg_clif_dir/build/bin/cg_clif -Zunstable-features -Cllvm-args=mode=jit -Cprefer-dynamic my_crate.rs
+$ $cg_clif_dir/build/rustc-clif -Zunstable-features -Cllvm-args=mode=jit -Cprefer-dynamic my_crate.rs
 ```
 
 There is also an experimental lazy jit mode. In this mode functions are only compiled once they are
@@ -54,7 +54,7 @@ These are a few functions that allow you to easily run rust code from the shell
 
 ```bash
 function jit_naked() {
-    echo "$@" | $cg_clif_dir/build/bin/cg_clif - -Zunstable-features -Cllvm-args=mode=jit -Cprefer-dynamic
+    echo "$@" | $cg_clif_dir/build/rustc-clif - -Zunstable-features -Cllvm-args=mode=jit -Cprefer-dynamic
 }
 
 function jit() {
diff --git a/scripts/cargo-clif.rs b/scripts/cargo-clif.rs
@@ -5,20 +5,11 @@ use std::path::PathBuf;
 use std::process::Command;
 
 fn main() {
-    if env::var("RUSTC_WRAPPER").map_or(false, |wrapper| wrapper.contains("sccache")) {
-        eprintln!(
-            "\x1b[1;93m=== Warning: Unsetting RUSTC_WRAPPER to prevent interference with sccache ===\x1b[0m"
-        );
-        env::remove_var("RUSTC_WRAPPER");
-    }
-
     let sysroot = PathBuf::from(env::current_exe().unwrap().parent().unwrap());
 
-    env::set_var("RUSTC", sysroot.join("bin/cg_clif".to_string() + env::consts::EXE_SUFFIX));
-
-    let mut rustdoc_flags = env::var("RUSTDOCFLAGS").unwrap_or(String::new());
-    rustdoc_flags.push_str(" -Cpanic=abort -Zpanic-abort-tests -Zcodegen-backend=");
-    rustdoc_flags.push_str(
+    let mut rustflags = String::new();
+    rustflags.push_str(" -Cpanic=abort -Zpanic-abort-tests -Zcodegen-backend=");
+    rustflags.push_str(
         sysroot
             .join(if cfg!(windows) { "bin" } else { "lib" })
             .join(
@@ -29,9 +20,10 @@ fn main() {
             .to_str()
             .unwrap(),
     );
-    rustdoc_flags.push_str(" --sysroot ");
-    rustdoc_flags.push_str(sysroot.to_str().unwrap());
-    env::set_var("RUSTDOCFLAGS", rustdoc_flags);
+    rustflags.push_str(" --sysroot ");
+    rustflags.push_str(sysroot.to_str().unwrap());
+    env::set_var("RUSTFLAGS", env::var("RUSTFLAGS").unwrap_or(String::new()) + &rustflags);
+    env::set_var("RUSTDOCFLAGS", env::var("RUSTDOCFLAGS").unwrap_or(String::new()) + &rustflags);
 
     // Ensure that the right toolchain is used
     env::set_var("RUSTUP_TOOLCHAIN", env!("RUSTUP_TOOLCHAIN"));
diff --git a/scripts/config.sh b/scripts/config.sh
diff --git a/scripts/ext_config.sh b/scripts/ext_config.sh
diff --git a/scripts/filter_profile.rs b/scripts/filter_profile.rs
@@ -2,8 +2,7 @@
 #![forbid(unsafe_code)]/* This line is ignored by bash
 # This block is ignored by rustc
 pushd $(dirname "$0")/../
-source scripts/config.sh
-RUSTC="$(pwd)/build/bin/cg_clif"
+RUSTC="$(pwd)/build/rustc-clif"
 popd
 PROFILE=$1 OUTPUT=$2 exec $RUSTC -Zunstable-options -Cllvm-args=mode=jit -Cprefer-dynamic $0
 #*/
diff --git a/scripts/rustc-clif.rs b/scripts/rustc-clif.rs
@@ -0,0 +1,36 @@
+use std::env;
+use std::ffi::OsString;
+#[cfg(unix)]
+use std::os::unix::process::CommandExt;
+use std::path::PathBuf;
+use std::process::Command;
+
+fn main() {
+    let sysroot = PathBuf::from(env::current_exe().unwrap().parent().unwrap());
+
+    let cg_clif_dylib_path = sysroot.join(if cfg!(windows) { "bin" } else { "lib" }).join(
+        env::consts::DLL_PREFIX.to_string() + "rustc_codegen_cranelift" + env::consts::DLL_SUFFIX,
+    );
+
+    let mut args = std::env::args_os().skip(1).collect::<Vec<_>>();
+    args.push(OsString::from("-Cpanic=abort"));
+    args.push(OsString::from("-Zpanic-abort-tests"));
+    let mut codegen_backend_arg = OsString::from("-Zcodegen-backend=");
+    codegen_backend_arg.push(cg_clif_dylib_path);
+    args.push(codegen_backend_arg);
+    if !args.contains(&OsString::from("--sysroot")) {
+        args.push(OsString::from("--sysroot"));
+        args.push(OsString::from(sysroot.to_str().unwrap()));
+    }
+
+    // Ensure that the right toolchain is used
+    env::set_var("RUSTUP_TOOLCHAIN", env!("RUSTUP_TOOLCHAIN"));
+
+    #[cfg(unix)]
+    Command::new("rustc").args(args).exec();
+
+    #[cfg(not(unix))]
+    std::process::exit(
+        Command::new("rustc").args(args).spawn().unwrap().wait().unwrap().code().unwrap_or(1),
+    );
+}
diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh
@@ -2,7 +2,6 @@
 set -e
 
 ./y.rs build --no-unstable-features
-source scripts/config.sh
 
 echo "[SETUP] Rust fork"
 git clone https://github.com/rust-lang/rust.git || true
@@ -52,7 +51,7 @@ changelog-seen = 2
 ninja = false
 
 [build]
-rustc = "$(pwd)/../build/bin/cg_clif"
+rustc = "$(pwd)/../build/rustc-clif"
 cargo = "$(rustup which cargo)"
 full-bootstrap = true
 local-rebuild = true
diff --git a/scripts/tests.sh b/scripts/tests.sh
@@ -2,10 +2,34 @@
 
 set -e
 
-source scripts/config.sh
-source scripts/ext_config.sh
-export RUSTC=false # ensure that cg_llvm isn't accidentally used
-MY_RUSTC="$(pwd)/build/bin/cg_clif $RUSTFLAGS -L crate=target/out --out-dir target/out -Cdebuginfo=2"
+export CG_CLIF_DISPLAY_CG_TIME=1
+export CG_CLIF_DISABLE_INCR_CACHE=1
+
+export HOST_TRIPLE=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ")
+export TARGET_TRIPLE=${TARGET_TRIPLE:-$HOST_TRIPLE}
+
+export RUN_WRAPPER=''
+export JIT_SUPPORTED=1
+if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
+   export JIT_SUPPORTED=0
+   if [[ "$TARGET_TRIPLE" == "aarch64-unknown-linux-gnu" ]]; then
+      # We are cross-compiling for aarch64. Use the correct linker and run tests in qemu.
+      export RUSTFLAGS='-Clinker=aarch64-linux-gnu-gcc '$RUSTFLAGS
+      export RUN_WRAPPER='qemu-aarch64 -L /usr/aarch64-linux-gnu'
+   elif [[ "$TARGET_TRIPLE" == "x86_64-pc-windows-gnu" ]]; then
+      # We are cross-compiling for Windows. Run tests in wine.
+      export RUN_WRAPPER='wine'
+   else
+      echo "Unknown non-native platform"
+   fi
+fi
+
+# FIXME fix `#[linkage = "extern_weak"]` without this
+if [[ "$(uname)" == 'Darwin' ]]; then
+   export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
+fi
+
+MY_RUSTC="$(pwd)/build/rustc-clif $RUSTFLAGS -L crate=target/out --out-dir target/out -Cdebuginfo=2"
 
 function no_sysroot_tests() {
     echo "[BUILD] mini_core"
@@ -97,7 +121,7 @@ function extended_sysroot_tests() {
     if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
         echo "[BENCH COMPILE] ebobby/simple-raytracer"
         hyperfine --runs "${RUN_RUNS:-10}" --warmup 1 --prepare "../build/cargo-clif clean" \
-        "RUSTC=rustc RUSTFLAGS='' cargo build" \
+        "RUSTFLAGS='' cargo build" \
         "../build/cargo-clif build"
 
         echo "[BENCH RUN] ebobby/simple-raytracer"
diff --git a/src/bin/cg_clif.rs b/src/bin/cg_clif.rs
diff --git a/src/bin/cg_clif_build_sysroot.rs b/src/bin/cg_clif_build_sysroot.rs

Original file line number	Diff line number	Diff line change
`@@ -86,6 +86,7 @@ pub fn main() {`
`86`	`86`	`arg => arg_error!("Unexpected argument {}", arg),`
`87`	`87`	`}`
`88`	`88`	`}`
	`89`	`+ target_dir = std::env::current_dir().unwrap().join(target_dir);`
`89`	`90`
`90`	`91`	`let host_triple = if let Ok(host_triple) = std::env::var("HOST_TRIPLE") {`
`91`	`92`	`host_triple`