Auto merge of #113749 - Kobzol:opt-dist-ci-group, r=clubby789

Use log groups in `opt-dist`

Some of the output was quite verbose in CI logs, this should help with that.

r? bootstrap

Author: bors

src/tools/opt-dist/src/main.rs

@@ -9,6 +9,7 @@ use crate::training::{gather_llvm_bolt_profiles, gather_llvm_profiles, gather_ru
 use crate::utils::io::reset_directory;
 use crate::utils::{
     clear_llvm_files, format_env_variables, print_binary_sizes, print_free_disk_space,
+    with_log_group,
 };
 
 mod environment;
@@ -29,7 +30,8 @@ fn execute_pipeline(
     dist_args: Vec<String>,
 ) -> anyhow::Result<()> {
     reset_directory(&env.opt_artifacts())?;
-    env.prepare_rustc_perf()?;
+
+    with_log_group("Building rustc-perf", || env.prepare_rustc_perf())?;
 
     // Stage 1: Build PGO instrumented rustc
     // We use a normal build of LLVM, because gathering PGO profiles for LLVM and `rustc` at the
@@ -141,12 +143,17 @@ fn main() -> anyhow::Result<()> {
         .init();
 
     let mut build_args: Vec<String> = std::env::args().skip(1).collect();
-    log::info!("Running optimized build pipeline with args `{}`", build_args.join(" "));
-    log::info!("Environment values\n{}", format_env_variables());
+    println!("Running optimized build pipeline with args `{}`", build_args.join(" "));
 
-    if let Ok(config) = std::fs::read_to_string("config.toml") {
-        log::info!("Contents of `config.toml`:\n{config}");
-    }
+    with_log_group("Environment values", || {
+        println!("Environment values\n{}", format_env_variables());
+    });
+
+    with_log_group("Printing config.toml", || {
+        if let Ok(config) = std::fs::read_to_string("config.toml") {
+            println!("Contents of `config.toml`:\n{config}");
+        }
+    });
 
     // Skip components that are not needed for try builds to speed them up
     if is_try_build() {

src/tools/opt-dist/src/training.rs

@@ -1,6 +1,7 @@
 use crate::environment::Environment;
 use crate::exec::{cmd, CmdBuilder};
 use crate::utils::io::{count_files, delete_directory};
+use crate::utils::with_log_group;
 use anyhow::Context;
 use camino::{Utf8Path, Utf8PathBuf};
 use humansize::BINARY;
@@ -108,9 +109,11 @@ pub fn gather_llvm_profiles(
 ) -> anyhow::Result<LlvmPGOProfile> {
     log::info!("Running benchmarks with PGO instrumented LLVM");
 
-    init_compiler_benchmarks(env, &["Debug", "Opt"], &["Full"], LLVM_PGO_CRATES)
-        .run()
-        .context("Cannot gather LLVM PGO profiles")?;
+    with_log_group("Running benchmarks", || {
+        init_compiler_benchmarks(env, &["Debug", "Opt"], &["Full"], LLVM_PGO_CRATES)
+            .run()
+            .context("Cannot gather LLVM PGO profiles")
+    })?;
 
     let merged_profile = env.opt_artifacts().join("llvm-pgo.profdata");
     log::info!("Merging LLVM PGO profiles to {merged_profile}");
@@ -141,10 +144,12 @@ pub fn gather_rustc_profiles(
 
     // Here we're profiling the `rustc` frontend, so we also include `Check`.
     // The benchmark set includes various stress tests that put the frontend under pressure.
-    init_compiler_benchmarks(env, &["Check", "Debug", "Opt"], &["All"], RUSTC_PGO_CRATES)
-        .env("LLVM_PROFILE_FILE", profile_template.as_str())
-        .run()
-        .context("Cannot gather rustc PGO profiles")?;
+    with_log_group("Running benchmarks", || {
+        init_compiler_benchmarks(env, &["Check", "Debug", "Opt"], &["All"], RUSTC_PGO_CRATES)
+            .env("LLVM_PROFILE_FILE", profile_template.as_str())
+            .run()
+            .context("Cannot gather rustc PGO profiles")
+    })?;
 
     let merged_profile = env.opt_artifacts().join("rustc-pgo.profdata");
     log::info!("Merging Rustc PGO profiles to {merged_profile}");
@@ -164,9 +169,11 @@ pub struct LlvmBoltProfile(pub Utf8PathBuf);
 pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result<LlvmBoltProfile> {
     log::info!("Running benchmarks with BOLT instrumented LLVM");
 
-    init_compiler_benchmarks(env, &["Check", "Debug", "Opt"], &["Full"], LLVM_BOLT_CRATES)
-        .run()
-        .context("Cannot gather LLVM BOLT profiles")?;
+    with_log_group("Running benchmarks", || {
+        init_compiler_benchmarks(env, &["Check", "Debug", "Opt"], &["Full"], LLVM_BOLT_CRATES)
+            .run()
+            .context("Cannot gather LLVM BOLT profiles")
+    })?;
 
     let merged_profile = env.opt_artifacts().join("bolt.profdata");
     let profile_root = Utf8PathBuf::from("/tmp/prof.fdata");
@@ -178,10 +185,12 @@ pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result<LlvmBo
     let mut merge_args = vec!["merge-fdata"];
     merge_args.extend(profiles.iter().map(|p| p.to_str().unwrap()));
 
-    cmd(&merge_args)
-        .redirect_output(merged_profile.clone())
-        .run()
-        .context("Cannot merge BOLT profiles")?;
+    with_log_group("Merging BOLT profiles", || {
+        cmd(&merge_args)
+            .redirect_output(merged_profile.clone())
+            .run()
+            .context("Cannot merge BOLT profiles")
+    })?;
 
     log::info!("LLVM BOLT statistics");
     log::info!(

src/tools/opt-dist/src/utils/mod.rs

@@ -56,3 +56,20 @@ pub fn clear_llvm_files(env: &dyn Environment) -> anyhow::Result<()> {
     delete_directory(&env.build_artifacts().join("lld"))?;
     Ok(())
 }
+
+/// Wraps all output produced within the `func` closure in a CI output group, if we're running in
+/// CI.
+pub fn with_log_group<F: FnOnce() -> R, R>(group: &str, func: F) -> R {
+    if is_in_ci() {
+        println!("::group::{group}");
+        let result = func();
+        println!("::endgroup::");
+        result
+    } else {
+        func()
+    }
+}
+
+fn is_in_ci() -> bool {
+    std::env::var("GITHUB_ACTIONS").is_ok()
+}