chore: Update BenchmarkDotNet.Samples project (dotnet#2898)

* chore: update samples project

* chore-add-support-DebugBuildConfig

* chore: fix ThreadingDiagnoser error when running benchmark on .NET Framework

* chore: fix wrong #if directive

* chore: reflect review comments and cleanup codes

Author: filzrev

samples/BenchmarkDotNet.Samples/Program.cs

@@ -1,9 +1,58 @@
-using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Exporters;
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Running;
+using System;
+using System.Linq;
 
-namespace BenchmarkDotNet.Samples
+namespace BenchmarkDotNet.Samples;
+
+public class Program
 {
-    public class Program
+    public static int Main(string[] args)
     {
-        public static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
+#if DEBUG
+        ConsoleLogger.Default.WriteLineWarning("Benchmark is executed with DEBUG configuration.");
+        ConsoleLogger.Default.WriteLine();
+#endif
+
+        if (args.Length != 0)
+        {
+            ConsoleLogger.Default.WriteLine($"Start benchmarks with args: {string.Join(" ", args)}");
+            ConsoleLogger.Default.WriteLine();
+        }
+
+        IConfig? config = GetConfig(ref args);
+
+        var summaries = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly)
+                                         .Run(args, config)
+                                         .ToArray();
+
+        if (summaries.HasError())
+            return 1;
+
+        return 0;
+    }
+
+    private static IConfig? GetConfig(ref string[] args)
+    {
+#if !DEBUG
+        return null; // `DefaultConfig.Instance` is used.
+#else
+        bool isInProcess = args.Contains("--inProcess");
+        if (isInProcess)
+            args = args.Where(x => x != "--inProcess").ToArray();
+
+        DebugConfig config = isInProcess
+            ? new DebugInProcessConfig()
+            : new DebugBuildConfig();
+
+        return config.AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray())
+                     .AddExporter(MarkdownExporter.Default)
+                     .AddValidator(DefaultConfig.Instance.GetValidators().ToArray())
+                     .WithArtifactsPath(DefaultConfig.Instance.ArtifactsPath);
+#endif
     }
-}
+}

samples/BenchmarkDotNet.Samples/Properties/launchSettings.json

@@ -0,0 +1,38 @@
+{
+  "profiles": {
+    "Default": {
+      "commandName": "Project",
+      "commandLineArgs": "",
+      "environmentVariables": {
+      }
+    },
+    "Run IntroBasic Benchmarks": {
+      "commandName": "Project",
+      "commandLineArgs": "--filter *IntroBasic*",
+      "environmentVariables": {
+      }
+    },
+    "Run IntroBasic Benchmarks with --inProcess": {
+      "commandName": "Project",
+      "commandLineArgs": "--filter *IntroBasic* --inProcess",
+      "environmentVariables": {
+      }
+    },
+    "--list": {
+      "commandName": "Project",
+      "commandLineArgs": "--list"
+    },
+    "--info": {
+      "commandName": "Project",
+      "commandLineArgs": "--info"
+    },
+    "--help": {
+      "commandName": "Project",
+      "commandLineArgs": "--help"
+    },
+    "--version": {
+      "commandName": "Project",
+      "commandLineArgs": "--version"
+    }
+  }
+}

src/BenchmarkDotNet/Extensions/ReportExtensions.cs

@@ -34,5 +34,20 @@ public static Statistics GetStatistics(this IReadOnlyCollection<Measurement> run
 
         public static Statistics GetStatistics(this IEnumerable<Measurement> runs) =>
             GetStatistics(runs.ToList());
+
+        public static bool HasError(this IEnumerable<Summary> summaries)
+        {
+            if (summaries.Count() == 0)
+            {
+                // When following argument specified. BenchmarkDotNet show information only.
+                var knownArguments = new HashSet<string>(["--help", "--list", "--info", "--version"]);
+                return !Environment.GetCommandLineArgs().Any(knownArguments.Contains);
+            }
+
+            if (summaries.Any(x => x.HasCriticalValidationErrors))
+                return true;
+
+            return summaries.Any(x => x.Reports.Any(r => !r.Success));
+        }
     }
 }