Skip to content

Commit c1359cb

Browse files
timcasselltimcassell
authored
Improve Benchmark Accuracy (#2336)
* Call benchmark method directly instead of via delegate. * Removed `Consumer` from benchmark actions. Overhead always returns `void`. * Removed workload return types from code gen. * Apply `NoOptimization` instead of `AggressiveOptimization` to loop methods. Count down loops instead of count up. Added IntroSmokeStringBuilder. Added more return type test cases. * Added assembly weaver. Reverted loop methods back to `AggressiveOptimization`. Added `NoInlining` to `__Overhead` to match weaved benchmark method. Updated ExpectedBenchmarkResultsTests. * Remove unused constant. * Remove InProcess check from validation. Update Weaver. * Fix comment. * Zero threshold. * Update built package version. * Fix test. * Use AsmResolver instead of Mono.Cecil. Run task after CoreCompile instead of AfterBuild. * Improve pack-weaver task to auto-delete old package. Updated comments. * Try latest beta version of AsmResolver.DotNet. * Fix removal of AggressiveInlining. * Make AsmResolver non-transitive. * Change target back to AfterBuild. * Make all packages non-transitive. * Update packages to latest after rebase. * Don't write module if it's unnecessary. Don't fail silently. Remove unused property. Embed the weaver into BenchmarkDotNet.Annotations package instead of another package dependency. * Rebuild after rebase. * Added comment. Fixed formatting. * Write to null stream before write to actual file. * Updated to latest AsmResolver. Write to MemoryStream first for efficiency. * Ensure WeaveAssemblies task does not run concurrently with Publish. * Condition on WeaveAssemblyTask instead of Target. * Fix compile errors and pack weaver after version-increment. * Update to latest AsmResolver. * Validate NoInlining in the benchmark process. * Add [UsedImplicitly].
1 parent 0767efe commit c1359cb

File tree

48 files changed

+754
-1983
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+754
-1983
lines changed

‎BenchmarkDotNet.sln‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,11 @@ EndProject
5858
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Analyzers", "src\BenchmarkDotNet.Analyzers\BenchmarkDotNet.Analyzers.csproj", "{AA4DDCA0-C1D8-ADA8-69FE-2F67C4CA96B1}"
5959
EndProject
6060
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Analyzers.Tests", "tests\BenchmarkDotNet.Analyzers.Tests\BenchmarkDotNet.Analyzers.Tests.csproj", "{7DE89F16-2160-42E3-004E-1F5064732121}"
61+
EndProject
6162
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Disassembler", "src\BenchmarkDotNet.Disassembler\BenchmarkDotNet.Disassembler.csproj", "{86DB5A20-73FA-DB96-1545-08FE5878637C}"
6263
EndProject
64+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Weaver", "src\BenchmarkDotNet.Weaver\BenchmarkDotNet.Weaver.csproj", "{7AD7DA82-ADDB-2171-A37B-07D617036CF5}"
65+
EndProject
6366
Global
6467
GlobalSection(SolutionConfigurationPlatforms) = preSolution
6568
Debug|Any CPU = Debug|Any CPU
@@ -166,6 +169,10 @@ Global
166169
{86DB5A20-73FA-DB96-1545-08FE5878637C}.Debug|Any CPU.Build.0 = Debug|Any CPU
167170
{86DB5A20-73FA-DB96-1545-08FE5878637C}.Release|Any CPU.ActiveCfg = Release|Any CPU
168171
{86DB5A20-73FA-DB96-1545-08FE5878637C}.Release|Any CPU.Build.0 = Release|Any CPU
172+
{7AD7DA82-ADDB-2171-A37B-07D617036CF5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
173+
{7AD7DA82-ADDB-2171-A37B-07D617036CF5}.Debug|Any CPU.Build.0 = Debug|Any CPU
174+
{7AD7DA82-ADDB-2171-A37B-07D617036CF5}.Release|Any CPU.ActiveCfg = Release|Any CPU
175+
{7AD7DA82-ADDB-2171-A37B-07D617036CF5}.Release|Any CPU.Build.0 = Release|Any CPU
169176
EndGlobalSection
170177
GlobalSection(SolutionProperties) = preSolution
171178
HideSolutionNode = FALSE
@@ -196,6 +203,7 @@ Global
196203
{AA4DDCA0-C1D8-ADA8-69FE-2F67C4CA96B1} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
197204
{7DE89F16-2160-42E3-004E-1F5064732121} = {14195214-591A-45B7-851A-19D3BA2413F9}
198205
{86DB5A20-73FA-DB96-1545-08FE5878637C} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
206+
{7AD7DA82-ADDB-2171-A37B-07D617036CF5} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
199207
EndGlobalSection
200208
GlobalSection(ExtensibilityGlobals) = postSolution
201209
SolutionGuid = {4D9AF12B-1F7F-45A7-9E8C-E4E46ADCBD1F}

‎NuGet.Config‎

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,23 @@
1-
<?xml version="1.0" encoding="utf-8"?>
2-
<configuration>
3-
<solution>
4-
<add key="disableSourceControlIntegration" value="true" />
5-
</solution>
6-
<packageSources>
7-
<!--To inherit the global NuGet package sources remove the <clear/> line below -->
8-
<clear />
9-
10-
<add key="api.nuget.org" value="https://api.nuget.org/v3/index.json" />
11-
<!-- required to run Mono AOT benchmarks -->
12-
<add key="dotnet6" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json" />
13-
<add key="dotnet7" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json" />
14-
<add key="dotnet8" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json" />
15-
<add key="dotnet9" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v3/index.json" />
16-
<add key="dotnet10" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json" />
17-
18-
<!-- required for Roslyn analyzers -->
19-
<add key="dotnet-tools" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" />
20-
</packageSources>
21-
</configuration>
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<configuration>
3+
<solution>
4+
<add key="disableSourceControlIntegration" value="true" />
5+
</solution>
6+
<packageSources>
7+
<!--To inherit the global NuGet package sources remove the <clear/> line below -->
8+
<clear />
9+
10+
<add key="api.nuget.org" value="https://api.nuget.org/v3/index.json" />
11+
<!-- required to run Mono AOT benchmarks -->
12+
<add key="dotnet6" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json" />
13+
<add key="dotnet7" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json" />
14+
<add key="dotnet8" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json" />
15+
<add key="dotnet9" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v3/index.json" />
16+
<add key="dotnet10" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json" />
17+
18+
<!-- required for Roslyn analyzers -->
19+
<add key="dotnet-tools" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" />
20+
21+
<add key="benchmarkdotnet.weaver" value="src/BenchmarkDotNet.Weaver/packages" />
22+
</packageSources>
23+
</configuration>

‎build/BenchmarkDotNet.Build/Program.cs‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,29 @@ public static int Main(string[] args)
1616
}
1717
}
1818

19+
[TaskName(Name)]
20+
[TaskDescription("Pack Weaver")]
21+
public class PackWeaverTask : FrostingTask<BuildContext>, IHelpProvider
22+
{
23+
private const string Name = "pack-weaver";
24+
25+
public override void Run(BuildContext context) => context.BuildRunner.PackWeaver();
26+
27+
public HelpInfo GetHelp()
28+
{
29+
return new HelpInfo
30+
{
31+
Examples = new[]
32+
{
33+
new Example(Name)
34+
}
35+
};
36+
}
37+
}
38+
1939
[TaskName(Name)]
2040
[TaskDescription("Restore NuGet packages")]
41+
[IsDependentOn(typeof(PackWeaverTask))]
2142
public class RestoreTask : FrostingTask<BuildContext>, IHelpProvider
2243
{
2344
private const string Name = "restore";
@@ -292,7 +313,11 @@ public class DocsBuildTask : FrostingTask<BuildContext>, IHelpProvider
292313
public class VersionIncrementTask : FrostingTask<BuildContext>, IHelpProvider
293314
{
294315
private const string Name = "version-increment";
295-
public override void Run(BuildContext context) => context.ReleaseRunner.VersionIncrement();
316+
public override void Run(BuildContext context)
317+
{
318+
context.ReleaseRunner.VersionIncrement();
319+
context.BuildRunner.PackWeaver();
320+
}
296321

297322
public HelpInfo GetHelp() => new()
298323
{

‎build/BenchmarkDotNet.Build/Runners/BuildRunner.cs‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
using Cake.Common.Tools.DotNet.Workload.Install;
99
using Cake.Core;
1010
using Cake.Core.IO;
11+
using System.IO;
12+
using System.Linq;
1113

1214
namespace BenchmarkDotNet.Build.Runners;
1315

@@ -20,6 +22,41 @@ public BuildRunner(BuildContext context)
2022
this.context = context;
2123
}
2224

25+
public void PackWeaver()
26+
{
27+
var weaverPath = context.AllPackableSrcProjects.Single(p => p.GetFilename() == "BenchmarkDotNet.Weaver.csproj");
28+
var outputPackageDir = weaverPath.GetDirectory().Combine("packages");
29+
30+
// Delete old package.
31+
foreach (var file in Directory.EnumerateFiles(outputPackageDir.FullPath))
32+
{
33+
File.Delete(file);
34+
}
35+
36+
context.DotNetRestore(weaverPath.GetDirectory().FullPath,
37+
new DotNetRestoreSettings
38+
{
39+
MSBuildSettings = context.MsBuildSettingsRestore
40+
});
41+
42+
context.Information("BuildSystemProvider: " + context.BuildSystem().Provider);
43+
context.DotNetBuild(weaverPath.FullPath, new DotNetBuildSettings
44+
{
45+
NoRestore = true,
46+
DiagnosticOutput = true,
47+
MSBuildSettings = context.MsBuildSettingsBuild,
48+
Configuration = context.BuildConfiguration,
49+
Verbosity = context.BuildVerbosity
50+
});
51+
52+
context.DotNetPack(weaverPath.FullPath, new DotNetPackSettings
53+
{
54+
OutputDirectory = outputPackageDir,
55+
MSBuildSettings = context.MsBuildSettingsPack,
56+
Configuration = context.BuildConfiguration
57+
});
58+
}
59+
2360
public void Restore()
2461
{
2562
context.DotNetRestore(context.SolutionFile.FullPath,
@@ -89,7 +126,7 @@ public void Pack()
89126
var settingsSrc = new DotNetPackSettings
90127
{
91128
OutputDirectory = context.ArtifactsDirectory,
92-
ArgumentCustomization = args => args.Append("--include-symbols").Append("-p:SymbolPackageFormat=snupkg"),
129+
ArgumentCustomization = args => args.Append("--include-symbols").Append("-p:SymbolPackageFormat=snupkg").Append("-p:IsFullPack=true"),
93130
MSBuildSettings = context.MsBuildSettingsPack,
94131
Configuration = context.BuildConfiguration
95132
};

‎build/common.props‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
</PropertyGroup>
4646

4747
<PropertyGroup Condition=" '$(VersionPrefix)' == '' ">
48+
<!-- When this is changed, optionally reset WeaverVersionSuffix, then run `build.cmd pack-weaver`. -->
4849
<VersionPrefix>0.16.0</VersionPrefix>
4950
</PropertyGroup>
5051

@@ -60,6 +61,11 @@
6061
<PackageVersion>$(Version)</PackageVersion>
6162
</PropertyGroup>
6263

64+
<PropertyGroup>
65+
<!-- Increment this when the BenchmarkDotNet.Weaver package needs to be re-packed. -->
66+
<WeaverVersionSuffix>-1</WeaverVersionSuffix>
67+
</PropertyGroup>
68+
6369
<ItemGroup>
6470
<PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.3">
6571
<PrivateAssets>all</PrivateAssets>

‎build/versions.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,4 +67,4 @@
6767
0.15.6
6868
0.15.7
6969
0.15.8
70-
0.16.0
70+
0.16.0

‎samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
<!-- MSBuild was complaing about InformationalVersion from common.props file, so I excluded them in conditional way -->
44
<IsFsharp>true</IsFsharp>
55
</PropertyGroup>
6+
<Import Project="..\..\build\common.props" />
67
<PropertyGroup>
78
<OutputType>Exe</OutputType>
89
<TargetFrameworks>net462;net8.0</TargetFrameworks>

‎samples/BenchmarkDotNet.Samples/IntroSmokeStringBuilder.cs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
using BenchmarkDotNet.Attributes;
2+
using System.Text;
3+
4+
namespace BenchmarkDotNet.Samples
5+
{
6+
[MemoryDiagnoser(false)]
7+
public class IntroSmokeStringBuilder
8+
{
9+
[Benchmark]
10+
[Arguments(1)]
11+
[Arguments(1_000)]
12+
public StringBuilder Append_Strings(int repeat)
13+
{
14+
StringBuilder builder = new StringBuilder();
15+
16+
// strings are not sorted by length to mimic real input
17+
for (int i = 0; i < repeat; i++)
18+
{
19+
builder.Append("12345");
20+
builder.Append("1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMN");
21+
builder.Append("1234567890abcdefghijklmnopqrstuvwxy");
22+
builder.Append("1234567890");
23+
builder.Append("1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHI");
24+
builder.Append("1234567890abcde");
25+
builder.Append("1234567890abcdefghijklmnopqrstuvwxyzABCD");
26+
builder.Append("1234567890abcdefghijklmnopqrst");
27+
builder.Append("1234567890abcdefghij");
28+
builder.Append("1234567890abcdefghijklmno");
29+
}
30+
31+
return builder;
32+
}
33+
}
34+
}

‎src/BenchmarkDotNet.Annotations/BenchmarkDotNet.Annotations.csproj‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<PropertyGroup>
44
<AssemblyTitle>Basic BenchmarkDotNet attributes that can be used to annotate your benchmarks</AssemblyTitle>
55
<TargetFrameworks>netstandard2.0</TargetFrameworks>
6-
<NoWarn>$(NoWarn);1701;1702;1705;1591;3005;NU1702;CA1825</NoWarn>
6+
<NoWarn>$(NoWarn);1701;1702;1705;1591;3005;NU1702;NU5100;CA1825</NoWarn>
77
<AssemblyName>BenchmarkDotNet.Annotations</AssemblyName>
88
<PackageId>BenchmarkDotNet.Annotations</PackageId>
99
<RootNamespace>BenchmarkDotNet</RootNamespace>
@@ -27,7 +27,24 @@
2727
<ItemGroup>
2828
<Compile Include="..\BenchmarkDotNet\Helpers\CodeAnnotations.cs" Link="Attributes\CodeAnnotations.cs" />
2929
</ItemGroup>
30+
3031
<ItemGroup>
3132
<Content Include="..\BenchmarkDotNet.Analyzers\bin\$(Configuration)\**\*.dll" Pack="true" PackagePath="analyzers/dotnet/" Visible="false" />
3233
</ItemGroup>
34+
35+
<Choose>
36+
<When Condition="'$(IsFullPack)' == 'true'">
37+
<ItemGroup>
38+
<!-- Include the BenchmarkDotNet.Weaver dlls in tasks without making it an explicit dependency. -->
39+
<Content Include="$(MSBuildThisFileDirectory)..\BenchmarkDotNet.Weaver\bin\Release\$(TargetFramework)\**\*.dll" Pack="true" PackagePath="tasks/$(TargetFramework)" />
40+
<Content Include="$(MSBuildThisFileDirectory)buildTransitive\**\*.targets" Pack="true" PackagePath="buildTransitive" />
41+
</ItemGroup>
42+
</When>
43+
<Otherwise>
44+
<!-- PackageReference for transitive weaver dependency for ProjectReferences to this. -->
45+
<ItemGroup>
46+
<PackageReference Include="BenchmarkDotNet.Weaver" Version="$(Version)$(WeaverVersionSuffix)" />
47+
</ItemGroup>
48+
</Otherwise>
49+
</Choose>
3350
</Project>

‎src/BenchmarkDotNet.Annotations/buildTransitive/netstandard2.0/BenchmarkDotNet.Annotations.targets‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
<ItemGroup>
4+
<Reference Include="BenchmarkDotNet.Annotations">
5+
<HintPath>$(MSBuildThisFileDirectory)..\..\lib\netstandard2.0\BenchmarkDotNet.Annotations.dll</HintPath>
6+
</Reference>
7+
</ItemGroup>
8+
9+
<UsingTask TaskName="BenchmarkDotNet.Weaver.WeaveAssemblyTask" AssemblyFile="$(MSBuildThisFileDirectory)..\..\tasks\netstandard2.0\BenchmarkDotNet.Weaver.dll" />
10+
11+
<Target Name="WeaveAssemblies" AfterTargets="AfterBuild" >
12+
<WeaveAssemblyTask TargetAssembly="$(TargetDir)$(TargetFileName)" />
13+
</Target>
14+
</Project>

0 commit comments

Comments
 (0)