Skip to content

Add support for running microbenchmarks using a manifest json #4912

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
Open

Add support for running microbenchmarks using a manifest json #4912

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
938bb8a
Add test manifest support for running microbenchmarks
caaavik-msft Aug 19, 2025
40df0c5
Add missing argument to script
caaavik-msft Aug 19, 2025
77b7fd4
Fix bugs
caaavik-msft Aug 19, 2025
abc025e
Fix command line parsing
caaavik-msft Aug 19, 2025
a5857af
Abstract out payload building
caaavik-msft Aug 20, 2025
7b0ef69
Minor fixes
caaavik-msft Aug 20, 2025
3a425ea
Abstracting out more parts
caaavik-msft Aug 20, 2025
16e88be
Some more cleanup/fixes
caaavik-msft Aug 20, 2025
8d56056
Remove python/yml changes (now in a separate PR)
caaavik-msft Aug 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions src/benchmarks/micro/MicroBenchmarks.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,12 +82,12 @@
<PackageReference Include="Microsoft.Extensions.Logging.EventSource" Version="$(ExtensionsVersion)" />
<PackageReference Include="Microsoft.Extensions.Primitives" Version="$(ExtensionsVersion)" />
<PackageReference Include="protobuf-net" Version="3.0.73" />
<PackageReference Include="System.Drawing.Common" Version="$(SystemVersion)" Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp' And '$(_TargetFrameworkVersionWithoutV)' &lt; '8.0'" />
<PackageReference Include="System.Formats.Cbor" Version="$(SystemVersion)" Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp' And '$(_TargetFrameworkVersionWithoutV)' &gt;= '5.0'" />
<PackageReference Include="System.Drawing.Common" Version="$(SystemVersion)" Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp' and !$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))" />
<PackageReference Include="System.Formats.Cbor" Version="$(SystemVersion)" Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp' and $([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net5.0'))" />
<PackageReference Include="System.IO.Hashing" Version="$(SystemVersion)" />
<PackageReference Include="System.Net.Http.WinHttpHandler" Version="$(SystemVersion)" Condition="'$(OS)' == 'Windows_NT'" />
<PackageReference Include="Utf8Json" Version="1.3.7" />
<PackageReference Include="System.Collections.Immutable" Version="8.0.*-*" Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework' Or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' And '$(_TargetFrameworkVersionWithoutV)' &lt; '8.0')" />
<PackageReference Include="System.Collections.Immutable" Version="8.0.*-*" Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework' Or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' And !$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0')))" />
</ItemGroup>
<ItemGroup Condition="'$(TargetFrameworkIdentifier)' != '.NETCoreApp'">
<PackageReference Include="System.Memory" Version="4.5.5" />
Expand All @@ -96,12 +96,12 @@
<PackageReference Include="System.IO.Pipelines" Version="$(SystemVersion)" />
<PackageReference Include="System.Threading.Channels" Version="$(SystemVersion)" />
</ItemGroup>
<ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp' And '$(_TargetFrameworkVersionWithoutV)' &lt; '10.0'">
<ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp' And !$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net10.0'))">
<PackageReference Include="System.Diagnostics.DiagnosticSource" Version="$(SystemVersion)" />
<PackageReference Include="System.IO.Pipelines" Version="$(SystemVersion)" />
<PackageReference Include="System.Threading.Channels" Version="$(SystemVersion)" />
</ItemGroup>
<ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp' And '$(_TargetFrameworkVersionWithoutV)' &gt;= '8.0'">
<ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp' And $([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))">
<!-- we want both net8.0 and net9.0 targets to use the 9.0.0 version of the NuGet package (since this is where the generic math APIs were first added) -->
<PackageReference Include="System.Numerics.Tensors" Version="$(SystemThreadingChannelsPackageVersion)" />
</ItemGroup>
Expand Down Expand Up @@ -149,7 +149,7 @@
<Compile Remove="runtime\Math\Functions\Single\**\*.cs" />
</ItemGroup>

<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' Or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' And '$(_TargetFrameworkVersionWithoutV)' &lt; '3.1')">
<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' Or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' And !$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'netcoreapp3.1')))">
<Compile Remove="runtime\Math\Functions\Double\CopySign.cs" />
<Compile Remove="runtime\Math\Functions\Double\FusedMultiplyAdd.cs" />
<Compile Remove="runtime\Math\Functions\Double\ILogB.cs" />
Expand All @@ -170,7 +170,7 @@
<Compile Remove="libraries\Common\MultiArrayBuffer.cs" />
</ItemGroup>

<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' Or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' And '$(_TargetFrameworkVersionWithoutV)' &lt; '5.0')">
<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' Or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' And !$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net5.0')))">
<Compile Remove="runtime\Interop\ComWrappers.cs" />
<Compile Remove="libraries\System.Diagnostics\Perf_Activity.cs" />
<Compile Remove="libraries\System.Text.Json\Serializer\ReadPreservedReferences.cs" />
Expand All @@ -180,7 +180,7 @@
<Compile Remove="libraries\System.Runtime\Perf.Half.cs" />
</ItemGroup>

<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' Or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' And '$(_TargetFrameworkVersionWithoutV)' &lt; '6.0')">
<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' Or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' And !$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net6.0')))">
<Compile Remove="runtime\Math\Functions\Double\SinCos.cs" />
<Compile Remove="runtime\Math\Functions\Single\SinCos.cs" />
<Compile Remove="libraries\System.Collections\PriorityQueue\Perf_PriorityQueue.cs" />
Expand All @@ -190,7 +190,7 @@
<Compile Remove="libraries\System.Text.Json\Node\*.cs" />
</ItemGroup>

<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' Or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' And '$(_TargetFrameworkVersionWithoutV)' &lt; '7.0')">
<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' Or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' And !$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net7.0')))">
<Compile Remove="runtime\Math\Functions\Double\AcosPi.cs" />
<Compile Remove="runtime\Math\Functions\Double\AsinPi.cs" />
<Compile Remove="runtime\Math\Functions\Double\AtanPi.cs" />
Expand Down Expand Up @@ -228,24 +228,24 @@
<Compile Remove="libraries\System.Runtime\Perf.Int128.cs" />
</ItemGroup>

<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' Or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' And '$(_TargetFrameworkVersionWithoutV)' &lt; '8.0')">
<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' Or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' And !$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0')))">
<Compile Remove="libraries\System.Buffers\SearchValuesByteTests.cs" />
<Compile Remove="libraries\System.Buffers\SearchValuesCharTests.cs" />
<Compile Remove="libraries\System.Text.Encoding\Perf.Ascii.cs" />
<Compile Remove="libraries\System.Numerics.Tensors\*.cs" />
</ItemGroup>

<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' Or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' And '$(_TargetFrameworkVersionWithoutV)' &lt; '9.0')">
<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' Or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' And !$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net9.0')))">
<Compile Remove="libraries\System.Reflection.Metadata\*.cs" />
</ItemGroup>

<!-- Remove Sve microbenchmarks when running on net versions < 9.0 -->
<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' Or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' And '$(_TargetFrameworkVersionWithoutV)' &lt; '9.0')">
<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' Or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' And !$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net9.0')))">
<Compile Remove="sve\*.cs" />
</ItemGroup>

<!-- This is not removing things from older Net versions, it is removing from newer Net versions -->
<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' Or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' And '$(_TargetFrameworkVersionWithoutV)' &gt;= '8.0')">
<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' Or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' And $([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0')))">
<Compile Remove="libraries\System.Drawing\*.cs" />
</ItemGroup>

Expand Down
112 changes: 78 additions & 34 deletions src/benchmarks/micro/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,57 +2,101 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using BenchmarkDotNet.Configs;
using BenchmarkDotNet.ConsoleArguments;
using BenchmarkDotNet.Exporters;
using BenchmarkDotNet.Extensions;
using BenchmarkDotNet.Loggers;
using BenchmarkDotNet.Running;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using BenchmarkDotNet.Running;
using System.IO;
using BenchmarkDotNet.Extensions;
using BenchmarkDotNet.Configs;
using System.Linq;

namespace MicroBenchmarks
{
class Program
{
static int Main(string[] args)
{
var argsList = new List<string>(args);
int? partitionCount;
int? partitionIndex;
List<string> exclusionFilterValue;
List<string> categoryExclusionFilterValue;
bool getDiffableDisasm;

// Parse and remove any additional parameters that we need that aren't part of BDN
try
{
argsList = CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-count", out partitionCount);
argsList = CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-index", out partitionIndex);
argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--exclusion-filter", out exclusionFilterValue);
argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--category-exclusion-filter", out categoryExclusionFilterValue);
CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--disasm-diff", out getDiffableDisasm);
if (!PerfLabCommandLineOptions.TryParse(args, out var options, out var bdnOnlyArgs))
return 1;

CommandLineOptions.ValidatePartitionParameters(partitionCount, partitionIndex);
var config =
RecommendedConfig.Create(
artifactsPath: new DirectoryInfo(Path.Combine(AppContext.BaseDirectory, "BenchmarkDotNet.Artifacts")),
mandatoryCategories: ImmutableHashSet.Create(Categories.Libraries, Categories.Runtime, Categories.ThirdParty),
options: options)
.AddValidator(new NoWasmValidator(Categories.NoWASM));

if (options.Manifest is BenchmarkManifest manifest)
return RunWithManifest(bdnOnlyArgs, manifest, config);

return BenchmarkSwitcher
.FromAssembly(typeof(Program).Assembly)
.Run(bdnOnlyArgs, config)
.ToExitCode();
}

private static int RunWithManifest(string[] args, BenchmarkManifest manifest, IConfig config)
{
var logger = config.GetLoggers().First();
var (isParsingSuccess, parsedConfig, options) = ConfigParser.Parse(args, logger, config);
if (!isParsingSuccess)
return 1; // ConfigParser.Parse will print the error message

var effectiveConfig = ManualConfig.Union(config, parsedConfig);

var (allTypesValid, allAvailableTypesWithRunnableBenchmarks) = TypeFilter.GetTypesWithRunnableBenchmarks(
types: Enumerable.Empty<Type>(),
assemblies: new [] { typeof(Program).Assembly },
logger);

if (!allTypesValid)
return 1; // TypeFilter.GetTypesWithRunnableBenchmarks will print the error message

if (allAvailableTypesWithRunnableBenchmarks.Count == 0)
{
logger.WriteLineError("No runnable benchmarks found before applying filters.");
return 1;
}
catch (ArgumentException e)

var filteredBenchmarks = TypeFilter.Filter(effectiveConfig, allAvailableTypesWithRunnableBenchmarks);
if (filteredBenchmarks.Length == 0)
{
Console.WriteLine("ArgumentException: {0}", e.Message);
logger.WriteLineError("No runnable benchmarks found after applying filters.");
return 1;
}

return BenchmarkSwitcher
.FromAssembly(typeof(Program).Assembly)
.Run(argsList.ToArray(),
RecommendedConfig.Create(
artifactsPath: new DirectoryInfo(Path.Combine(AppContext.BaseDirectory, "BenchmarkDotNet.Artifacts")),
mandatoryCategories: ImmutableHashSet.Create(Categories.Libraries, Categories.Runtime, Categories.ThirdParty),
partitionCount: partitionCount,
partitionIndex: partitionIndex,
exclusionFilterValue: exclusionFilterValue,
categoryExclusionFilterValue: categoryExclusionFilterValue,
getDiffableDisasm: getDiffableDisasm)
.AddValidator(new NoWasmValidator(Categories.NoWASM)))
.ToExitCode();
if (manifest.BenchmarkCaseRunOverrides is not null)
{
var overriddenBenchmarks = new List<BenchmarkRunInfo>();
foreach (var benchmarkRunInfo in filteredBenchmarks)
{
var updatedCases = new List<BenchmarkCase>(benchmarkRunInfo.BenchmarksCases.Length);
foreach (var benchmarkCase in benchmarkRunInfo.BenchmarksCases)
{
var benchmarkName = FullNameProvider.GetBenchmarkName(benchmarkCase);
if (manifest.BenchmarkCaseRunOverrides.TryGetValue(benchmarkName, out var overrideRunInfo))
{
var updatedJob = overrideRunInfo.ModifyJob(benchmarkCase.Job, benchmarkCase);
var newBenchmarkCase = BenchmarkCase.Create(benchmarkCase.Descriptor, updatedJob, benchmarkCase.Parameters, benchmarkCase.Config);
updatedCases.Add(newBenchmarkCase);
}
else
{
updatedCases.Add(benchmarkCase);
}
}

overriddenBenchmarks.Add(new BenchmarkRunInfo(updatedCases.ToArray(), benchmarkRunInfo.Type, benchmarkRunInfo.Config));
}

filteredBenchmarks = overriddenBenchmarks.ToArray();
}

return BenchmarkRunner.Run(filteredBenchmarks).ToExitCode();
}
}
}
1 change: 1 addition & 0 deletions src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
<PropertyGroup>
<OutputType>Library</OutputType>
<TargetFramework>netstandard2.0</TargetFramework>
<Nullable>enable</Nullable>
</PropertyGroup>
<PropertyGroup Condition="'$(BenchmarkDotNetSources)' != ''">
<_BenchmarkDotNetSourcesN>$([MSBuild]::NormalizeDirectory('$(BenchmarkDotNetSources)'))</_BenchmarkDotNetSourcesN>
Expand Down
Loading
Loading