diff --git a/src/benchmarks/micro/MicroBenchmarks.csproj b/src/benchmarks/micro/MicroBenchmarks.csproj
index cdaafe3ce76..a6772d66017 100644
--- a/src/benchmarks/micro/MicroBenchmarks.csproj
+++ b/src/benchmarks/micro/MicroBenchmarks.csproj
@@ -82,12 +82,12 @@
-
-
+
+
-
+
@@ -96,12 +96,12 @@
-
+
-
+
@@ -149,7 +149,7 @@
-
+
@@ -170,7 +170,7 @@
-
+
@@ -180,7 +180,7 @@
-
+
@@ -190,7 +190,7 @@
-
+
@@ -228,24 +228,24 @@
-
+
-
+
-
+
-
+
diff --git a/src/benchmarks/micro/Program.cs b/src/benchmarks/micro/Program.cs
index 4cce892d064..d1598c701b0 100644
--- a/src/benchmarks/micro/Program.cs
+++ b/src/benchmarks/micro/Program.cs
@@ -2,13 +2,17 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.ConsoleArguments;
+using BenchmarkDotNet.Exporters;
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Running;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
-using BenchmarkDotNet.Running;
using System.IO;
-using BenchmarkDotNet.Extensions;
-using BenchmarkDotNet.Configs;
+using System.Linq;
namespace MicroBenchmarks
{
@@ -16,43 +20,83 @@ class Program
{
static int Main(string[] args)
{
- var argsList = new List(args);
- int? partitionCount;
- int? partitionIndex;
- List exclusionFilterValue;
- List categoryExclusionFilterValue;
- bool getDiffableDisasm;
-
- // Parse and remove any additional parameters that we need that aren't part of BDN
- try
- {
- argsList = CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-count", out partitionCount);
- argsList = CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-index", out partitionIndex);
- argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--exclusion-filter", out exclusionFilterValue);
- argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--category-exclusion-filter", out categoryExclusionFilterValue);
- CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--disasm-diff", out getDiffableDisasm);
+ if (!PerfLabCommandLineOptions.TryParse(args, out var options, out var bdnOnlyArgs))
+ return 1;
- CommandLineOptions.ValidatePartitionParameters(partitionCount, partitionIndex);
+ var config =
+ RecommendedConfig.Create(
+ artifactsPath: new DirectoryInfo(Path.Combine(AppContext.BaseDirectory, "BenchmarkDotNet.Artifacts")),
+ mandatoryCategories: ImmutableHashSet.Create(Categories.Libraries, Categories.Runtime, Categories.ThirdParty),
+ options: options)
+ .AddValidator(new NoWasmValidator(Categories.NoWASM));
+
+ if (options.Manifest is BenchmarkManifest manifest)
+ return RunWithManifest(bdnOnlyArgs, manifest, config);
+
+ return BenchmarkSwitcher
+ .FromAssembly(typeof(Program).Assembly)
+ .Run(bdnOnlyArgs, config)
+ .ToExitCode();
+ }
+
+ private static int RunWithManifest(string[] args, BenchmarkManifest manifest, IConfig config)
+ {
+ var logger = config.GetLoggers().First();
+ var (isParsingSuccess, parsedConfig, options) = ConfigParser.Parse(args, logger, config);
+ if (!isParsingSuccess)
+ return 1; // ConfigParser.Parse will print the error message
+
+ var effectiveConfig = ManualConfig.Union(config, parsedConfig);
+
+ var (allTypesValid, allAvailableTypesWithRunnableBenchmarks) = TypeFilter.GetTypesWithRunnableBenchmarks(
+ types: Enumerable.Empty(),
+ assemblies: new [] { typeof(Program).Assembly },
+ logger);
+
+ if (!allTypesValid)
+ return 1; // TypeFilter.GetTypesWithRunnableBenchmarks will print the error message
+
+ if (allAvailableTypesWithRunnableBenchmarks.Count == 0)
+ {
+ logger.WriteLineError("No runnable benchmarks found before applying filters.");
+ return 1;
}
- catch (ArgumentException e)
+
+ var filteredBenchmarks = TypeFilter.Filter(effectiveConfig, allAvailableTypesWithRunnableBenchmarks);
+ if (filteredBenchmarks.Length == 0)
{
- Console.WriteLine("ArgumentException: {0}", e.Message);
+ logger.WriteLineError("No runnable benchmarks found after applying filters.");
return 1;
}
- return BenchmarkSwitcher
- .FromAssembly(typeof(Program).Assembly)
- .Run(argsList.ToArray(),
- RecommendedConfig.Create(
- artifactsPath: new DirectoryInfo(Path.Combine(AppContext.BaseDirectory, "BenchmarkDotNet.Artifacts")),
- mandatoryCategories: ImmutableHashSet.Create(Categories.Libraries, Categories.Runtime, Categories.ThirdParty),
- partitionCount: partitionCount,
- partitionIndex: partitionIndex,
- exclusionFilterValue: exclusionFilterValue,
- categoryExclusionFilterValue: categoryExclusionFilterValue,
- getDiffableDisasm: getDiffableDisasm)
- .AddValidator(new NoWasmValidator(Categories.NoWASM)))
- .ToExitCode();
+ if (manifest.BenchmarkCaseRunOverrides is not null)
+ {
+ var overriddenBenchmarks = new List();
+ foreach (var benchmarkRunInfo in filteredBenchmarks)
+ {
+ var updatedCases = new List(benchmarkRunInfo.BenchmarksCases.Length);
+ foreach (var benchmarkCase in benchmarkRunInfo.BenchmarksCases)
+ {
+ var benchmarkName = FullNameProvider.GetBenchmarkName(benchmarkCase);
+ if (manifest.BenchmarkCaseRunOverrides.TryGetValue(benchmarkName, out var overrideRunInfo))
+ {
+ var updatedJob = overrideRunInfo.ModifyJob(benchmarkCase.Job, benchmarkCase);
+ var newBenchmarkCase = BenchmarkCase.Create(benchmarkCase.Descriptor, updatedJob, benchmarkCase.Parameters, benchmarkCase.Config);
+ updatedCases.Add(newBenchmarkCase);
+ }
+ else
+ {
+ updatedCases.Add(benchmarkCase);
+ }
+ }
+
+ overriddenBenchmarks.Add(new BenchmarkRunInfo(updatedCases.ToArray(), benchmarkRunInfo.Type, benchmarkRunInfo.Config));
+ }
+
+ filteredBenchmarks = overriddenBenchmarks.ToArray();
+ }
+
+ return BenchmarkRunner.Run(filteredBenchmarks).ToExitCode();
}
}
}
\ No newline at end of file
diff --git a/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj b/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj
index c459f9eb696..9d5f5946f1a 100644
--- a/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj
+++ b/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj
@@ -2,6 +2,7 @@
Library
netstandard2.0
+ enable
<_BenchmarkDotNetSourcesN>$([MSBuild]::NormalizeDirectory('$(BenchmarkDotNetSources)'))
diff --git a/src/harness/BenchmarkDotNet.Extensions/BenchmarkManifest.cs b/src/harness/BenchmarkDotNet.Extensions/BenchmarkManifest.cs
new file mode 100644
index 00000000000..f8ea85b2b24
--- /dev/null
+++ b/src/harness/BenchmarkDotNet.Extensions/BenchmarkManifest.cs
@@ -0,0 +1,545 @@
+using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Environments;
+using BenchmarkDotNet.Exporters;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Toolchains;
+using BenchmarkDotNet.Toolchains.CoreRun;
+using BenchmarkDotNet.Toolchains.CsProj;
+using BenchmarkDotNet.Toolchains.DotNetCli;
+using BenchmarkDotNet.Toolchains.InProcess.Emit;
+using BenchmarkDotNet.Toolchains.InProcess.NoEmit;
+using BenchmarkDotNet.Toolchains.Mono;
+using BenchmarkDotNet.Toolchains.MonoAotLLVM;
+using BenchmarkDotNet.Toolchains.MonoWasm;
+using BenchmarkDotNet.Toolchains.NativeAot;
+using BenchmarkDotNet.Toolchains.Roslyn;
+using Perfolizer.Horology;
+using Perfolizer.Mathematics.OutlierDetection;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
+namespace BenchmarkDotNet.Extensions
+{
+ public class BenchmarkManifest
+ {
+ public List? BenchmarkCases { get; set; }
+ public JobSettings? BaseJob { get; set; }
+ public Dictionary? Jobs { get; set; }
+ public Dictionary? BenchmarkCaseRunOverrides { get; set; }
+
+ public class JobSettings
+ {
+ public EnvironmentSettings? Environment { get; set; }
+ public RunSettings? Run { get; set; }
+ public InfrastructureSettings? Infrastructure { get; set; }
+ public AccuracySettings? Accuracy { get; set; }
+
+ public Job ModifyJob(Job job)
+ {
+ if (Environment is not null)
+ job = Environment.ModifyJob(job);
+ if (Run is not null)
+ job = Run.ModifyJob(job);
+ if (Infrastructure is not null)
+ job = Infrastructure.ModifyJob(job);
+ if (Accuracy is not null)
+ job = Accuracy.ModifyJob(job);
+ return job;
+ }
+ }
+
+ public class EnvironmentSettings
+ {
+ public Platform? Platform { get; set; }
+ public Jit? Jit { get; set; }
+ public RuntimeSettings? Runtime { get; set; }
+ public int? Affinity { get; set; }
+ public GcSettings? Gc { get; set; }
+ public Dictionary? EnvironmentVariables { get; set; }
+ public PowerPlan? PowerPlan { get; set; }
+ public Guid? PowerPlanGuid { get; set; }
+ public bool? LargeAddressAware { get; set; }
+
+ public Job ModifyJob(Job job)
+ {
+ if (Platform is Platform platform)
+ job = job.WithPlatform(platform);
+ if (Jit is Jit jit)
+ job = job.WithJit(jit);
+ if (Runtime is not null)
+ job = Runtime.ModifyJob(job);
+ if (Affinity is int affinity)
+ job = job.WithAffinity((IntPtr)affinity);
+ if (Gc is not null)
+ job = Gc.ModifyJob(job);
+ if (EnvironmentVariables is not null)
+ job = job.WithEnvironmentVariables(EnvironmentVariables.Select(e => new EnvironmentVariable(e.Key, e.Value)).ToArray());
+ if (PowerPlan is PowerPlan powerPlan)
+ job = job.WithPowerPlan(powerPlan);
+ if (PowerPlanGuid is Guid guid)
+ job = job.WithPowerPlan(guid);
+ if (LargeAddressAware is bool)
+ job = job.WithLargeAddressAware(LargeAddressAware.Value);
+ return job;
+ }
+ }
+
+ public class RuntimeSettings
+ {
+ public RuntimeType Type { get; set; }
+ public string? Tfm { get; set; }
+ public string? DisplayName { get; set; } // Only settable for some custom runtimes
+
+ // Clr
+ public string? ClrVersion { get; set; } // Only for a custom .NET Framework version
+
+ // Mono
+ public string? MonoPath { get; set; }
+ public string? AotArgs { get; set; }
+ public string? MonoBclPath { get; set; }
+
+ // MonoAotLLVM
+ public string? AOTCompilerPath { get; set; }
+ public MonoAotCompilerMode? AOTCompilerMode { get; set; }
+
+ // Wasm and WasmAot
+ public string? WasmJavascriptEnginePath { get; set; }
+ public string? WasmJavascriptEngineArguments { get; set; }
+ public string? WasmDataDirectory { get; set; }
+
+ public Job ModifyJob(Job job)
+ {
+ Runtime runtime = Type switch
+ {
+ RuntimeType.Clr => GetClrRuntime(),
+ RuntimeType.Core => GetCoreRuntime(),
+ RuntimeType.Mono => new MonoRuntime(DisplayName ?? "Mono", MonoPath!, AotArgs!, MonoBclPath!),
+ RuntimeType.MonoAotLLVM => GetMonoAotLLVMRuntime(),
+ RuntimeType.NativeAot => GetNativeAotRuntime(),
+ RuntimeType.Wasm => GetWasmRuntime(isAot: false),
+ RuntimeType.WasmAot => GetWasmRuntime(isAot: true),
+ _ => throw new Exception("Runtime type must be specified")
+ };
+
+ return job.WithRuntime(runtime);
+ }
+
+ private ClrRuntime GetClrRuntime()
+ {
+ if (ClrVersion is not null)
+ return ClrRuntime.CreateForLocalFullNetFrameworkBuild(ClrVersion);
+
+ return Tfm switch
+ {
+ "4.6.1" => ClrRuntime.Net461,
+ "4.6.2" => ClrRuntime.Net462,
+ "4.7" => ClrRuntime.Net47,
+ "4.7.1" => ClrRuntime.Net471,
+ "4.7.2" => ClrRuntime.Net472,
+ "4.8" => ClrRuntime.Net48,
+ "4.8.1" => ClrRuntime.Net481,
+ null => throw new Exception("TFM cannot be null for CLR runtime"),
+ _ => throw new Exception($"Unknown TFM '{Tfm}' for CLR runtime"),
+ };
+ }
+
+ private CoreRuntime GetCoreRuntime()
+ {
+ return Tfm switch
+ {
+ // Commented out as they are not supported by BenchmarkDotNet
+ //"netcoreapp2.0" => CoreRuntime.Core20,
+ //"netcoreapp2.1" => CoreRuntime.Core21,
+ //"netcoreapp2.2" => CoreRuntime.Core22,
+ //"netcoreapp3.0" => CoreRuntime.Core30,
+ "netcoreapp3.1" => CoreRuntime.Core31,
+ "net5.0" => CoreRuntime.Core50,
+ "net6.0" => CoreRuntime.Core60,
+ "net7.0" => CoreRuntime.Core70,
+ "net8.0" => CoreRuntime.Core80,
+ "net9.0" => CoreRuntime.Core90,
+ "net10.0" => CoreRuntime.Core10_0,
+ null => throw new Exception("TFM cannot be null for Core runtime"),
+ _ => CoreRuntime.CreateForNewVersion(Tfm, DisplayName ?? Tfm),
+ };
+ }
+
+ private MonoAotLLVMRuntime GetMonoAotLLVMRuntime()
+ {
+ if (AOTCompilerPath is null)
+ throw new Exception("AOTCompilerPath must be set for MonoAotLLVM runtime");
+ FileInfo aotCompilerPath = new FileInfo(AOTCompilerPath);
+ if (AOTCompilerMode is null)
+ throw new Exception("AOTCompilerMode must be set for MonoAotLLVM runtime");
+ if (Tfm is null)
+ throw new Exception("TFM must be set for MonoAotLLVM runtime");
+
+ var moniker = Tfm switch
+ {
+ "net6.0" => RuntimeMoniker.MonoAOTLLVMNet60,
+ "net7.0" => RuntimeMoniker.MonoAOTLLVMNet70,
+ "net8.0" => RuntimeMoniker.MonoAOTLLVMNet80,
+ "net9.0" => RuntimeMoniker.MonoAOTLLVMNet90,
+ "net10.0" => RuntimeMoniker.MonoAOTLLVMNet10_0,
+ _ => RuntimeMoniker.MonoAOTLLVM,
+ };
+
+ return new MonoAotLLVMRuntime(aotCompilerPath, AOTCompilerMode.Value, Tfm!, DisplayName ?? "MonoAotLLVM", moniker);
+ }
+
+ private NativeAotRuntime GetNativeAotRuntime()
+ {
+ return Tfm switch
+ {
+ "net6.0" => NativeAotRuntime.Net60,
+ "net7.0" => NativeAotRuntime.Net70,
+ "net8.0" => NativeAotRuntime.Net80,
+ "net9.0" => NativeAotRuntime.Net90,
+ "net10.0" => NativeAotRuntime.Net10_0,
+ _ => throw new Exception($"Unsupported TFM '{Tfm}' for NativeAot runtime"),
+ };
+ }
+
+ private WasmRuntime GetWasmRuntime(bool isAot)
+ {
+ if (Tfm is null)
+ throw new Exception("TFM must be set for Wasm runtime");
+
+ var moniker = Tfm switch
+ {
+ "net5.0" => RuntimeMoniker.WasmNet50,
+ "net6.0" => RuntimeMoniker.WasmNet60,
+ "net7.0" => RuntimeMoniker.WasmNet70,
+ "net8.0" => RuntimeMoniker.WasmNet80,
+ "net9.0" => RuntimeMoniker.WasmNet90,
+ "net10.0" => RuntimeMoniker.WasmNet10_0,
+ _ => RuntimeMoniker.Wasm,
+ };
+
+ return new WasmRuntime(
+ Tfm,
+ DisplayName ?? "Wasm",
+ WasmJavascriptEnginePath ?? "v8",
+ WasmJavascriptEngineArguments ?? "--expose_wasm",
+ isAot,
+ WasmDataDirectory,
+ moniker
+ );
+ }
+ }
+
+ public enum RuntimeType
+ {
+ Clr,
+ Core,
+ Mono, // For MonoVM (e.g. built from .NET runtime, use Core)
+ MonoAotLLVM,
+ NativeAot,
+ Wasm,
+ WasmAot
+ }
+
+ public class GcSettings
+ {
+ public bool? Server { get; set; }
+ public bool? Concurrent { get; set; }
+ public bool? CpuGroups { get; set; }
+ public bool? Force { get; set; }
+ public bool? AllowVeryLargeObjects { get; set; }
+ public bool? RetainVm { get; set; }
+ public bool? NoAffinitize { get; set; }
+ public int? HeapAffinitizeMask { get; set; }
+ public int? HeapCount { get; set; }
+
+ public Job ModifyJob(Job job)
+ {
+ if (Server is bool server)
+ job = job.WithGcServer(server);
+ if (Concurrent is bool concurrent)
+ job = job.WithGcConcurrent(concurrent);
+ if (CpuGroups is bool cpuGroups)
+ job = job.WithGcCpuGroups(cpuGroups);
+ if (Force is bool force)
+ job = job.WithGcForce(force);
+ if (AllowVeryLargeObjects is bool allowVeryLargeObjects)
+ job = job.WithGcAllowVeryLargeObjects(allowVeryLargeObjects);
+ if (RetainVm is bool retainVm)
+ job = job.WithGcRetainVm(retainVm);
+ if (NoAffinitize is bool noAffinitize)
+ job = job.WithNoAffinitize(noAffinitize);
+ if (HeapAffinitizeMask is int heapAffinitizeMask)
+ job = job.WithHeapAffinitizeMask(heapAffinitizeMask);
+ if (HeapCount is int heapCount)
+ job = job.WithHeapCount(heapCount);
+ return job;
+ }
+ }
+
+ public class RunSettings
+ {
+ public RunStrategy? RunStrategy { get; set; }
+ public int? LaunchCount { get; set; }
+ public int? WarmupCount { get; set; }
+ public int? IterationCount { get; set; }
+ public double? IterationTimeMilliseconds { get; set; }
+ public long? InvocationCount { get; set; }
+ public long? OperationCount { get; set; } // InvocationCount == OperationCount / OperationsPerInvoke
+ public int? UnrollFactor { get; set; }
+ public int? MinIterationCount { get; set; }
+ public int? MaxIterationCount { get; set; }
+ public int? MinWarmupIterationCount { get; set; }
+ public int? MaxWarmupIterationCount { get; set; }
+ public bool? MemoryRandomization { get; set; }
+
+ public Job ModifyJob(Job job, BenchmarkCase? benchmark = null)
+ {
+ if (RunStrategy is RunStrategy runStrategy)
+ job = job.WithStrategy(runStrategy);
+ if (LaunchCount is int launchCount)
+ job = job.WithLaunchCount(launchCount);
+ if (WarmupCount is int warmupCount)
+ job = job.WithWarmupCount(warmupCount);
+ if (IterationCount is int iterationCount)
+ job = job.WithIterationCount(iterationCount);
+ if (IterationTimeMilliseconds is double iterationTime)
+ job = job.WithIterationTime(TimeInterval.FromMilliseconds(iterationTime));
+ if (InvocationCount is long && benchmark is null)
+ throw new Exception("InvocationCount can only be set per benchmark");
+ if (OperationCount is long && benchmark is null)
+ throw new Exception("OperationCount can only be set per benchmark");
+ if (UnrollFactor is int unrollFactor)
+ job = job.WithUnrollFactor(unrollFactor);
+ if (MinIterationCount is int minIterationCount)
+ job = job.WithMinIterationCount(minIterationCount);
+ if (MaxIterationCount is int maxIterationCount)
+ job = job.WithMaxIterationCount(maxIterationCount);
+ if (MinWarmupIterationCount is int minWarmupIterationCount)
+ job = job.WithMinWarmupCount(minWarmupIterationCount);
+ if (MaxWarmupIterationCount is int maxWarmupIterationCount)
+ job = job.WithMaxWarmupCount(maxWarmupIterationCount);
+ if (MemoryRandomization is bool memoryRandomization)
+ job = job.WithMemoryRandomization(memoryRandomization);
+
+ if (benchmark is not null)
+ {
+ var benchmarkName = FullNameProvider.GetBenchmarkName(benchmark);
+ long? invocationCount = InvocationCount;
+ if (invocationCount is null && OperationCount is long operationCount)
+ {
+ var operationsPerInvoke = benchmark.Descriptor.OperationsPerInvoke;
+ if (operationsPerInvoke % operationsPerInvoke != 0)
+ throw new Exception($"OperationCount ({operationCount}) must be divisible by OperationsPerInvoke ({operationsPerInvoke}) for benchmark '{benchmarkName}'");
+
+ invocationCount = operationCount / operationsPerInvoke;
+ }
+
+ if (invocationCount is not null)
+ {
+ if (benchmark.Descriptor.IterationSetupMethod is not null || benchmark.Descriptor.IterationCleanupMethod is not null)
+ throw new Exception($"OperationCount or InvocationCount cannot be set for benchmark '{benchmarkName}' as it has iteration setup or cleanup methods.");
+
+ if (UnrollFactor is not null)
+ unrollFactor = UnrollFactor.Value;
+ else if (job.HasValue(RunMode.UnrollFactorCharacteristic))
+ unrollFactor = job.Run.UnrollFactor;
+ else if (invocationCount < 64) // This is a deviation from base BDN which uses unroll factor of 16 if invocation count > 16
+ unrollFactor = 1;
+ else
+ unrollFactor = 16;
+
+ if (invocationCount % unrollFactor != 0)
+ throw new Exception($"InvocationCount ({invocationCount}) must be divisible by UnrollFactor ({unrollFactor}) for benchmark '{benchmarkName}'");
+
+ job = job.WithInvocationCount(invocationCount.Value).WithUnrollFactor(unrollFactor);
+ }
+ }
+
+ return job;
+ }
+ }
+
+ public class InfrastructureSettings
+ {
+ public ToolchainSettings? Toolchain { get; set; }
+ public string? BuildConfiguration { get; set; }
+ public List? MonoArguments { get; set; }
+ public List? MsBuildArguments { get; set; }
+
+ public Job ModifyJob(Job job)
+ {
+ if (Toolchain is not null)
+ job = Toolchain.ModifyJob(job);
+ if (BuildConfiguration is string buildConfiguration)
+ job = job.WithCustomBuildConfiguration(buildConfiguration);
+ var arguments = new List();
+ if (MonoArguments is not null && MonoArguments.Count > 0)
+ arguments.AddRange(MonoArguments.Select(arg => new MonoArgument(arg)));
+ if (MsBuildArguments is not null && MsBuildArguments.Count > 0)
+ arguments.AddRange(MsBuildArguments.Select(arg => new MsBuildArgument(arg)));
+ if (arguments.Count > 0)
+ job = job.WithArguments(arguments.ToArray());
+ return job;
+ }
+ }
+
+ public class ToolchainSettings
+ {
+ public ToolchainType? Type { get; set; }
+ public string? DisplayName { get; set; }
+ public string? Tfm { get; set; } // TODO: Can we reuse the Tfm from RuntimeSettings?
+ public string? CliPath { get; set; }
+ public string? RestorePath { get; set; }
+
+ // CoreRun
+ public string? CoreRunPath { get; set; }
+
+ // CsProjCore, Mono, NativeAot
+ public string? RuntimeFrameworkVersion { get; set; }
+
+ // InProcessEmit and InProcessNoEmit
+ public bool? InProcessLogOutput { get; set; }
+ public double? InProcessTimeoutSeconds { get; set; }
+
+ // MonoAotLLVM, Wasm
+ public string? CustomRuntimePack { get; set; }
+ public string? AOTCompilerPath { get; set; } // TODO: Can we reuse the AOTCompilerPath from RuntimeSettings?
+ public MonoAotCompilerMode? AOTCompilerMode { get; set; } // TODO: Can we reuse the AOTCompilerMode from RuntimeSettings?
+
+ // NativeAot
+ public string? IlcPackagesDirectory { get; set; }
+ public string? ILCompilerVersion { get; set; }
+ public string? NativeAotNugetFeed { get; set; }
+ public bool? RootAllApplicationAssemblies { get; set; }
+ public bool? IlcGenerateCompleteTypeMetadata { get; set; }
+ public bool? IlcGenerateStackTraceData { get; set; }
+ public string? IlcOptimizationPreference { get; set; } // "Size" or "Speed"
+ public string? IlcInstructionSet { get; set; }
+
+ public Job ModifyJob(Job job)
+ {
+ var tfm = Tfm ?? job.Environment.Runtime?.MsBuildMoniker ?? CoreRuntime.Latest.MsBuildMoniker;
+
+ var netCoreAppSettings = new NetCoreAppSettings(
+ targetFrameworkMoniker: tfm,
+ runtimeFrameworkVersion: RuntimeFrameworkVersion,
+ name: DisplayName ?? job.Environment.Runtime?.Name ?? tfm,
+ customDotNetCliPath: CliPath,
+ packagesPath: RestorePath,
+ customRuntimePack: CustomRuntimePack,
+ aotCompilerPath: AOTCompilerPath,
+ aotCompilerMode: AOTCompilerMode ?? MonoAotCompilerMode.mini);
+
+ IToolchain toolchain = Type switch
+ {
+ ToolchainType.CoreRun => new CoreRunToolchain(
+ new FileInfo(CoreRunPath ?? throw new Exception("CoreRunPath must be set for CoreRun toolchain")),
+ createCopy: true,
+ targetFrameworkMoniker: tfm,
+ customDotNetCliPath: CliPath is not null ? new FileInfo(CliPath) : null,
+ restorePath: RestorePath is not null ? new DirectoryInfo(RestorePath) : null,
+ displayName: DisplayName ?? "CoreRun"),
+ ToolchainType.CsProjClassicNet => CsProjClassicNetToolchain.From(tfm, RestorePath, CliPath),
+ ToolchainType.CsProjCore => CsProjCoreToolchain.From(netCoreAppSettings),
+ ToolchainType.InProcessEmit => new InProcessEmitToolchain(
+ InProcessTimeoutSeconds is null ? TimeSpan.Zero : TimeSpan.FromSeconds(InProcessTimeoutSeconds.Value),
+ InProcessLogOutput ?? true),
+ ToolchainType.InProcessNoEmit => new InProcessNoEmitToolchain(
+ InProcessTimeoutSeconds is null ? TimeSpan.Zero : TimeSpan.FromSeconds(InProcessTimeoutSeconds.Value),
+ InProcessLogOutput ?? true),
+ ToolchainType.Mono => MonoToolchain.From(netCoreAppSettings),
+ ToolchainType.MonoAot => MonoAotToolchain.Instance,
+ ToolchainType.MonoAotLLVM => MonoAotLLVMToolChain.From(netCoreAppSettings),
+ ToolchainType.NativeAot => GetNativeAotToolchain(tfm),
+ ToolchainType.Roslyn => RoslynToolchain.Instance,
+ ToolchainType.Wasm => WasmToolchain.From(netCoreAppSettings),
+ _ => throw new ArgumentException("Toolchain type must be specified")
+ };
+
+ return job.WithToolchain(toolchain);
+ }
+
+ private IToolchain GetNativeAotToolchain(string tfm)
+ {
+ var builder = NativeAotToolchain.CreateBuilder();
+
+ builder.TargetFrameworkMoniker(tfm);
+
+ if (CliPath is not null)
+ builder.DotNetCli(CliPath);
+
+ if (RestorePath is not null)
+ builder.PackagesRestorePath(RestorePath);
+
+ if (IlcPackagesDirectory is not null)
+ builder.UseLocalBuild(new DirectoryInfo(IlcPackagesDirectory));
+ else if (ILCompilerVersion is not null)
+ builder.UseNuGet(ILCompilerVersion ?? "", NativeAotNugetFeed ?? "https://api.nuget.org/v3/index.json");
+
+ if (RootAllApplicationAssemblies is bool rootAllApplicationAssemblies)
+ builder.RootAllApplicationAssemblies(rootAllApplicationAssemblies);
+
+ if (IlcGenerateCompleteTypeMetadata is bool generateCompleteTypeMetadata)
+ builder.IlcGenerateCompleteTypeMetadata(generateCompleteTypeMetadata);
+
+ if (IlcGenerateStackTraceData is bool generateStackTraceData)
+ builder.IlcGenerateStackTraceData(generateStackTraceData);
+
+ if (IlcOptimizationPreference is string optimizationPreference)
+ builder.IlcOptimizationPreference(optimizationPreference);
+
+ if (IlcInstructionSet is string instructionSet)
+ builder.IlcInstructionSet(instructionSet);
+
+ return builder.ToToolchain();
+ }
+ }
+
+ public enum ToolchainType
+ {
+ CoreRun,
+ CsProjClassicNet,
+ CsProjCore,
+ InProcessEmit,
+ InProcessNoEmit,
+ Mono,
+ MonoAot,
+ MonoAotLLVM,
+ NativeAot,
+ Roslyn,
+ Wasm,
+ }
+
+ public class AccuracySettings
+ {
+ public double? MaxRelativeError { get; set; }
+ public double? MaxAbsoluteErrorNanoseconds { get; set; }
+ public double? MinIterationTimeMilliseconds { get; set; }
+ public int? MinInvokeCount { get; set; }
+ public bool? EvaluateOverhead { get; set; }
+ public OutlierMode? OutlierMode { get; set; }
+ public bool? AnalyzeLaunchVariance { get; set; }
+
+ public Job ModifyJob(Job job)
+ {
+ if (MaxRelativeError is double maxRelativeError)
+ job = job.WithMaxRelativeError(maxRelativeError);
+ if (MaxAbsoluteErrorNanoseconds is double maxAbsoluteErrorNanoseconds)
+ job = job.WithMaxAbsoluteError(TimeInterval.FromNanoseconds(maxAbsoluteErrorNanoseconds));
+ if (MinIterationTimeMilliseconds is double minIterationTime)
+ job = job.WithMinIterationTime(TimeInterval.FromMilliseconds(minIterationTime));
+ if (MinInvokeCount is int minInvokeCount)
+ job = job.WithMinInvokeCount(minInvokeCount);
+ if (EvaluateOverhead is bool evaluateOverhead)
+ job = job.WithEvaluateOverhead(evaluateOverhead);
+ if (OutlierMode is OutlierMode outlierMode)
+ job = job.WithOutlierMode(outlierMode);
+ if (AnalyzeLaunchVariance is bool analyzeLaunchVariance)
+ job = job.WithAnalyzeLaunchVariance(analyzeLaunchVariance);
+ return job;
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/harness/BenchmarkDotNet.Extensions/CommandLineOptions.cs b/src/harness/BenchmarkDotNet.Extensions/CommandLineOptions.cs
deleted file mode 100644
index 3c8b343fc57..00000000000
--- a/src/harness/BenchmarkDotNet.Extensions/CommandLineOptions.cs
+++ /dev/null
@@ -1,97 +0,0 @@
-using System;
-using System.Collections.Generic;
-
-namespace BenchmarkDotNet.Extensions
-{
- public class CommandLineOptions
- {
- // Find and parse given parameter with expected int value, then remove it and its value from the list of arguments to then pass to BenchmarkDotNet
- // Throws ArgumentException if the parameter does not have a value or that value is not parsable as an int
- public static List ParseAndRemoveIntParameter(List argsList, string parameter, out int? parameterValue)
- {
- int parameterIndex = argsList.IndexOf(parameter);
- parameterValue = null;
-
- if (parameterIndex != -1)
- {
- if (parameterIndex + 1 < argsList.Count && Int32.TryParse(argsList[parameterIndex+1], out int parsedParameterValue))
- {
- // remove --partition-count args
- parameterValue = parsedParameterValue;
- argsList.RemoveAt(parameterIndex+1);
- argsList.RemoveAt(parameterIndex);
- }
- else
- {
- throw new ArgumentException(String.Format("{0} must be followed by an integer", parameter));
- }
- }
-
- return argsList;
- }
-
- public static List ParseAndRemoveStringsParameter(List argsList, string parameter, out List parameterValue)
- {
- int parameterIndex = argsList.IndexOf(parameter);
- parameterValue = new List();
-
- if (parameterIndex + 1 < argsList.Count)
- {
- while (parameterIndex + 1 < argsList.Count && !argsList[parameterIndex + 1].StartsWith("-"))
- {
- // remove each filter string and stop when we get to the next argument flag
- parameterValue.Add(argsList[parameterIndex + 1]);
- argsList.RemoveAt(parameterIndex + 1);
- }
- }
- //We only want to remove the --exclusion-filter if it exists
- if (parameterIndex != -1)
- {
- argsList.RemoveAt(parameterIndex);
- }
-
- return argsList;
- }
-
- public static void ParseAndRemoveBooleanParameter(List argsList, string parameter, out bool parameterValue)
- {
- int parameterIndex = argsList.IndexOf(parameter);
-
- if (parameterIndex != -1)
- {
- argsList.RemoveAt(parameterIndex);
-
- parameterValue = true;
- }
- else
- {
- parameterValue = false;
- }
- }
-
- public static void ValidatePartitionParameters(int? count, int? index)
- {
- // Either count and index must both be specified or neither specified
- if (!(count.HasValue == index.HasValue))
- {
- throw new ArgumentException("If either --partition-count or --partition-index is specified, both must be specified");
- }
- // Check values of count and index parameters
- else if (count.HasValue && index.HasValue)
- {
- if (count < 2)
- {
- throw new ArgumentException("When specified, value of --partition-count must be greater than 1");
- }
- else if (!(index < count))
- {
- throw new ArgumentException("Value of --partition-index must be less than --partition-count");
- }
- else if (index < 0)
- {
- throw new ArgumentException("Value of --partition-index must be greater than or equal to 0");
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/src/harness/BenchmarkDotNet.Extensions/ExclusionFilter.cs b/src/harness/BenchmarkDotNet.Extensions/ExclusionFilter.cs
index b3ee453123f..542873a4cba 100644
--- a/src/harness/BenchmarkDotNet.Extensions/ExclusionFilter.cs
+++ b/src/harness/BenchmarkDotNet.Extensions/ExclusionFilter.cs
@@ -8,7 +8,7 @@ namespace BenchmarkDotNet.Extensions
{
class ExclusionFilter : IFilter
{
- private readonly GlobFilter globFilter;
+ private readonly GlobFilter? globFilter;
public ExclusionFilter(List _filter)
{
@@ -30,7 +30,7 @@ public bool Predicate(BenchmarkCase benchmarkCase)
class CategoryExclusionFilter : IFilter
{
- private readonly AnyCategoriesFilter filter;
+ private readonly AnyCategoriesFilter? filter;
public CategoryExclusionFilter(List patterns)
{
diff --git a/src/harness/BenchmarkDotNet.Extensions/PerfLabCommandLineOptions.cs b/src/harness/BenchmarkDotNet.Extensions/PerfLabCommandLineOptions.cs
new file mode 100644
index 00000000000..47fbd370882
--- /dev/null
+++ b/src/harness/BenchmarkDotNet.Extensions/PerfLabCommandLineOptions.cs
@@ -0,0 +1,128 @@
+using BenchmarkDotNet.ConsoleArguments;
+using CommandLine;
+using Newtonsoft.Json;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+
+namespace BenchmarkDotNet.Extensions
+{
+ public class PerfLabCommandLineOptions : CommandLineOptions
+ {
+ [Option("partition-count", Required = false, HelpText = "Number of partitions that the run has been split into")]
+ public int? PartitionCount { get; set; }
+
+ [Option("partition-index", Required = false, HelpText = "Index of the partition that this run is for (0-based)")]
+ public int? PartitionIndex { get; set; }
+
+ [Option("exclusion-filter", Required = false, HelpText = "Glob patterns to exclude from being run")]
+ public IEnumerable? ExclusionFilters { get; set; }
+
+ [Option("category-exclusion-filter", Required = false, HelpText = "Categories to exclude from being run")]
+ public IEnumerable? CategoryExclusionFilters { get; set; }
+
+ [Option("disasm-diff", Required = false, Default = false, HelpText = "Enable diffable disassembly output")]
+ public bool GetDiffableDisasm { get; set; }
+
+ [Option("manifest", Required = false, HelpText = "Path to the json manifest file that contains the list of benchmarks to run")]
+ public FileInfo? ManifestFile { get; set; }
+
+ public BenchmarkManifest? Manifest { get; set; }
+
+ public static bool TryParse(string[] args, out PerfLabCommandLineOptions? options, out string[]? bdnOnlyArgs)
+ {
+ using var parser = new Parser(settings =>
+ {
+ settings.CaseInsensitiveEnumValues = true;
+ settings.CaseSensitive = false;
+ settings.EnableDashDash = true;
+ settings.IgnoreUnknownArguments = false;
+ });
+
+ var result = parser.ParseArguments(args);
+ if (result is not Parsed parsed || !ValidateOptions(parsed.Value))
+ {
+ options = null;
+ bdnOnlyArgs = null;
+ return false;
+ }
+
+ options = parsed.Value;
+
+ // Get all custom options using reflection
+ var customArgs = typeof(PerfLabCommandLineOptions)
+ .GetProperties(BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly)
+ .Select(p => p.GetCustomAttribute()?.LongName)
+ .Where(p => p is not null)
+ .ToList();
+
+ var bdnArgs = new List();
+ bool inCustomArg = false;
+ foreach (var arg in args)
+ {
+ if (inCustomArg && arg.StartsWith("-"))
+ inCustomArg = false;
+
+ if (!inCustomArg)
+ inCustomArg = customArgs.Any(customArgs => arg.StartsWith($"--{customArgs}"));
+
+ if (!inCustomArg)
+ bdnArgs.Add(arg);
+ }
+
+ bdnOnlyArgs = bdnArgs.ToArray();
+ return true;
+ }
+
+ public static bool ValidateOptions(PerfLabCommandLineOptions options)
+ {
+ if (options.PartitionIndex is int index)
+ {
+ if (options.PartitionCount is not int count)
+ {
+ Console.Error.WriteLine("If --partition-index is specified, --partition-count must also be specified");
+ return false;
+ }
+
+ if (count < 2)
+ {
+ Console.Error.WriteLine("When specified, value of --partition-count must be greater than 1");
+ return false;
+ }
+ else if (index >= count)
+ {
+ Console.Error.WriteLine("Value of --partition-index must be less than --partition-count");
+ return false;
+ }
+ else if (index < 0)
+ {
+ Console.Error.WriteLine("Value of --partition-index must be greater than or equal to 0");
+ return false;
+ }
+ }
+ else if (options.PartitionCount is int)
+ {
+ Console.Error.WriteLine("If --partition-count is specified, --partition-index must also be specified");
+ return false;
+ }
+
+ if (options.ManifestFile is FileInfo manifest)
+ {
+ try
+ {
+ var fileContent = File.ReadAllText(manifest.FullName);
+ options.Manifest = JsonConvert.DeserializeObject(fileContent)!;
+ }
+ catch (Exception ex)
+ {
+ Console.Error.WriteLine($"Failed to read manifest file '{manifest.FullName}': {ex.Message}");
+ return false;
+ }
+ }
+
+ return true;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs b/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs
index ce34e0983bd..417a95ee6a5 100644
--- a/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs
+++ b/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs
@@ -26,8 +26,8 @@ public override void ExportToLog(Summary summary, ILogger logger)
var hasCriticalErrors = summary.HasCriticalValidationErrors;
- DisassemblyDiagnoser disassemblyDiagnoser = summary.Reports
- .FirstOrDefault()? // dissasembler was either enabled for all or none of them (so we use the first one)
+ DisassemblyDiagnoser? disassemblyDiagnoser = summary.Reports
+ .FirstOrDefault()? // disassembler was either enabled for all or none of them (so we use the first one)
.BenchmarkCase.Config.GetDiagnosers().OfType().FirstOrDefault();
foreach (var report in summary.Reports)
diff --git a/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs b/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs
index 8486edf2d1c..4f704e9ab07 100644
--- a/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs
+++ b/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs
@@ -23,23 +23,9 @@ public static class RecommendedConfig
public static IConfig Create(
DirectoryInfo artifactsPath,
ImmutableHashSet mandatoryCategories,
- int? partitionCount = null,
- int? partitionIndex = null,
- List exclusionFilterValue = null,
- List categoryExclusionFilterValue = null,
- Job job = null,
- bool getDiffableDisasm = false)
+ PerfLabCommandLineOptions? options = null,
+ Job? baseJob = null)
{
- if (job is null)
- {
- job = Job.Default
- .WithWarmupCount(1) // 1 warmup is enough for our purpose
- .WithIterationTime(TimeInterval.FromMilliseconds(250)) // the default is 0.5s per iteration, which is slighlty too much for us
- .WithMinIterationCount(15)
- .WithMaxIterationCount(20) // we don't want to run more that 20 iterations
- .DontEnforcePowerPlan(); // make sure BDN does not try to enforce High Performance power plan on Windows
- }
-
var config = ManualConfig.CreateEmpty()
.WithBuildTimeout(TimeSpan.FromMinutes(15)) // for slow machines
.AddLogger(ConsoleLogger.Default) // log output to console
@@ -47,12 +33,8 @@ public static IConfig Create(
.AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray()) // copy default analysers
.AddExporter(MarkdownExporter.GitHub) // export to GitHub markdown
.AddColumnProvider(DefaultColumnProviders.Instance) // display default columns (method name, args etc)
- .AddJob(job.AsDefault()) // tell BDN that this are our default settings
.WithArtifactsPath(artifactsPath.FullName)
.AddDiagnoser(MemoryDiagnoser.Default) // MemoryDiagnoser is enabled by default
- .AddFilter(new PartitionFilter(partitionCount, partitionIndex))
- .AddFilter(new ExclusionFilter(exclusionFilterValue))
- .AddFilter(new CategoryExclusionFilter(categoryExclusionFilterValue))
.AddExporter(JsonExporter.Full) // make sure we export to Json
.AddColumn(StatisticColumn.Median, StatisticColumn.Min, StatisticColumn.Max)
.AddValidator(new MandatoryCategoryValidator(mandatoryCategories))
@@ -60,14 +42,52 @@ public static IConfig Create(
.AddValidator(new UniqueArgumentsValidator()) // don't allow for duplicated arguments #404
.WithSummaryStyle(SummaryStyle.Default.WithMaxParameterColumnWidth(36)); // the default is 20 and trims too aggressively some benchmark results
- if (Environment.IsLabEnvironment())
+ if (baseJob is null && options?.Manifest is not null)
{
- config = config.AddExporter(new PerfLabExporter());
+ if (options?.Manifest?.BaseJob is not null)
+ baseJob = options.Manifest.BaseJob.ModifyJob(Job.Default);
}
+ else
+ {
+ baseJob ??= Job.Default
+ .WithWarmupCount(1) // 1 warmup is enough for our purpose
+ .WithIterationTime(TimeInterval.FromMilliseconds(250)) // the default is 0.5s per iteration, which is slightly too much for us
+ .WithMinIterationCount(15)
+ .WithMaxIterationCount(20) // we don't want to run more that 20 iterations
+ .DontEnforcePowerPlan(); // make sure BDN does not try to enforce High Performance power plan on Windows
+ }
+
+ if (baseJob is not null)
+ config.AddJob(baseJob.AsDefault());
- if (getDiffableDisasm)
+ if (Environment.IsLabEnvironment())
+ config.AddExporter(new PerfLabExporter());
+
+ if (options is not null)
{
- config = config.AddDiagnoser(CreateDisassembler());
+ if (options.ExclusionFilters is IEnumerable exclusionFilters)
+ config.AddFilter(new ExclusionFilter([.. exclusionFilters]));
+
+ if (options.CategoryExclusionFilters is IEnumerable categoryExclusionFilters)
+ config.AddFilter(new CategoryExclusionFilter([.. categoryExclusionFilters]));
+
+ if (options.PartitionCount is int partitionCount && options.PartitionIndex is int partitionIndex)
+ config.AddFilter(new PartitionFilter(partitionCount, partitionIndex));
+
+ if (options.GetDiffableDisasm)
+ config.AddDiagnoser(CreateDisassembler());
+
+ if (options.Manifest?.BenchmarkCases is List benchmarkCases)
+ config.AddFilter(new TestNameFilter([.. benchmarkCases]));
+
+ if (options.Manifest?.Jobs is Dictionary jobs)
+ {
+ var jobsToAdd = jobs
+ .Select(job => job.Value.ModifyJob(Job.Default).WithId(job.Key))
+ .ToArray();
+
+ config.AddJob(jobsToAdd);
+ }
}
return config;
diff --git a/src/harness/BenchmarkDotNet.Extensions/TestNameFilter.cs b/src/harness/BenchmarkDotNet.Extensions/TestNameFilter.cs
new file mode 100644
index 00000000000..fb9f3348ff5
--- /dev/null
+++ b/src/harness/BenchmarkDotNet.Extensions/TestNameFilter.cs
@@ -0,0 +1,27 @@
+using BenchmarkDotNet.Exporters;
+using BenchmarkDotNet.Filters;
+using BenchmarkDotNet.Running;
+using System.Collections.Generic;
+
+namespace BenchmarkDotNet.Extensions
+{
+ // Includes only benchmarks whose FullName (via FullNameProvider) exists in provided hash set
+ public class TestNameFilter : IFilter
+ {
+ private readonly HashSet _allowedNames; // empty or null => disabled
+
+ public TestNameFilter(HashSet allowedNames)
+ {
+ _allowedNames = allowedNames;
+ }
+
+ public bool Predicate(BenchmarkCase benchmarkCase)
+ {
+ if (_allowedNames == null || _allowedNames.Count == 0)
+ return true; // disabled
+
+ var fullName = FullNameProvider.GetBenchmarkName(benchmarkCase);
+ return _allowedNames.Contains(fullName);
+ }
+ }
+}
diff --git a/src/harness/BenchmarkDotNet.Extensions/ValuesGenerator.cs b/src/harness/BenchmarkDotNet.Extensions/ValuesGenerator.cs
index 25695ecbab1..107c298e93f 100644
--- a/src/harness/BenchmarkDotNet.Extensions/ValuesGenerator.cs
+++ b/src/harness/BenchmarkDotNet.Extensions/ValuesGenerator.cs
@@ -23,9 +23,9 @@ public static class ValuesGenerator
public static T GetNonDefaultValue()
{
if (typeof(T) == typeof(byte) || typeof(T) == typeof(sbyte)) // we can't use ArrayOfUniqueValues for byte/sbyte (but they have the same range)
- return Array(byte.MaxValue).First(value => !value.Equals(default(T)));
+ return Array(byte.MaxValue).First(value => !value!.Equals(default(T)));
else
- return ArrayOfUniqueValues(2).First(value => !value.Equals(default(T)));
+ return ArrayOfUniqueValues(2).First(value => !value!.Equals(default(T)));
}
///
diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj
index e62703595ca..4c0eae25121 100644
--- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj
+++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj
@@ -4,7 +4,7 @@
$(PERFLAB_TARGET_FRAMEWORKS)
- net7.0
+ net10.0
false
diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/CommandLineOptionsTests.cs b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/CommandLineOptionsTests.cs
deleted file mode 100644
index 4fbe6a7a10e..00000000000
--- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/CommandLineOptionsTests.cs
+++ /dev/null
@@ -1,132 +0,0 @@
-// Licensed to the .NET Foundation under one or more agreements.
-// The .NET Foundation licenses this file to you under the MIT license.
-// See the LICENSE file in the project root for more information.
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using BenchmarkDotNet.Extensions;
-using Xunit;
-
-namespace Tests
-{
- public class CommandLineOptionsTests
- {
- [Fact]
- public void ArgsListContainsPartitionsCountAndIndex()
- {
- List argsList = new List {
- "--partition-count",
- "10",
- "--partition-index",
- "0"
- };
-
- int? count;
- int? index;
- CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-count", out count);
- CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-index", out index);
-
- CommandLineOptions.ValidatePartitionParameters(count, index);
-
- Assert.Equal(10, count);
- Assert.Equal(0, index);
- }
- [Fact]
- public void ArgsListContainsNeitherPartitionsCountAndIndex()
- {
- List argsList = new List {};
-
- int? count;
- int? index;
- CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-count", out count);
- CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-index", out index);
-
- CommandLineOptions.ValidatePartitionParameters(count, index);
-
- Assert.Null(count);
- Assert.Null(index);
- }
- [Fact]
- public void ArgsListContainsPartitionsCount()
- {
- List argsList = new List {
- "--partition-count",
- "10"
- };
-
- int? count;
- int? index;
- CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-count", out count);
- CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-index", out index);
-
- Assert.Throws(() => CommandLineOptions.ValidatePartitionParameters(count, index));
- }
- [Fact]
- public void ArgsListContainsPartitionsIndex()
- {
- List argsList = new List {
- "--partition-index",
- "10"
- };
-
- int? count;
- int? index;
- CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-count", out count);
- CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-index", out index);
-
- Assert.Throws(() => CommandLineOptions.ValidatePartitionParameters(count, index));
- }
- [Fact]
- public void BadPartitionCountValue()
- {
- List argsList = new List {
- "--partition-count",
- "0",
- "--partition-index",
- "0"
- };
-
- int? count;
- int? index;
- CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-count", out count);
- CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-index", out index);
-
- Assert.Throws(() => CommandLineOptions.ValidatePartitionParameters(count, index));
- }
- [Fact]
- public void BadPartitionIndexValue()
- {
- List argsList = new List {
- "--partition-count",
- "10",
- "--partition-index",
- "-1"
- };
-
- int? count;
- int? index;
- CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-count", out count);
- CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-index", out index);
-
- Assert.Throws(() => CommandLineOptions.ValidatePartitionParameters(count, index));
- }
- [Fact]
- public void PartitionIndexValueGreaterThanCount()
- {
- List argsList = new List {
- "--partition-count",
- "10",
- "--partition-index",
- "11"
- };
-
- int? count;
- int? index;
- CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-count", out count);
- CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-index", out index);
-
- Assert.Throws(() => CommandLineOptions.ValidatePartitionParameters(count, index));
- }
- }
-}