From e03e9caa26d409f9658e373b4f9cc6a8d1c89484 Mon Sep 17 00:00:00 2001 From: Adrian Codrington Date: Mon, 2 Oct 2023 13:16:23 +0800 Subject: [PATCH 1/8] Add xunit runner --- source/Halibut.Tests/Halibut.Tests.csproj | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/Halibut.Tests/Halibut.Tests.csproj b/source/Halibut.Tests/Halibut.Tests.csproj index 4bcd47f28..b790a6f9a 100644 --- a/source/Halibut.Tests/Halibut.Tests.csproj +++ b/source/Halibut.Tests/Halibut.Tests.csproj @@ -47,6 +47,10 @@ + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + From 9031480efacd68183dc8f3b180d7e83a3ab519d9 Mon Sep 17 00:00:00 2001 From: Stephen Burman Date: Tue, 3 Oct 2023 09:22:18 +1100 Subject: [PATCH 2/8] Converting test cases to xUnit --- ...nedByHalibutProxyExtensionMethodFixture.cs | 48 ++++++----- .../Halibut.Tests/Properties/AssemblyInfo.cs | 2 + ...entAndServiceVersionsTestCasesAttribute.cs | 68 ++++++++++++--- ...lientAndLatestServiceTestCasesAttribute.cs | 84 +++++++++++++++---- ...eviousServiceVersionsTestCasesAttribute.cs | 55 +++++++++++- .../ClientAndServiceTestCasesBuilder.cs | 23 +++-- 6 files changed, 216 insertions(+), 64 deletions(-) diff --git a/source/Halibut.Tests/Diagnostics/ExceptionReturnedByHalibutProxyExtensionMethodFixture.cs b/source/Halibut.Tests/Diagnostics/ExceptionReturnedByHalibutProxyExtensionMethodFixture.cs index 8a4e48f50..a7d49c1fb 100644 --- a/source/Halibut.Tests/Diagnostics/ExceptionReturnedByHalibutProxyExtensionMethodFixture.cs +++ b/source/Halibut.Tests/Diagnostics/ExceptionReturnedByHalibutProxyExtensionMethodFixture.cs @@ -11,7 +11,7 @@ using Halibut.Tests.TestServices; using Halibut.Tests.TestServices.Async; using Halibut.TestUtils.Contracts; -using NUnit.Framework; +using Xunit; namespace Halibut.Tests.Diagnostics { @@ -19,7 +19,7 @@ public static class ExceptionReturnedByHalibutProxyExtensionMethodFixture { public class WhenGivenA { - [Test] + [Fact] public void MethodNotFoundHalibutClientException_ItIsNotANetworkError() { new MethodNotFoundHalibutClientException("").IsNetworkError() @@ -27,7 +27,7 @@ public void MethodNotFoundHalibutClientException_ItIsNotANetworkError() .Be(HalibutNetworkExceptionType.NotANetworkError); } - [Test] + [Fact] public void ServiceNotFoundHalibutClientException_ItIsNotANetworkError() { new ServiceNotFoundHalibutClientException("").IsNetworkError() @@ -35,7 +35,7 @@ public void ServiceNotFoundHalibutClientException_ItIsNotANetworkError() .Be(HalibutNetworkExceptionType.NotANetworkError); } - [Test] + [Fact] public void AmbiguousMethodMatchHalibutClientException_ItIsNotANetworkError() { new AmbiguousMethodMatchHalibutClientException("").IsNetworkError() @@ -44,9 +44,10 @@ public void AmbiguousMethodMatchHalibutClientException_ItIsNotANetworkError() } } - public class WhenTheHalibutProxyThrowsAnException : BaseTest + public class WhenTheHalibutProxyThrowsAnException : BaseTestXUnit { - [LatestClientAndLatestServiceTestCases(testNetworkConditions:false)] + [Theory] + [LatestClientAndLatestServiceTestCasesXUnit(testNetworkConditions:false)] public async Task WhenTheConnectionTerminatesWaitingForAResponse(ClientAndServiceTestCase clientAndServiceTestCase) { await using (var clientAndService = await clientAndServiceTestCase.CreateTestCaseBuilder() @@ -74,8 +75,9 @@ public async Task WhenTheConnectionTerminatesWaitingForAResponse(ClientAndServic because: "This isn't the best message, really the connection was closed before we got the data we were expecting resulting in us reading past the end of the stream"); } } - - [LatestClientAndLatestServiceTestCases(testNetworkConditions:false, + + [Theory] + [LatestClientAndLatestServiceTestCasesXUnit(testNetworkConditions:false, testWebSocket:false // Since websockets do not timeout )] public async Task WhenTheConnectionPausesWaitingForAResponse(ClientAndServiceTestCase clientAndServiceTestCase) @@ -101,8 +103,8 @@ public async Task WhenTheConnectionPausesWaitingForAResponse(ClientAndServiceTes } } - [Test] - [LatestClientAndLatestServiceTestCases(testNetworkConditions: false, testListening:false, testWebSocket: false)] + [Theory] + [LatestClientAndLatestServiceTestCasesXUnit(testNetworkConditions: false, testListening:false, testWebSocket: false)] public async Task BecauseThePollingRequestWasNotCollected(ClientAndServiceTestCase clientAndServiceTestCase) { var services = new DelegateServiceFactory(); @@ -121,8 +123,8 @@ public async Task BecauseThePollingRequestWasNotCollected(ClientAndServiceTestCa } } - [Test] - [LatestClientAndLatestServiceTestCases(testNetworkConditions: false, testPolling: false, testWebSocket: false)] + [Theory] + [LatestClientAndLatestServiceTestCasesXUnit(testNetworkConditions: false, testPolling: false, testWebSocket: false)] public async Task BecauseTheListeningTentacleIsNotResponding(ClientAndServiceTestCase clientAndServiceTestCase) { await using (var clientAndService = await clientAndServiceTestCase.CreateTestCaseBuilder() @@ -138,8 +140,8 @@ public async Task BecauseTheListeningTentacleIsNotResponding(ClientAndServiceTes } } - [Test] - [LatestClientAndLatestServiceTestCases(testNetworkConditions: false, testWebSocket: false, testPolling: false)] + [Theory] + [LatestClientAndLatestServiceTestCasesXUnit(testNetworkConditions: false, testWebSocket: false, testPolling: false)] public async Task BecauseTheProxyIsNotResponding_TheExceptionShouldBeANetworkError(ClientAndServiceTestCase clientAndServiceTestCase) { await using (var clientAndService = await clientAndServiceTestCase.CreateTestCaseBuilder() @@ -161,8 +163,8 @@ public async Task BecauseTheProxyIsNotResponding_TheExceptionShouldBeANetworkErr } } - [Test] - [LatestClientAndLatestServiceTestCases(testNetworkConditions: false, testPolling: false, testWebSocket: false)] + [Theory] + [LatestClientAndLatestServiceTestCasesXUnit(testNetworkConditions: false, testPolling: false, testWebSocket: false)] public async Task BecauseOfAInvalidCertificateException_WhenConnectingToListening_ItIsNotANetworkError(ClientAndServiceTestCase clientAndServiceTestCase) { await using (var clientAndService = await clientAndServiceTestCase.CreateTestCaseBuilder() @@ -181,8 +183,8 @@ public async Task BecauseOfAInvalidCertificateException_WhenConnectingToListenin } } - [Test] - [LatestClientAndLatestServiceTestCases(testNetworkConditions: false)] + [Theory] + [LatestClientAndLatestServiceTestCasesXUnit(testNetworkConditions: false)] public async Task BecauseTheDataStreamHadAnErrorOpeningTheFileWithFileStream_WhenSending_ItIsNotANetworkError(ClientAndServiceTestCase clientAndServiceTestCase) { await using (var clientAndService = await clientAndServiceTestCase.CreateTestCaseBuilder() @@ -208,8 +210,8 @@ public async Task BecauseTheDataStreamHadAnErrorOpeningTheFileWithFileStream_Whe } } - [Test] - [LatestClientAndLatestServiceTestCases(testNetworkConditions: false)] + [Theory] + [LatestClientAndLatestServiceTestCasesXUnit(testNetworkConditions: false)] public async Task BecauseTheDataStreamThrowAFileNotFoundException_WhenSending_ItIsNotANetworkError(ClientAndServiceTestCase clientAndServiceTestCase) { await using (var clientAndService = await clientAndServiceTestCase.CreateTestCaseBuilder() @@ -222,7 +224,7 @@ public async Task BecauseTheDataStreamThrowAFileNotFoundException_WhenSending_It _ => throw new FileNotFoundException(), async (_, _) => { - await Task.CompletedTask.ConfigureAwait(false); + await Task.CompletedTask; throw new FileNotFoundException(); }); @@ -234,8 +236,8 @@ public async Task BecauseTheDataStreamThrowAFileNotFoundException_WhenSending_It } } - [Test] - [LatestAndPreviousClientAndServiceVersionsTestCases(testNetworkConditions: false)] + [Theory] + [LatestAndPreviousClientAndServiceVersionsTestCasesXUnit(testNetworkConditions: false)] public async Task BecauseTheServiceThrowAnException_ItIsNotANetworkError(ClientAndServiceTestCase clientAndServiceTestCase) { await using (var clientAndService = await clientAndServiceTestCase.CreateTestCaseBuilder() diff --git a/source/Halibut.Tests/Properties/AssemblyInfo.cs b/source/Halibut.Tests/Properties/AssemblyInfo.cs index 9d65eada2..fa571acb8 100644 --- a/source/Halibut.Tests/Properties/AssemblyInfo.cs +++ b/source/Halibut.Tests/Properties/AssemblyInfo.cs @@ -2,6 +2,7 @@ using System.Reflection; using Halibut.Tests.Support.TestAttributes; using NUnit.Framework; +using Xunit; // Information about this assembly is defined by the following attributes. // Change them to the values specific to your project. @@ -11,3 +12,4 @@ [assembly: FixtureLifeCycle(LifeCycle.InstancePerTestCase)] [assembly: TestTimeout] [assembly: CustomLevelOfParallelism] +//[assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly, MaxParallelThreads = -1)] diff --git a/source/Halibut.Tests/Support/TestAttributes/LatestAndPreviousClientAndServiceVersionsTestCasesAttribute.cs b/source/Halibut.Tests/Support/TestAttributes/LatestAndPreviousClientAndServiceVersionsTestCasesAttribute.cs index 4402ddd5f..70064109b 100644 --- a/source/Halibut.Tests/Support/TestAttributes/LatestAndPreviousClientAndServiceVersionsTestCasesAttribute.cs +++ b/source/Halibut.Tests/Support/TestAttributes/LatestAndPreviousClientAndServiceVersionsTestCasesAttribute.cs @@ -1,9 +1,12 @@ using System; -using System.Collections; +using System.Collections.Generic; using System.Linq; +using System.Reflection; using Halibut.Tests.Support.BackwardsCompatibility; using Halibut.Tests.Support.TestCases; using Halibut.Util; +using Xunit.Sdk; +using static Halibut.Tests.Support.TestAttributes.LatestAndPreviousClientAndServiceVersionsTestCasesAttribute; namespace Halibut.Tests.Support.TestAttributes { @@ -17,17 +20,18 @@ public LatestAndPreviousClientAndServiceVersionsTestCasesAttribute( bool testPolling = true, bool testAsyncAndSyncClients = true, bool testAsyncServicesAsWell = false // False means only the sync service will be tested. - ) : + ) : base( typeof(LatestAndPreviousClientAndServiceVersionsTestCases), nameof(LatestAndPreviousClientAndServiceVersionsTestCases.GetEnumerator), - new object[] { testWebSocket, testNetworkConditions, testListening, testPolling, testAsyncAndSyncClients, testAsyncServicesAsWell}) + new object[] { testWebSocket, testNetworkConditions, testListening, testPolling, testAsyncAndSyncClients, testAsyncServicesAsWell }) { } - - static class LatestAndPreviousClientAndServiceVersionsTestCases + + public static class LatestAndPreviousClientAndServiceVersionsTestCases { - public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkConditions, bool testListening, bool testPolling, bool testAsyncAndSyncClients, bool testAsyncServicesAsWell) + //TODO: @server-at-scale - When NUnit is removed, remove this class, and make this method the body of LatestAndPreviousClientAndServiceVersionsTestCasesXUnitAttribute.GetData + public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkConditions, bool testListening, bool testPolling, bool testAsyncAndSyncClients, bool testAsyncServicesAsWell) { var serviceConnectionTypes = ServiceConnectionTypes.All.ToList(); @@ -51,15 +55,16 @@ public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkCond { clientProxyTypesToTest = ForceClientProxyTypeValues.All; } - + var serviceAsyncHalibutFeatureTestCases = AsyncHalibutFeatureValues.All().ToList(); if (!testAsyncServicesAsWell) { serviceAsyncHalibutFeatureTestCases.Remove(AsyncHalibutFeature.Enabled); } - + var builder = new ClientAndServiceTestCasesBuilder( - new[] { + new[] + { ClientAndServiceTestVersion.Latest(), ClientAndServiceTestVersion.ClientOfVersion(PreviousVersions.v5_0_236_Used_In_Tentacle_6_3_417.ClientVersion), ClientAndServiceTestVersion.ServiceOfVersion(PreviousVersions.v5_0_236_Used_In_Tentacle_6_3_417.ServiceVersion), @@ -69,10 +74,53 @@ public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkCond testNetworkConditions ? NetworkConditionTestCase.All : new[] { NetworkConditionTestCase.NetworkConditionPerfect }, clientProxyTypesToTest, serviceAsyncHalibutFeatureTestCases - ); + ); return builder.Build(); } } } + + //TODO: @server-at-scale - When NUnit is removed, replace the above LatestAndPreviousClientAndServiceVersionsTestCasesAttribute with this. + [AttributeUsage(AttributeTargets.Method, AllowMultiple = true, Inherited = false)] + public class LatestAndPreviousClientAndServiceVersionsTestCasesXUnitAttribute : DataAttribute + { + readonly bool testWebSocket; + readonly bool testNetworkConditions; + readonly bool testListening; + readonly bool testPolling; + readonly bool testAsyncAndSyncClients; + readonly bool testAsyncServicesAsWell; + + public LatestAndPreviousClientAndServiceVersionsTestCasesXUnitAttribute( + bool testWebSocket = true, + bool testNetworkConditions = true, + bool testListening = true, + bool testPolling = true, + bool testAsyncAndSyncClients = true, + bool testAsyncServicesAsWell = false // False means only the sync service will be tested. + ) + { + this.testWebSocket = testWebSocket; + this.testNetworkConditions = testNetworkConditions; + this.testListening = testListening; + this.testPolling = testPolling; + this.testAsyncAndSyncClients = testAsyncAndSyncClients; + this.testAsyncServicesAsWell = testAsyncServicesAsWell; + } + + public override IEnumerable GetData(MethodInfo testMethod) + { + //TODO: @server-at-scale - When NUnit is removed, move LatestClientAndLatestServiceTestCases.GetEnumerator into here. + + return LatestAndPreviousClientAndServiceVersionsTestCases.GetEnumerator( + testWebSocket, + testNetworkConditions, + testListening, + testPolling, + testAsyncAndSyncClients, + testAsyncServicesAsWell); + } + } + } diff --git a/source/Halibut.Tests/Support/TestAttributes/LatestClientAndLatestServiceTestCasesAttribute.cs b/source/Halibut.Tests/Support/TestAttributes/LatestClientAndLatestServiceTestCasesAttribute.cs index 0a7c36fce..70a5a3fca 100644 --- a/source/Halibut.Tests/Support/TestAttributes/LatestClientAndLatestServiceTestCasesAttribute.cs +++ b/source/Halibut.Tests/Support/TestAttributes/LatestClientAndLatestServiceTestCasesAttribute.cs @@ -1,9 +1,11 @@ using System; -using System.Collections; using System.Collections.Generic; using System.Linq; +using System.Reflection; using Halibut.Tests.Support.TestCases; using Halibut.Util; +using Xunit.Sdk; +using static Halibut.Tests.Support.TestAttributes.LatestClientAndLatestServiceTestCasesAttribute; namespace Halibut.Tests.Support.TestAttributes { @@ -24,17 +26,18 @@ public LatestClientAndLatestServiceTestCasesAttribute( bool testAsyncServicesAsWell = false, // False means only the sync service will be tested. bool testSyncService = true, params object[] additionalParameters - ) : + ) : base( typeof(LatestClientAndLatestServiceTestCases), nameof(LatestClientAndLatestServiceTestCases.GetEnumerator), new object[] { testWebSocket, testNetworkConditions, testListening, testPolling, testSyncClients, testAsyncClients, testAsyncServicesAsWell, testSyncService, additionalParameters }) { } - - static class LatestClientAndLatestServiceTestCases + + public static class LatestClientAndLatestServiceTestCases { - public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkConditions, bool testListening, bool testPolling, bool testSyncClients, bool testAsyncClients, bool testAsyncServicesAsWell, bool testSyncService, object[] additionalParameters) + //TODO: @server-at-scale - When NUnit is removed, remove this class, and make this method the body of LatestClientAndLatestServiceTestCasesXUnitAttribute.GetData + public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkConditions, bool testListening, bool testPolling, bool testSyncClients, bool testAsyncClients, bool testAsyncServicesAsWell, bool testSyncService, object[] additionalParameters) { var serviceConnectionTypes = ServiceConnectionTypes.All.ToList(); @@ -52,9 +55,9 @@ public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkCond { serviceConnectionTypes.Remove(ServiceConnectionType.Polling); } - + List clientProxyTypesToTest = new(); - + if (testAsyncClients) { clientProxyTypesToTest.Add(ForceClientProxyType.AsyncClient); @@ -64,7 +67,7 @@ public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkCond clientProxyTypesToTest.Add(ForceClientProxyType.SyncClient); } } - + var serviceAsyncHalibutFeatureTestCases = AsyncHalibutFeatureValues.All().ToList(); if (!testAsyncServicesAsWell) { @@ -83,7 +86,7 @@ public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkCond clientProxyTypesToTest, serviceAsyncHalibutFeatureTestCases ); - + foreach (var clientAndServiceTestCase in builder.Build()) { if (additionalParameters.Any()) @@ -98,16 +101,67 @@ public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkCond } } - static object[] CombineTestCaseWithAdditionalParameters(ClientAndServiceTestCase clientAndServiceTestCase, object[] additionalParameters) + static object[] CombineTestCaseWithAdditionalParameters(object[] clientAndServiceTestCase, object[] additionalParameters) { - var parameters = new object[1 + additionalParameters.Length]; - - parameters[0] = clientAndServiceTestCase; - - Array.Copy(additionalParameters, 0, parameters, 1, additionalParameters.Length); + var parameters = new object[clientAndServiceTestCase.Length + additionalParameters.Length]; + + Array.Copy(clientAndServiceTestCase, 0, parameters, 0, clientAndServiceTestCase.Length); + Array.Copy(additionalParameters, 0, parameters, clientAndServiceTestCase.Length, additionalParameters.Length); return parameters; } } } + + //TODO: @server-at-scale - When NUnit is removed, replace the above LatestClientAndLatestServiceTestCasesAttribute with this. + public class LatestClientAndLatestServiceTestCasesXUnitAttribute : DataAttribute + { + readonly bool testWebSocket; + readonly bool testNetworkConditions; + readonly bool testListening; + readonly bool testPolling; + readonly bool testSyncClients; + readonly bool testAsyncClients; + readonly bool testAsyncServicesAsWell; + readonly bool testSyncService; + readonly object[] additionalParameters; + + public LatestClientAndLatestServiceTestCasesXUnitAttribute( + bool testWebSocket = true, + bool testNetworkConditions = true, + bool testListening = true, + bool testPolling = true, + bool testSyncClients = true, + bool testAsyncClients = true, + bool testAsyncServicesAsWell = false, // False means only the sync service will be tested. + bool testSyncService = true, + params object[] additionalParameters) + { + this.testWebSocket = testWebSocket; + this.testNetworkConditions = testNetworkConditions; + this.testListening = testListening; + this.testPolling = testPolling; + this.testSyncClients = testSyncClients; + this.testAsyncClients = testAsyncClients; + this.testAsyncServicesAsWell = testAsyncServicesAsWell; + this.testSyncService = testSyncService; + this.additionalParameters = additionalParameters; + } + + public override IEnumerable GetData(MethodInfo testMethod) + { + //TODO: @server-at-scale - When NUnit is removed, move LatestClientAndLatestServiceTestCases.GetEnumerator into here. + + return LatestClientAndLatestServiceTestCases.GetEnumerator( + testWebSocket, + testNetworkConditions, + testListening, + testPolling, + testSyncClients, + testAsyncClients, + testAsyncServicesAsWell, + testSyncService, + additionalParameters); + } + } } diff --git a/source/Halibut.Tests/Support/TestAttributes/LatestClientAndPreviousServiceVersionsTestCasesAttribute.cs b/source/Halibut.Tests/Support/TestAttributes/LatestClientAndPreviousServiceVersionsTestCasesAttribute.cs index b832e0ad0..cc7fdfed9 100644 --- a/source/Halibut.Tests/Support/TestAttributes/LatestClientAndPreviousServiceVersionsTestCasesAttribute.cs +++ b/source/Halibut.Tests/Support/TestAttributes/LatestClientAndPreviousServiceVersionsTestCasesAttribute.cs @@ -1,10 +1,12 @@ using System; -using System.Collections; using System.Collections.Generic; using System.Linq; +using System.Reflection; using Halibut.Tests.Support.BackwardsCompatibility; using Halibut.Tests.Support.TestCases; using Halibut.Util; +using Xunit.Sdk; +using static Halibut.Tests.Support.TestAttributes.LatestClientAndPreviousServiceVersionsTestCasesAttribute; namespace Halibut.Tests.Support.TestAttributes { @@ -26,9 +28,10 @@ public LatestClientAndPreviousServiceVersionsTestCasesAttribute(bool testWebSock { } - static class LatestClientAndPreviousServiceVersionsTestCases + public static class LatestClientAndPreviousServiceVersionsTestCases { - public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkConditions, bool testListening, bool testPolling, bool testAsyncClients, bool testSyncClients, bool testAsyncServicesAsWell) + //TODO: @server-at-scale - When NUnit is removed, remove this class, and make this method the body of LatestClientAndPreviousServiceVersionsTestCasesXUnitAttribute.GetData + public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkConditions, bool testListening, bool testPolling, bool testAsyncClients, bool testSyncClients, bool testAsyncServicesAsWell) { var serviceConnectionTypes = ServiceConnectionTypes.All.ToList(); @@ -81,4 +84,50 @@ public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkCond } } } + + //TODO: @server-at-scale - When NUnit is removed, replace the above LatestClientAndPreviousServiceVersionsTestCasesAttribute with this. + [AttributeUsage(AttributeTargets.Method, AllowMultiple = true, Inherited = false)] + public class LatestClientAndPreviousServiceVersionsTestCasesXUnitAttribute : DataAttribute + { + readonly bool testWebSocket; + readonly bool testNetworkConditions; + readonly bool testListening; + readonly bool testPolling; + readonly bool testAsyncClients; + readonly bool testSyncClients; + readonly bool testAsyncServicesAsWell; + + public LatestClientAndPreviousServiceVersionsTestCasesXUnitAttribute( + bool testWebSocket = true, + bool testNetworkConditions = true, + bool testListening = true, + bool testPolling = true, + bool testAsyncClients = true, + bool testSyncClients = true, + bool testAsyncServicesAsWell = false // False means only the sync service will be tested. + ) + { + this.testWebSocket = testWebSocket; + this.testNetworkConditions = testNetworkConditions; + this.testListening = testListening; + this.testPolling = testPolling; + this.testAsyncClients = testAsyncClients; + this.testSyncClients = testSyncClients; + this.testAsyncServicesAsWell = testAsyncServicesAsWell; + } + + public override IEnumerable GetData(MethodInfo testMethod) + { + //TODO: @server-at-scale - When NUnit is removed, move LatestClientAndLatestServiceTestCases.GetEnumerator into here. + + return LatestClientAndPreviousServiceVersionsTestCases.GetEnumerator( + testWebSocket, + testNetworkConditions, + testListening, + testPolling, + testAsyncClients, + testSyncClients, + testAsyncServicesAsWell); + } + } } diff --git a/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCasesBuilder.cs b/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCasesBuilder.cs index 5fa4ed5bc..9e7eb7149 100644 --- a/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCasesBuilder.cs +++ b/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCasesBuilder.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; using Halibut.Tests.Util; @@ -26,16 +27,16 @@ public ClientAndServiceTestCasesBuilder( this.networkConditionTestCases = networkConditionTestCases.Distinct().ToArray(); this.forceClientProxyTypes = forceClientProxyTypes.Distinct().ToArray(); } - - public IEnumerable Build() + + public IEnumerable Build() { - return BuildDistinct(); + return BuildTestCases() + .Distinct() + .Select(tc => new object[]{tc}); } - List BuildDistinct() + IEnumerable BuildTestCases() { - var cases = new List(); - foreach (var clientServiceTestVersion in clientServiceTestVersions) { foreach (var serviceConnectionType in serviceConnectionTypes) @@ -59,19 +60,19 @@ List BuildDistinct() if (!forceClientProxyTypes.Any()) { - cases.Add(new ClientAndServiceTestCase(serviceConnectionType, networkConditionTestCase, recommendedIterations, clientServiceTestVersion, null, serviceAsyncHalibutFeatureTestCase)); + yield return new ClientAndServiceTestCase(serviceConnectionType, networkConditionTestCase, recommendedIterations, clientServiceTestVersion, null, serviceAsyncHalibutFeatureTestCase); } else { if (clientServiceTestVersion.IsPreviousClient()) { - cases.Add(new ClientAndServiceTestCase(serviceConnectionType, networkConditionTestCase, recommendedIterations, clientServiceTestVersion, null, serviceAsyncHalibutFeatureTestCase)); + yield return new ClientAndServiceTestCase(serviceConnectionType, networkConditionTestCase, recommendedIterations, clientServiceTestVersion, null, serviceAsyncHalibutFeatureTestCase); } else { foreach (var forceClientProxyType in forceClientProxyTypes) { - cases.Add(new ClientAndServiceTestCase(serviceConnectionType, networkConditionTestCase, recommendedIterations, clientServiceTestVersion, forceClientProxyType, serviceAsyncHalibutFeatureTestCase)); + yield return new ClientAndServiceTestCase(serviceConnectionType, networkConditionTestCase, recommendedIterations, clientServiceTestVersion, forceClientProxyType, serviceAsyncHalibutFeatureTestCase); } } } @@ -79,10 +80,6 @@ List BuildDistinct() } } } - - var distinct = cases.Distinct().ToList(); - - return distinct; } } } From 23959064016cc1975968e97a5feacfdbb4dfec08 Mon Sep 17 00:00:00 2001 From: Stephen Burman Date: Tue, 3 Oct 2023 15:48:54 +1100 Subject: [PATCH 3/8] Making tests parallel --- .../Halibut.Tests/Properties/AssemblyInfo.cs | 2 +- .../TestCases/ClientAndServiceTestCase.cs | 15 ++- .../Util/ParallelTestFramework.cs | 114 ++++++++++++++++++ 3 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 source/Halibut.Tests/Util/ParallelTestFramework.cs diff --git a/source/Halibut.Tests/Properties/AssemblyInfo.cs b/source/Halibut.Tests/Properties/AssemblyInfo.cs index fa571acb8..b501ac393 100644 --- a/source/Halibut.Tests/Properties/AssemblyInfo.cs +++ b/source/Halibut.Tests/Properties/AssemblyInfo.cs @@ -12,4 +12,4 @@ [assembly: FixtureLifeCycle(LifeCycle.InstancePerTestCase)] [assembly: TestTimeout] [assembly: CustomLevelOfParallelism] -//[assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly, MaxParallelThreads = -1)] +[assembly: TestFramework("Halibut.Tests.Util.ParallelTestFramework", "Halibut.Tests")] \ No newline at end of file diff --git a/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCase.cs b/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCase.cs index 40204c500..59627c484 100644 --- a/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCase.cs +++ b/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCase.cs @@ -3,10 +3,11 @@ using System.Text; using Halibut.Tests.Support.TestAttributes; using Halibut.Util; +using Xunit.Abstractions; namespace Halibut.Tests.Support.TestCases { - public class ClientAndServiceTestCase + public class ClientAndServiceTestCase : IXunitSerializable { public ClientAndServiceTestVersion ClientAndServiceTestVersion { get; } @@ -26,6 +27,10 @@ public class ClientAndServiceTestCase public ForceClientProxyType? ForceClientProxyType { get; } public SyncOrAsync SyncOrAsync => ForceClientProxyType.ToSyncOrAsync(); + public ClientAndServiceTestCase() + { + } + public ClientAndServiceTestCase(ServiceConnectionType serviceConnectionType, NetworkConditionTestCase networkConditionTestCase, int recommendedIterations, @@ -114,5 +119,13 @@ public override int GetHashCode() { return ToString().GetHashCode(); } + + public void Deserialize(IXunitSerializationInfo info) + { + } + + public void Serialize(IXunitSerializationInfo info) + { + } } } diff --git a/source/Halibut.Tests/Util/ParallelTestFramework.cs b/source/Halibut.Tests/Util/ParallelTestFramework.cs new file mode 100644 index 000000000..14d1239c6 --- /dev/null +++ b/source/Halibut.Tests/Util/ParallelTestFramework.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Halibut.Tests.Util +{ + // To make this work, we have to hook it up as the TestFramework in AssemblyInfo (including namespace) + sealed class ParallelTestFramework : XunitTestFramework + { + public ParallelTestFramework(IMessageSink diagnosticMessageSink) + : base(diagnosticMessageSink) + { + } + + protected override ITestFrameworkExecutor CreateExecutor(AssemblyName assemblyName) + { + return new CustomTestExecutor(assemblyName, SourceInformationProvider, DiagnosticMessageSink); + } + + sealed class CustomTestExecutor : XunitTestFrameworkExecutor + { + public CustomTestExecutor(AssemblyName assemblyName, ISourceInformationProvider sourceInformationProvider, IMessageSink diagnosticMessageSink) + : base(assemblyName, sourceInformationProvider, diagnosticMessageSink) + { + } + + protected override async void RunTestCases(IEnumerable testCases, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions) + { + var newTestCases = SetUpTestCaseParallelization(testCases); + + using var assemblyRunner = new XunitTestAssemblyRunner(TestAssembly, newTestCases, DiagnosticMessageSink, executionMessageSink, executionOptions); + await assemblyRunner.RunAsync(); + } + + /// + /// By default, all test cases in a test class share the same collection instance which ensures they run synchronously. + /// By providing a unique test collection instance to every test case in a test class you can make them all run in parallel. + /// + IEnumerable SetUpTestCaseParallelization(IEnumerable testCases) + { + var result = new List(); + foreach (var testCase in testCases) + { + var oldTestMethod = testCase.TestMethod; + var oldTestClass = oldTestMethod.TestClass; + var oldTestCollection = oldTestMethod.TestClass.TestCollection; + + // If the collection is explicitly set, don't try to parallelize test execution + if (oldTestCollection.CollectionDefinition != null || oldTestClass.Class.GetCustomAttributes(typeof(CollectionAttribute)).Any()) + { + result.Add(testCase); + continue; + } + + // Create a new collection with a unique id for the test case. + var newTestCollection = + new TestCollection( + oldTestCollection.TestAssembly, + oldTestCollection.CollectionDefinition, + displayName: $"{oldTestCollection.DisplayName} {oldTestCollection.UniqueID}"); + newTestCollection.UniqueID = Guid.NewGuid(); + + // Duplicate the test and assign it to the new collection + var newTestClass = new TestClass(newTestCollection, oldTestClass.Class); + var newTestMethod = new TestMethod(newTestClass, oldTestMethod.Method); + switch (testCase) + { + // Used by Theory having DisableDiscoveryEnumeration or non-serializable data + case XunitTheoryTestCase xunitTheoryTestCase: + result.Add(new XunitTheoryTestCase( + DiagnosticMessageSink, + GetTestMethodDisplay(xunitTheoryTestCase), + GetTestMethodDisplayOptions(xunitTheoryTestCase), + newTestMethod)); + break; + + // Used by all other tests + case XunitTestCase xunitTestCase: + result.Add(new XunitTestCase( + DiagnosticMessageSink, + GetTestMethodDisplay(xunitTestCase), + GetTestMethodDisplayOptions(xunitTestCase), + newTestMethod, + xunitTestCase.TestMethodArguments)); + break; + + default: + throw new ArgumentOutOfRangeException("Test case " + testCase.GetType() + " not supported"); + } + } + + return result; + + static TestMethodDisplay GetTestMethodDisplay(TestMethodTestCase testCase) + { + return (TestMethodDisplay)typeof(TestMethodTestCase) + .GetProperty("DefaultMethodDisplay", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(testCase)!; + } + + static TestMethodDisplayOptions GetTestMethodDisplayOptions(TestMethodTestCase testCase) + { + return (TestMethodDisplayOptions)typeof(TestMethodTestCase) + .GetProperty("DefaultMethodDisplayOptions", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(testCase)!; + } + } + } + } +} \ No newline at end of file From 2292e1ac7861a28553c1a9791f2aa6e6bca94e4a Mon Sep 17 00:00:00 2001 From: Stephen Burman Date: Wed, 4 Oct 2023 13:30:00 +1100 Subject: [PATCH 4/8] Fixing test cases --- .../TestAttributes/HalibutTestCaseSourceAttribute.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/source/Halibut.Tests/Support/TestAttributes/HalibutTestCaseSourceAttribute.cs b/source/Halibut.Tests/Support/TestAttributes/HalibutTestCaseSourceAttribute.cs index 6d068625a..47b0847ee 100644 --- a/source/Halibut.Tests/Support/TestAttributes/HalibutTestCaseSourceAttribute.cs +++ b/source/Halibut.Tests/Support/TestAttributes/HalibutTestCaseSourceAttribute.cs @@ -162,6 +162,18 @@ private IEnumerable GetTestCasesFor(IMethodInfo method) { parms.Properties.Add(PropertyNames.Category, testCase.ServiceConnectionType.ToString()); parms.Properties.Add(PropertyNames.Category, testCase.SyncOrAsync.ToString()); + } + else if (item is IEnumerable enumerableItem) + { + foreach (var itemFromEnumerable in enumerableItem) + { + if (itemFromEnumerable is ClientAndServiceTestCase testCaseFromEnumerable) + { + parms.Properties.Add(PropertyNames.Category, testCaseFromEnumerable.ServiceConnectionType.ToString()); + parms.Properties.Add(PropertyNames.Category, testCaseFromEnumerable.SyncOrAsync.ToString()); + break; + } + } } data.Add(parms); From 12ed3894cbf21874ac4a15ee908fe257b1496182 Mon Sep 17 00:00:00 2001 From: Stephen Burman Date: Thu, 12 Oct 2023 13:09:43 +1100 Subject: [PATCH 5/8] Assembly info --- source/Halibut.Tests/Properties/AssemblyInfo.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/Halibut.Tests/Properties/AssemblyInfo.cs b/source/Halibut.Tests/Properties/AssemblyInfo.cs index b501ac393..4bb7dba26 100644 --- a/source/Halibut.Tests/Properties/AssemblyInfo.cs +++ b/source/Halibut.Tests/Properties/AssemblyInfo.cs @@ -1,6 +1,7 @@ using System; using System.Reflection; using Halibut.Tests.Support.TestAttributes; +using Halibut.Tests.Util; using NUnit.Framework; using Xunit; @@ -12,4 +13,4 @@ [assembly: FixtureLifeCycle(LifeCycle.InstancePerTestCase)] [assembly: TestTimeout] [assembly: CustomLevelOfParallelism] -[assembly: TestFramework("Halibut.Tests.Util.ParallelTestFramework", "Halibut.Tests")] \ No newline at end of file +[assembly: TestFramework("Halibut.Tests.Util." + nameof(ParallelTestFramework), "Halibut.Tests")] \ No newline at end of file From f3e678a89951f60b0311dcb8ed81fa89305354e6 Mon Sep 17 00:00:00 2001 From: Stephen Burman Date: Thu, 12 Oct 2023 16:13:44 +1100 Subject: [PATCH 6/8] Serialization --- .../PreviousVersions.cs | 27 ++++++++-- .../TestCases/ClientAndServiceTestCase.cs | 30 +++++++---- .../TestCases/ClientAndServiceTestVersion.cs | 23 ++++++-- .../TestCases/NetworkConditionTestCase.cs | 54 +++++++++++++++---- 4 files changed, 109 insertions(+), 25 deletions(-) diff --git a/source/Halibut.Tests/Support/BackwardsCompatibility/PreviousVersions.cs b/source/Halibut.Tests/Support/BackwardsCompatibility/PreviousVersions.cs index 525a4261b..83c205eda 100644 --- a/source/Halibut.Tests/Support/BackwardsCompatibility/PreviousVersions.cs +++ b/source/Halibut.Tests/Support/BackwardsCompatibility/PreviousVersions.cs @@ -1,4 +1,5 @@ using System; +using Xunit.Abstractions; namespace Halibut.Tests.Support.BackwardsCompatibility { @@ -22,11 +23,15 @@ public static class PreviousVersions #endif } - public class HalibutVersion : IEquatable + public class HalibutVersion : IEquatable, IXunitSerializable { - readonly Version pollingVersion; - readonly Version listeningVersion; - readonly Version pollingOverWebSocketVersion; + Version pollingVersion; + Version listeningVersion; + Version pollingOverWebSocketVersion; + + public HalibutVersion() + { + } internal HalibutVersion( Version version, @@ -77,6 +82,20 @@ public override int GetHashCode() return hashCode; } } + + public void Deserialize(IXunitSerializationInfo info) + { + pollingVersion = Version.Parse(info.GetValue(nameof(pollingVersion))); + listeningVersion = Version.Parse(info.GetValue(nameof(listeningVersion))); + pollingOverWebSocketVersion = Version.Parse(info.GetValue(nameof(pollingOverWebSocketVersion))); + } + + public void Serialize(IXunitSerializationInfo info) + { + info.AddValue(nameof(pollingVersion), pollingVersion.ToString()); + info.AddValue(nameof(listeningVersion), listeningVersion.ToString()); + info.AddValue(nameof(pollingOverWebSocketVersion), pollingOverWebSocketVersion.ToString()); + } } public class HalibutVersions diff --git a/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCase.cs b/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCase.cs index 59627c484..27d53b982 100644 --- a/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCase.cs +++ b/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCase.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.Text; using Halibut.Tests.Support.TestAttributes; using Halibut.Util; @@ -9,29 +8,30 @@ namespace Halibut.Tests.Support.TestCases { public class ClientAndServiceTestCase : IXunitSerializable { - public ClientAndServiceTestVersion ClientAndServiceTestVersion { get; } + public ClientAndServiceTestVersion ClientAndServiceTestVersion { get; private set; } - public NetworkConditionTestCase NetworkConditionTestCase { get; } + public NetworkConditionTestCase NetworkConditionTestCase { get; private set; } - public ServiceConnectionType ServiceConnectionType { get; } + public ServiceConnectionType ServiceConnectionType { get; private set; } - public AsyncHalibutFeature ServiceAsyncHalibutFeature { get; } + public AsyncHalibutFeature ServiceAsyncHalibutFeature { get; private set; } /// /// If running a test which wants to make the same or similar calls multiple time, this has a "good" /// number of times to do that. It takes care of generally not picking such a high number the tests /// don't take a long time. /// - public int RecommendedIterations { get; } + public int RecommendedIterations { get; private set; } - public ForceClientProxyType? ForceClientProxyType { get; } + public ForceClientProxyType? ForceClientProxyType { get; private set; } public SyncOrAsync SyncOrAsync => ForceClientProxyType.ToSyncOrAsync(); - + public ClientAndServiceTestCase() { } - public ClientAndServiceTestCase(ServiceConnectionType serviceConnectionType, + public ClientAndServiceTestCase( + ServiceConnectionType serviceConnectionType, NetworkConditionTestCase networkConditionTestCase, int recommendedIterations, ClientAndServiceTestVersion clientAndServiceTestVersion, @@ -122,10 +122,22 @@ public override int GetHashCode() public void Deserialize(IXunitSerializationInfo info) { + ClientAndServiceTestVersion = info.GetValue(nameof(ClientAndServiceTestVersion)); + NetworkConditionTestCase = info.GetValue(nameof(NetworkConditionTestCase)); + ServiceConnectionType = info.GetValue(nameof(ServiceConnectionType)); + ServiceAsyncHalibutFeature = info.GetValue(nameof(ServiceAsyncHalibutFeature)); + RecommendedIterations = info.GetValue(nameof(RecommendedIterations)); + ForceClientProxyType = info.GetValue(nameof(ForceClientProxyType)); } public void Serialize(IXunitSerializationInfo info) { + info.AddValue(nameof(ClientAndServiceTestVersion), ClientAndServiceTestVersion); + info.AddValue(nameof(NetworkConditionTestCase), NetworkConditionTestCase); + info.AddValue(nameof(ServiceConnectionType), ServiceConnectionType); + info.AddValue(nameof(ServiceAsyncHalibutFeature), ServiceAsyncHalibutFeature); + info.AddValue(nameof(RecommendedIterations), RecommendedIterations); + info.AddValue(nameof(ForceClientProxyType), ForceClientProxyType); } } } diff --git a/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestVersion.cs b/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestVersion.cs index 5668cbe30..2aebd3839 100644 --- a/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestVersion.cs +++ b/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestVersion.cs @@ -1,18 +1,23 @@ using System; using Halibut.Tests.Support.BackwardsCompatibility; +using Xunit.Abstractions; namespace Halibut.Tests.Support.TestCases { /// /// Defines what version of the Client and the Service that will be used in a test. /// - public class ClientAndServiceTestVersion : IEquatable + public class ClientAndServiceTestVersion : IEquatable, IXunitSerializable { // null means latest. - public HalibutVersion? ClientVersion { get; } + public HalibutVersion? ClientVersion { get; private set; } // null means latest. - public HalibutVersion? ServiceVersion { get; } + public HalibutVersion? ServiceVersion { get; private set; } + + public ClientAndServiceTestVersion() + { + } ClientAndServiceTestVersion(HalibutVersion? clientVersion, HalibutVersion? serviceVersion) { @@ -131,6 +136,18 @@ public override int GetHashCode() return hashCode; } } + + public void Deserialize(IXunitSerializationInfo info) + { + ClientVersion = info.GetValue(nameof(ClientVersion)); + ServiceVersion = info.GetValue(nameof(ServiceVersion)); + } + + public void Serialize(IXunitSerializationInfo info) + { + info.AddValue(nameof(ClientVersion), ClientVersion); + info.AddValue(nameof(ServiceVersion), ServiceVersion); + } } public class ClientAndServiceBuilderFactory diff --git a/source/Halibut.Tests/Support/TestCases/NetworkConditionTestCase.cs b/source/Halibut.Tests/Support/TestCases/NetworkConditionTestCase.cs index f80e2b84d..7f5c11859 100644 --- a/source/Halibut.Tests/Support/TestCases/NetworkConditionTestCase.cs +++ b/source/Halibut.Tests/Support/TestCases/NetworkConditionTestCase.cs @@ -1,10 +1,11 @@ using System; using Octopus.TestPortForwarder; using Serilog; +using Xunit.Abstractions; namespace Halibut.Tests.Support.TestCases { - public class NetworkConditionTestCase + public class NetworkConditionTestCase : IXunitSerializable { public static NetworkConditionTestCase[] All => new[] { @@ -16,17 +17,17 @@ public class NetworkConditionTestCase }; public static NetworkConditionTestCase NetworkConditionPerfect = - new (null, + new (0, "Perfect", "Perfect"); public static NetworkConditionTestCase NetworkCondition20MsLatency = - new ((i, logger) => PortForwarderBuilder.ForwardingToLocalPort(i, logger).WithSendDelay(TimeSpan.FromMilliseconds(20)).Build(), + new (1, "20ms SendDelay", "20msLatency"); public static NetworkConditionTestCase NetworkCondition20MsLatencyWithLastByteArrivingLate = - new ((i, logger) => PortForwarderBuilder.ForwardingToLocalPort(i, logger).WithSendDelay(TimeSpan.FromMilliseconds(20)).WithNumberOfBytesToDelaySending(1).Build(), + new (2, "20ms SendDelay last byte arrives late", "20msLatency&LastByteLate"); @@ -38,15 +39,36 @@ public class NetworkConditionTestCase // new ((i, logger) => PortForwarderBuilder.ForwardingToLocalPort(i, logger).WithSendDelay(TimeSpan.FromMilliseconds(20)).WithNumberOfBytesToDelaySending(3).Build(), // "20ms send delay with last 3 bytes arriving late"); - public Func? PortForwarderFactory { get; } + //TODO: Make nicer. + int factory; + public Func? PortForwarderFactory + { + get + { + switch (factory) + { + case 1: + return (i, logger) => PortForwarderBuilder.ForwardingToLocalPort(i, logger).WithSendDelay(TimeSpan.FromMilliseconds(20)).Build(); + case 2: + return (i, logger) => PortForwarderBuilder.ForwardingToLocalPort(i, logger).WithSendDelay(TimeSpan.FromMilliseconds(20)).WithNumberOfBytesToDelaySending(1).Build(); + } + + return null; + } + } - public string NetworkConditionDescription { get; } + public string NetworkConditionDescription { get; private set; } - public string ShortNetworkConditionDescription { get; } + public string ShortNetworkConditionDescription { get; private set; } + + public NetworkConditionTestCase() + { + + } - public NetworkConditionTestCase(Func? portForwarderFactory, string networkConditionDescription, string shortNetworkConditionDescription) + public NetworkConditionTestCase(int factory, string networkConditionDescription, string shortNetworkConditionDescription) { - PortForwarderFactory = portForwarderFactory; + this.factory = factory; NetworkConditionDescription = networkConditionDescription; ShortNetworkConditionDescription = shortNetworkConditionDescription; @@ -61,5 +83,19 @@ public string ToShortString() { return $"Net: '{ShortNetworkConditionDescription}'"; } + + public void Deserialize(IXunitSerializationInfo info) + { + factory = info.GetValue(nameof(factory)); + NetworkConditionDescription = info.GetValue(nameof(NetworkConditionDescription)); + ShortNetworkConditionDescription = info.GetValue(nameof(ShortNetworkConditionDescription)); + } + + public void Serialize(IXunitSerializationInfo info) + { + info.AddValue(nameof(factory), factory); + info.AddValue(nameof(NetworkConditionDescription), NetworkConditionDescription); + info.AddValue(nameof(ShortNetworkConditionDescription), ShortNetworkConditionDescription); + } } } \ No newline at end of file From 930b5247f83c7f30e144ba347b5eab60a7ed008e Mon Sep 17 00:00:00 2001 From: Stephen Burman Date: Mon, 16 Oct 2023 16:00:50 +1100 Subject: [PATCH 7/8] Parallel testing POC --- source/Halibut.Tests/Properties/AssemblyInfo.cs | 3 ++- .../Halibut.Tests/Util/DoNotRunInParallelGlobally.cs | 11 +++++++++++ source/Halibut.sln | 12 ++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 source/Halibut.Tests/Util/DoNotRunInParallelGlobally.cs diff --git a/source/Halibut.Tests/Properties/AssemblyInfo.cs b/source/Halibut.Tests/Properties/AssemblyInfo.cs index 4bb7dba26..802e380e1 100644 --- a/source/Halibut.Tests/Properties/AssemblyInfo.cs +++ b/source/Halibut.Tests/Properties/AssemblyInfo.cs @@ -13,4 +13,5 @@ [assembly: FixtureLifeCycle(LifeCycle.InstancePerTestCase)] [assembly: TestTimeout] [assembly: CustomLevelOfParallelism] -[assembly: TestFramework("Halibut.Tests.Util." + nameof(ParallelTestFramework), "Halibut.Tests")] \ No newline at end of file +[assembly: TestFramework("Halibut.Tests.Util." + nameof(ParallelTestFramework), "Halibut.Tests")] +[assembly: CollectionBehavior(MaxParallelThreads = 2)] \ No newline at end of file diff --git a/source/Halibut.Tests/Util/DoNotRunInParallelGlobally.cs b/source/Halibut.Tests/Util/DoNotRunInParallelGlobally.cs new file mode 100644 index 000000000..72c76f047 --- /dev/null +++ b/source/Halibut.Tests/Util/DoNotRunInParallelGlobally.cs @@ -0,0 +1,11 @@ +using System; +using Xunit; + +namespace Halibut.Tests.Util +{ + // CollectionDefinition does NOT WORK when put on a test class itself. So we make this separate collection to use CollectionDefinition. + [CollectionDefinition(nameof(DoNotRunInParallelGlobally), DisableParallelization = true)] + public class DoNotRunInParallelGlobally + { + } +} \ No newline at end of file diff --git a/source/Halibut.sln b/source/Halibut.sln index d8e0f3781..40800d4d7 100644 --- a/source/Halibut.sln +++ b/source/Halibut.sln @@ -72,6 +72,18 @@ Global {65BD1243-FCB9-40C6-A926-8B47E10904AA}.Release|Mixed Platforms.Build.0 = Release|Any CPU {65BD1243-FCB9-40C6-A926-8B47E10904AA}.Release|x86.ActiveCfg = Release|Any CPU {65BD1243-FCB9-40C6-A926-8B47E10904AA}.Release|x86.Build.0 = Release|Any CPU + {55223D02-0CE9-4325-9A4D-5E88A544B633}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {55223D02-0CE9-4325-9A4D-5E88A544B633}.Debug|Any CPU.Build.0 = Debug|Any CPU + {55223D02-0CE9-4325-9A4D-5E88A544B633}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {55223D02-0CE9-4325-9A4D-5E88A544B633}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {55223D02-0CE9-4325-9A4D-5E88A544B633}.Debug|x86.ActiveCfg = Debug|Any CPU + {55223D02-0CE9-4325-9A4D-5E88A544B633}.Debug|x86.Build.0 = Debug|Any CPU + {55223D02-0CE9-4325-9A4D-5E88A544B633}.Release|Any CPU.ActiveCfg = Release|Any CPU + {55223D02-0CE9-4325-9A4D-5E88A544B633}.Release|Any CPU.Build.0 = Release|Any CPU + {55223D02-0CE9-4325-9A4D-5E88A544B633}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {55223D02-0CE9-4325-9A4D-5E88A544B633}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {55223D02-0CE9-4325-9A4D-5E88A544B633}.Release|x86.ActiveCfg = Release|Any CPU + {55223D02-0CE9-4325-9A4D-5E88A544B633}.Release|x86.Build.0 = Release|Any CPU {BBC31D50-5170-49D9-A42D-625768705B7B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {BBC31D50-5170-49D9-A42D-625768705B7B}.Debug|Any CPU.Build.0 = Debug|Any CPU {BBC31D50-5170-49D9-A42D-625768705B7B}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU From 260a04f0d924ec414fc4b86f622af0ef73f2238c Mon Sep 17 00:00:00 2001 From: Stephen Burman Date: Mon, 16 Oct 2023 16:01:22 +1100 Subject: [PATCH 8/8] Revert "Parallel testing POC" This reverts commit 930b5247f83c7f30e144ba347b5eab60a7ed008e. --- source/Halibut.sln | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/source/Halibut.sln b/source/Halibut.sln index 40800d4d7..d8e0f3781 100644 --- a/source/Halibut.sln +++ b/source/Halibut.sln @@ -72,18 +72,6 @@ Global {65BD1243-FCB9-40C6-A926-8B47E10904AA}.Release|Mixed Platforms.Build.0 = Release|Any CPU {65BD1243-FCB9-40C6-A926-8B47E10904AA}.Release|x86.ActiveCfg = Release|Any CPU {65BD1243-FCB9-40C6-A926-8B47E10904AA}.Release|x86.Build.0 = Release|Any CPU - {55223D02-0CE9-4325-9A4D-5E88A544B633}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {55223D02-0CE9-4325-9A4D-5E88A544B633}.Debug|Any CPU.Build.0 = Debug|Any CPU - {55223D02-0CE9-4325-9A4D-5E88A544B633}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU - {55223D02-0CE9-4325-9A4D-5E88A544B633}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU - {55223D02-0CE9-4325-9A4D-5E88A544B633}.Debug|x86.ActiveCfg = Debug|Any CPU - {55223D02-0CE9-4325-9A4D-5E88A544B633}.Debug|x86.Build.0 = Debug|Any CPU - {55223D02-0CE9-4325-9A4D-5E88A544B633}.Release|Any CPU.ActiveCfg = Release|Any CPU - {55223D02-0CE9-4325-9A4D-5E88A544B633}.Release|Any CPU.Build.0 = Release|Any CPU - {55223D02-0CE9-4325-9A4D-5E88A544B633}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU - {55223D02-0CE9-4325-9A4D-5E88A544B633}.Release|Mixed Platforms.Build.0 = Release|Any CPU - {55223D02-0CE9-4325-9A4D-5E88A544B633}.Release|x86.ActiveCfg = Release|Any CPU - {55223D02-0CE9-4325-9A4D-5E88A544B633}.Release|x86.Build.0 = Release|Any CPU {BBC31D50-5170-49D9-A42D-625768705B7B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {BBC31D50-5170-49D9-A42D-625768705B7B}.Debug|Any CPU.Build.0 = Debug|Any CPU {BBC31D50-5170-49D9-A42D-625768705B7B}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU