Skip to content

Commit d949630

Browse files
muhammad-othmanmuhammad-othman
committed
Add CborPerformanceBenchmarks for Echo and SecretsManager services
1 parent 5bf6ad0 commit d949630

16 files changed

+524
-37
lines changed

‎sdk/test/Performance/CborPerformanceBenchmarks/CborPerformanceBenchmarks.sln‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AWSSDK.SecretsManager.NetSt
2323
EndProject
2424
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AWSSDK.Core.NetStandard", "..\..\..\..\..\aws-sdk-net-old-protocols\sdk\src\Core\AWSSDK.Core.NetStandard.csproj", "{90C1FD54-C1B9-474A-3CD0-6E7C9BBAC59E}"
2525
EndProject
26+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AWSSDK.Echo.NetStandard", "..\..\Services\Echo\AWSSDK.Echo.NetStandard.csproj", "{12C3757A-C7C0-F7C2-4B18-626A3F60C4A0}"
27+
EndProject
28+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AWSSDK.Echo.NetStandard", "..\..\..\..\..\aws-sdk-net-old-protocols\sdk\test\Services\Echo\AWSSDK.Echo.NetStandard.csproj", "{77CADAE9-F953-5A51-1788-0521FB7E7906}"
29+
EndProject
2630
Global
2731
GlobalSection(SolutionConfigurationPlatforms) = preSolution
2832
Debug|Any CPU = Debug|Any CPU
@@ -61,6 +65,14 @@ Global
6165
{90C1FD54-C1B9-474A-3CD0-6E7C9BBAC59E}.Debug|Any CPU.Build.0 = Debug|Any CPU
6266
{90C1FD54-C1B9-474A-3CD0-6E7C9BBAC59E}.Release|Any CPU.ActiveCfg = Release|Any CPU
6367
{90C1FD54-C1B9-474A-3CD0-6E7C9BBAC59E}.Release|Any CPU.Build.0 = Release|Any CPU
68+
{12C3757A-C7C0-F7C2-4B18-626A3F60C4A0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
69+
{12C3757A-C7C0-F7C2-4B18-626A3F60C4A0}.Debug|Any CPU.Build.0 = Debug|Any CPU
70+
{12C3757A-C7C0-F7C2-4B18-626A3F60C4A0}.Release|Any CPU.ActiveCfg = Release|Any CPU
71+
{12C3757A-C7C0-F7C2-4B18-626A3F60C4A0}.Release|Any CPU.Build.0 = Release|Any CPU
72+
{77CADAE9-F953-5A51-1788-0521FB7E7906}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
73+
{77CADAE9-F953-5A51-1788-0521FB7E7906}.Debug|Any CPU.Build.0 = Debug|Any CPU
74+
{77CADAE9-F953-5A51-1788-0521FB7E7906}.Release|Any CPU.ActiveCfg = Release|Any CPU
75+
{77CADAE9-F953-5A51-1788-0521FB7E7906}.Release|Any CPU.Build.0 = Release|Any CPU
6476
EndGlobalSection
6577
GlobalSection(SolutionProperties) = preSolution
6678
HideSolutionNode = FALSE
@@ -73,6 +85,8 @@ Global
7385
{8651B417-2687-A38D-F39E-DF0D0C48A8A5} = {938A4185-962C-4ABB-A344-97BAA2B8C64A}
7486
{EF094BCE-D7B0-97C9-A335-7BDFF0F3DE8F} = {938A4185-962C-4ABB-A344-97BAA2B8C64A}
7587
{90C1FD54-C1B9-474A-3CD0-6E7C9BBAC59E} = {938A4185-962C-4ABB-A344-97BAA2B8C64A}
88+
{12C3757A-C7C0-F7C2-4B18-626A3F60C4A0} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
89+
{77CADAE9-F953-5A51-1788-0521FB7E7906} = {938A4185-962C-4ABB-A344-97BAA2B8C64A}
7690
EndGlobalSection
7791
GlobalSection(ExtensibilityGlobals) = postSolution
7892
SolutionGuid = {C32806F1-F388-4C10-A94F-4BCC88584182}

‎sdk/test/Performance/CborPerformanceBenchmarks/CborPerformanceBenchmarksRunner/BaseBenchmarks.cs‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
using System.Text;
88

99
[MemoryDiagnoser]
10-
[Orderer(SummaryOrderPolicy.Method)]
1110
public abstract class BaseBenchmarks
1211
{
1312
public abstract int DimensionValue { get; set; }
@@ -47,17 +46,21 @@ public virtual void AfterMarshall()
4746
#if USE_CBOR
4847
RequestSizeBytes = Math.Max(RequestSizeBytes, MarshalledRequest.Content.Length);
4948
#else
50-
if (Service == "CloudWatch")
49+
if (Protocol == "Query")
5150
{
5251
string queryString = Utils.GetParametersAsString(MarshalledRequest.ParameterCollection);
5352
var content = Encoding.UTF8.GetBytes(queryString);
5453

5554
RequestSizeBytes = Math.Max(RequestSizeBytes, content.Length);
5655
}
57-
else
56+
else if (Protocol == "JSON")
5857
{
5958
RequestSizeBytes = Math.Max(RequestSizeBytes, MarshalledRequest.Content.Length);
6059
}
60+
else
61+
{
62+
throw new NotImplementedException();
63+
}
6164
#endif
6265
var record = new BenchmarkRecord(Service, TestCase, Protocol, DimensionValue, "Request payload size (bytes)", RequestSizeBytes, RequestSizeBytes, RequestSizeBytes);
6366
Utils.StoreBenchmarkRecords(new List<BenchmarkRecord> { record });

‎sdk/test/Performance/CborPerformanceBenchmarks/CborPerformanceBenchmarksRunner/CborPerformanceBenchmarksRunner.csproj‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,15 @@
1212
<ProjectReference Include="..\..\..\..\..\..\aws-sdk-net-old-protocols\sdk\src\Core\AWSSDK.Core.NetStandard.csproj" />
1313
<ProjectReference Include="..\..\..\..\..\..\aws-sdk-net-old-protocols\sdk\src\Services\CloudWatch\AWSSDK.CloudWatch.NetStandard.csproj" />
1414
<ProjectReference Include="..\..\..\..\..\..\aws-sdk-net-old-protocols\sdk\src\Services\SecretsManager\AWSSDK.SecretsManager.NetStandard.csproj" />
15+
<ProjectReference Include="..\..\..\..\..\..\aws-sdk-net-old-protocols\sdk\test\Services\Echo\AWSSDK.Echo.NetStandard.csproj" />
1516
</ItemGroup>
1617

1718
<ItemGroup Condition="'$(UseCbor)' == 'true'">
1819
<ProjectReference Include="..\..\..\..\..\extensions\src\AWSSDK.Extensions.CborProtocol\AWSSDK.Extensions.CborProtocol.NetStandard.csproj" />
1920
<ProjectReference Include="..\..\..\..\src\Core\AWSSDK.Core.NetStandard.csproj" />
2021
<ProjectReference Include="..\..\..\..\src\Services\CloudWatch\AWSSDK.CloudWatch.NetStandard.csproj" />
2122
<ProjectReference Include="..\..\..\..\src\Services\SecretsManager\AWSSDK.SecretsManager.NetStandard.csproj" />
23+
<ProjectReference Include="..\..\..\Services\Echo\AWSSDK.Echo.NetStandard.csproj" />
2224
</ItemGroup>
2325

2426
<ItemGroup>

‎sdk/test/Performance/CborPerformanceBenchmarks/CborPerformanceBenchmarksRunner/CloudWatchBenchmarks/ListMetricsBenchmarks.cs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,6 @@ public override void GenerateRequest()
6464
};
6565
}
6666

67-
6867
[Benchmark(Description = "Deserialization time (ms)")]
6968
public override AmazonWebServiceResponse Unmarshall()
7069
{
@@ -74,9 +73,9 @@ public override AmazonWebServiceResponse Unmarshall()
7473

7574
public override void TotalRequestBeforeIteration()
7675
{
77-
if (IterationCount % 50 == 0)
76+
if (IterationCount % 25 == 0)
7877
{
79-
Thread.Sleep(2000); // Sleep 2 seconds every 50 iterations to avoid throttling
78+
Thread.Sleep(2000); // Sleep 2 seconds every 25 iterations to avoid throttling
8079
}
8180
base.TotalRequestBeforeIteration();
8281
}

‎sdk/test/Performance/CborPerformanceBenchmarks/CborPerformanceBenchmarksRunner/CustomColumns.cs‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
using BenchmarkDotNet.Columns;
2+
using BenchmarkDotNet.Reports;
3+
using BenchmarkDotNet.Running;
4+
5+
public class ServiceColumn : IColumn
6+
{
7+
public virtual string Id => nameof(ServiceColumn);
8+
public virtual string ColumnName => "Service";
9+
public bool AlwaysShow => true;
10+
public ColumnCategory Category => ColumnCategory.Job;
11+
public virtual int PriorityInCategory => 0;
12+
public bool IsNumeric => false;
13+
public UnitType UnitType => UnitType.Dimensionless;
14+
15+
public string Legend => "";
16+
17+
public bool IsAvailable(Summary summary) => true;
18+
19+
public virtual string GetValue(Summary summary, BenchmarkCase benchmarkCase)
20+
{
21+
var benchmarkObject = (BaseBenchmarks)Activator.CreateInstance(benchmarkCase.Descriptor.Type);
22+
return benchmarkObject.Service;
23+
}
24+
25+
public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style)
26+
=> GetValue(summary, benchmarkCase);
27+
28+
public override string ToString() => ColumnName;
29+
30+
public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase)
31+
{
32+
return false;
33+
}
34+
}
35+
36+
public class TestCaseColumn : ServiceColumn
37+
{
38+
public override string Id => nameof(TestCaseColumn);
39+
public override string ColumnName => "Test Case";
40+
41+
public override string GetValue(Summary summary, BenchmarkCase benchmarkCase)
42+
{
43+
var benchmarkObject = (BaseBenchmarks)Activator.CreateInstance(benchmarkCase.Descriptor.Type);
44+
var testCase = benchmarkObject.TestCase;
45+
46+
return testCase;
47+
}
48+
}
49+
50+
public class BenchmarkColumn : ServiceColumn
51+
{
52+
public override string Id => nameof(BenchmarkColumn);
53+
public override string ColumnName => "Benchmark";
54+
public override string GetValue(Summary summary, BenchmarkCase benchmarkCase)
55+
{
56+
var displayInfo = benchmarkCase.Descriptor.WorkloadMethodDisplayInfo.Replace(" (ms)", "").Replace("'", "").Replace("'", "");
57+
return displayInfo;
58+
}
59+
}

‎sdk/test/Performance/CborPerformanceBenchmarks/CborPerformanceBenchmarksRunner/EchoBenchmarks/AllTypesBenchmarks.cs‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
using Amazon;
2+
using Amazon.Echo;
3+
using Amazon.Echo.Model;
4+
using Amazon.Echo.Model.Internal.MarshallTransformations;
5+
using Amazon.Runtime;
6+
using Amazon.Util;
7+
using BenchmarkDotNet.Attributes;
8+
using BenchmarkDotNet.Order;
9+
using System;
10+
using System.Text;
11+
12+
namespace CborPerformanceBenchmarksRunner.EchoBenchmarks;
13+
14+
public class AllTypesBenchmarks : BaseBenchmarks
15+
{
16+
protected Random _random = new Random();
17+
18+
[Params(0)]
19+
public override int DimensionValue { get; set; }
20+
public override string Service { get; } = "EchoService";
21+
public override string TestCase { get; } = "All types";
22+
23+
protected IAmazonEcho _client;
24+
public override void CreateServiceClient()
25+
{
26+
_client = new AmazonEchoClient(new AmazonEchoConfig
27+
{
28+
DisableRequestCompression = true,
29+
MaxErrorRetry = 0,
30+
RegionEndpoint = RegionEndpoint.USWest2,
31+
});
32+
}
33+
34+
protected EchoOperationRequest request;
35+
36+
[Benchmark(Description = "Serialization time (ms)")]
37+
public override byte[] Marshall()
38+
{
39+
var iRequest = EchoOperationRequestMarshaller.Instance.Marshall(request);
40+
MarshalledRequest = iRequest;
41+
42+
return iRequest.Content;
43+
}
44+
45+
[Benchmark(Description = "Total request time (ms)")]
46+
public async override Task<AmazonWebServiceResponse> TotalRequest()
47+
{
48+
var response = await _client.EchoOperationAsync(request);
49+
ResponseSizeBytes = Math.Max(ResponseSizeBytes, response.ContentLength);
50+
51+
return response;
52+
}
53+
54+
public override void GenerateRequest()
55+
{
56+
request = new EchoOperationRequest
57+
{
58+
BooleanMember = _random.Next(2) == 0,
59+
StringMember = GenerateRandomASCIIString(32),
60+
IntegerMember = _random.Next(int.MinValue, int.MaxValue),
61+
LongMember = GenerateRandomLong(),
62+
FloatMember = GenerateRandomFloat(),
63+
DoubleMember = GenerateRandomDouble(),
64+
TimestampMember = DateTimeOffset.FromUnixTimeSeconds(Utils.RunStartTimestamp).UtcDateTime,
65+
BlobMember = new MemoryStream(GenerateRandomBytes(128)),
66+
ListOfStringsMember = Enumerable.Range(0, 8)
67+
.Select(_ => GenerateRandomASCIIString(32))
68+
.ToList(),
69+
MapOfStringToStringMember = Enumerable.Range(0, 8)
70+
.ToDictionary(
71+
_ => GenerateRandomASCIIString(32),
72+
_ => GenerateRandomASCIIString(64)
73+
),
74+
ComplexStructMember = new ComplexStructure
75+
{
76+
StringMember = GenerateRandomASCIIString(32),
77+
ComplexStructMember = new ComplexStructure
78+
{
79+
StringMember = GenerateRandomASCIIString(32)
80+
}
81+
}
82+
83+
};
84+
}
85+
protected long GenerateRandomLong()
86+
{
87+
var buffer = new byte[8];
88+
_random.NextBytes(buffer);
89+
return BitConverter.ToInt64(buffer, 0);
90+
}
91+
92+
protected float GenerateRandomFloat()
93+
{
94+
var bytes = new byte[4];
95+
_random.NextBytes(bytes);
96+
return BitConverter.ToSingle(bytes, 0);
97+
}
98+
99+
protected double GenerateRandomDouble()
100+
{
101+
var bytes = new byte[8];
102+
_random.NextBytes(bytes);
103+
return BitConverter.ToDouble(bytes, 0);
104+
}
105+
106+
protected byte[] GenerateRandomBytes(int length)
107+
{
108+
var buffer = new byte[length];
109+
_random.NextBytes(buffer);
110+
return buffer;
111+
}
112+
113+
protected string GenerateRandomASCIIString(int length = 32)
114+
{
115+
var sb = new StringBuilder(length);
116+
117+
for (int i = 0; i < length; i++)
118+
{
119+
// ASCII visible characters from 33 to 126
120+
char randomChar = (char)_random.Next(33, 127);
121+
sb.Append(randomChar);
122+
}
123+
124+
return sb.ToString();
125+
}
126+
127+
128+
[Benchmark(Description = "Deserialization time (ms)")]
129+
public override AmazonWebServiceResponse Unmarshall()
130+
{
131+
var response = EchoOperationResponseUnmarshaller.Instance.Unmarshall(UnmarshallerContext);
132+
return response;
133+
}
134+
}

‎sdk/test/Performance/CborPerformanceBenchmarks/CborPerformanceBenchmarksRunner/EchoBenchmarks/ComplexObjectBenchmarks.cs‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
using Amazon.Echo.Model;
2+
using BenchmarkDotNet.Attributes;
3+
using BenchmarkDotNet.Order;
4+
5+
namespace CborPerformanceBenchmarksRunner.EchoBenchmarks;
6+
7+
public class ComplexObjectBenchmarks : AllTypesBenchmarks
8+
{
9+
public override string TestCase { get; } = "Complex object";
10+
11+
public override void GenerateRequest()
12+
{
13+
request = new EchoOperationRequest
14+
{
15+
ComplexStructMember = new ComplexStructure
16+
{
17+
BooleanMember = _random.Next(2) == 0,
18+
BlobMember = new MemoryStream(GenerateRandomBytes(128)),
19+
StringMember = GenerateRandomASCIIString(32),
20+
ComplexStructMember = new ComplexStructure
21+
{
22+
IntegerMember = _random.Next(int.MinValue, int.MaxValue),
23+
LongMember = GenerateRandomLong(),
24+
StringMember = GenerateRandomASCIIString(32),
25+
ComplexStructMember = new ComplexStructure
26+
{
27+
FloatMember = GenerateRandomFloat(),
28+
DoubleMember = GenerateRandomDouble(),
29+
StringMember = GenerateRandomASCIIString(32),
30+
ComplexStructMember = new ComplexStructure
31+
{
32+
ListOfStringsMember = Enumerable.Range(0, 8)
33+
.Select(_ => GenerateRandomASCIIString(32))
34+
.ToList(),
35+
MapOfStringToStringMember = Enumerable.Range(0, 8)
36+
.ToDictionary(
37+
_ => GenerateRandomASCIIString(32),
38+
_ => GenerateRandomASCIIString(64)
39+
),
40+
}
41+
42+
}
43+
44+
}
45+
}
46+
47+
};
48+
}
49+
}

‎sdk/test/Performance/CborPerformanceBenchmarks/CborPerformanceBenchmarksRunner/EchoBenchmarks/ListOfComplexObjectBenchmarks.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
using Amazon.Echo.Model;
2+
3+
namespace CborPerformanceBenchmarksRunner.EchoBenchmarks;
4+
5+
public class ListOfComplexObjectBenchmarks : AllTypesBenchmarks
6+
{
7+
public override string TestCase { get; } = "List of complex objects";
8+
public override void GenerateRequest()
9+
{
10+
request = new EchoOperationRequest
11+
{
12+
ListOfComplexObjectMember = Enumerable.Range(0, 64).Select(_ => new ComplexStructure
13+
{
14+
BooleanMember = _random.Next(2) == 0,
15+
StringMember = GenerateRandomASCIIString(32),
16+
LongMember = GenerateRandomLong(),
17+
DoubleMember = GenerateRandomDouble(),
18+
TimestampMember = DateTimeOffset.FromUnixTimeSeconds(Utils.RunStartTimestamp).UtcDateTime,
19+
ListOfStringsMember = Enumerable.Range(0, 8)
20+
.Select(_ => GenerateRandomASCIIString(32))
21+
.ToList(),
22+
}).ToList()
23+
};
24+
}
25+
}

0 commit comments

Comments
 (0)