From de55fbf9d71e5cf0a89738903faae3547cfaaf04 Mon Sep 17 00:00:00 2001 From: Shankar Reddy Sama Date: Wed, 20 Dec 2023 17:00:15 -0800 Subject: [PATCH 1/2] [ServiceFabricProvider] Enbled customer serializer for reliable collections --- DurableTask.ServiceFabric.sln | 7 +- .../AssemblySetup.cs | 6 +- .../TestFabricApplication.sfproj | 13 +- .../TestFabricApplication/packages.config | 2 +- .../CustomDataContractStateSerializer.cs | 111 ++++++++++++++ .../CustomerXmlDictionaryReader.cs | 135 ++++++++++++++++++ .../TestStatefulService.cs | 7 + .../TestStatefulService.csproj | 4 + .../TaskMessageItem.cs | 14 +- 9 files changed, 286 insertions(+), 13 deletions(-) create mode 100644 Test/TestFabricApplication/TestStatefulService/CustomDataContractStateSerializer.cs create mode 100644 Test/TestFabricApplication/TestStatefulService/CustomerXmlDictionaryReader.cs diff --git a/DurableTask.ServiceFabric.sln b/DurableTask.ServiceFabric.sln index 3307fd48b..245448581 100644 --- a/DurableTask.ServiceFabric.sln +++ b/DurableTask.ServiceFabric.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.26403.7 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.34301.259 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{501E1168-418C-4832-B88C-617735BD02C9}" ProjectSection(SolutionItems) = preProject @@ -51,6 +51,7 @@ Global EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {CA924781-AE52-45FA-A817-C39C822FB782}.Debug|Any CPU.ActiveCfg = Debug|x64 + {CA924781-AE52-45FA-A817-C39C822FB782}.Debug|Any CPU.Build.0 = Debug|x64 {CA924781-AE52-45FA-A817-C39C822FB782}.Debug|x64.ActiveCfg = Debug|x64 {CA924781-AE52-45FA-A817-C39C822FB782}.Debug|x64.Build.0 = Debug|x64 {CA924781-AE52-45FA-A817-C39C822FB782}.Debug|x64.Deploy.0 = Debug|x64 @@ -59,6 +60,7 @@ Global {CA924781-AE52-45FA-A817-C39C822FB782}.Release|x64.Build.0 = Release|x64 {CA924781-AE52-45FA-A817-C39C822FB782}.Release|x64.Deploy.0 = Release|x64 {95D6C9C7-F7A0-4125-840D-1854BEC24978}.Debug|Any CPU.ActiveCfg = Debug|x64 + {95D6C9C7-F7A0-4125-840D-1854BEC24978}.Debug|Any CPU.Build.0 = Debug|x64 {95D6C9C7-F7A0-4125-840D-1854BEC24978}.Debug|x64.ActiveCfg = Debug|x64 {95D6C9C7-F7A0-4125-840D-1854BEC24978}.Debug|x64.Build.0 = Debug|x64 {95D6C9C7-F7A0-4125-840D-1854BEC24978}.Release|Any CPU.ActiveCfg = Release|x64 @@ -170,5 +172,6 @@ Global EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution EnterpriseLibraryConfigurationToolBinariesPath = packages\TransientFaultHandling.Core.5.1.1209.1\lib\NET4 + SolutionGuid = {BCE9A2EE-C1D3-4F37-8500-E112D840827F} EndGlobalSection EndGlobal diff --git a/Test/DurableTask.ServiceFabric.Test/AssemblySetup.cs b/Test/DurableTask.ServiceFabric.Test/AssemblySetup.cs index 9776be272..3271adf52 100644 --- a/Test/DurableTask.ServiceFabric.Test/AssemblySetup.cs +++ b/Test/DurableTask.ServiceFabric.Test/AssemblySetup.cs @@ -42,9 +42,9 @@ static string TestApplicationRootPath { get { - var sourceRoot = Environment.GetEnvironmentVariable("SourceRoot") ?? string.Empty; - var applicationPath = Path.Combine(sourceRoot.Trim(), "Test", "TestFabricApplication", "TestFabricApplication"); - + //var sourceRoot = Environment.GetEnvironmentVariable("SourceRoot") ?? string.Empty; + //var applicationPath = Path.Combine(sourceRoot.Trim(), "Test", "TestFabricApplication", "TestFabricApplication"); + var applicationPath = Path.Combine(Directory.GetCurrentDirectory(), @"..\..\..\TestFabricApplication\TestFabricApplication"); if (!Directory.Exists(applicationPath)) { throw new Exception("Could not find test application path, define SourceRoot environment variable to the source path"); diff --git a/Test/TestFabricApplication/TestFabricApplication/TestFabricApplication.sfproj b/Test/TestFabricApplication/TestFabricApplication/TestFabricApplication.sfproj index edcf3f147..4234f82d2 100644 --- a/Test/TestFabricApplication/TestFabricApplication/TestFabricApplication.sfproj +++ b/Test/TestFabricApplication/TestFabricApplication/TestFabricApplication.sfproj @@ -1,10 +1,11 @@  - + ca924781-ae52-45fa-a817-c39c822fb782 - 1.6 + 2.1 1.5 + 1.7.6 @@ -38,9 +39,9 @@ $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Service Fabric Tools\Microsoft.VisualStudio.Azure.Fabric.ApplicationProject.targets - - - - + + + + \ No newline at end of file diff --git a/Test/TestFabricApplication/TestFabricApplication/packages.config b/Test/TestFabricApplication/TestFabricApplication/packages.config index c5f59e48e..ad8665e2d 100644 --- a/Test/TestFabricApplication/TestFabricApplication/packages.config +++ b/Test/TestFabricApplication/TestFabricApplication/packages.config @@ -1,4 +1,4 @@  - + \ No newline at end of file diff --git a/Test/TestFabricApplication/TestStatefulService/CustomDataContractStateSerializer.cs b/Test/TestFabricApplication/TestStatefulService/CustomDataContractStateSerializer.cs new file mode 100644 index 000000000..e387010f0 --- /dev/null +++ b/Test/TestFabricApplication/TestStatefulService/CustomDataContractStateSerializer.cs @@ -0,0 +1,111 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + + +namespace TestStatefulService +{ + using System.IO; + using System.Runtime.Serialization; + using System.Xml; + using Microsoft.ServiceFabric.Data; + + internal class CustomDataContractStateSerializer : IStateSerializer + { + /// + /// The serializer. + /// + private readonly DataContractSerializer serializer = new DataContractSerializer(typeof(T)); + + /// + /// Converts byte[] to T + /// + /// Reader containing the serialized data. + /// Deserialized version of T. + public T Read(BinaryReader binaryReader) + { + var size = binaryReader.ReadInt32(); + + var bytes = binaryReader.ReadBytes(size); + + using (var memoryStream = new MemoryStream(bytes)) + { + using (var reader = XmlDictionaryReader.CreateBinaryReader(memoryStream, XmlDictionaryReaderQuotas.Max)) + using (var customReader = new CustomerXmlDictionaryReader(reader, typeof(T).Namespace)) + { + return (T) this.serializer.ReadObject(customReader); + } + } + } + + /// + /// Converts IEnumerable of byte[] to T + /// + /// + /// Reader containing the serialized data. + /// Deserialized version of T. + public T Read(T baseValue, BinaryReader reader) + { + return this.Read(reader); + } + + /// + /// Converts T to byte array. + /// + /// T to be serialized. + /// + /// Serialized version of T. + public void Write(T value, BinaryWriter binaryWriter) + { + using (var memoryStream = new MemoryStream()) + { + using (var innerWriter = new BinaryWriter(memoryStream)) + { + innerWriter.Write(int.MinValue); + + using ( + var binaryDictionaryWriter = XmlDictionaryWriter.CreateBinaryWriter( + memoryStream, + null, + null, + false)) + { + this.serializer.WriteObject(binaryDictionaryWriter, value); + binaryDictionaryWriter.Flush(); + } + + var lastPosition = (int) memoryStream.Position; + + memoryStream.Position = 0; + + innerWriter.Write(lastPosition - sizeof(int)); + + memoryStream.Position = lastPosition; + + binaryWriter.Write(memoryStream.GetBuffer(), 0, lastPosition); + } + } + } + + /// + /// Converts T to byte array. + /// + /// + /// Writer to which the serialized data should be written. + /// + /// Serialized version of T. + public void Write(T currentValue, T newValue, BinaryWriter binaryWriter) + { + this.Write(newValue, binaryWriter); + } + } +} \ No newline at end of file diff --git a/Test/TestFabricApplication/TestStatefulService/CustomerXmlDictionaryReader.cs b/Test/TestFabricApplication/TestStatefulService/CustomerXmlDictionaryReader.cs new file mode 100644 index 000000000..f3c81c659 --- /dev/null +++ b/Test/TestFabricApplication/TestStatefulService/CustomerXmlDictionaryReader.cs @@ -0,0 +1,135 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Xml; + +namespace TestStatefulService +{ + internal class CustomerXmlDictionaryReader : XmlDictionaryReader + { + XmlDictionaryReader innerReader; + string targetNamespace; + public CustomerXmlDictionaryReader(XmlDictionaryReader innerReader, string targetNamespace) + { + this.innerReader = innerReader; + this.targetNamespace = targetNamespace; + + } + + public override int AttributeCount => this.innerReader.AttributeCount; + + public override string BaseURI => this.innerReader.BaseURI; + + public override int Depth => this.innerReader.Depth; + + public override bool EOF => this.innerReader.EOF; + + public override bool IsEmptyElement => this.innerReader.IsEmptyElement; + + public override string LocalName => this.innerReader.LocalName; + + public override string NamespaceURI + { + get + { + // Alter the old namespace + UriBuilder builder = new UriBuilder(innerReader.NamespaceURI); + var segments = builder.Uri.Segments; + if (!segments.LastOrDefault().EndsWith("/") && segments.LastOrDefault() != this.targetNamespace) + { + segments[segments.Length - 1] = this.targetNamespace; + builder.Path = String.Join("", segments); + return builder.Uri.ToString(); + } + + return innerReader.NamespaceURI; + } + } + + public override XmlNameTable NameTable => this.innerReader.NameTable; + + public override XmlNodeType NodeType => this.innerReader.NodeType; + + public override string Prefix => this.innerReader.Prefix; + + public override ReadState ReadState => this.innerReader.ReadState; + + public override string Value => this.innerReader.Value; + + public override string GetAttribute(int i) + { + return this.innerReader.GetAttribute(i); + } + + public override string GetAttribute(string name) + { + return this.innerReader.GetAttribute(name); + } + + public override string GetAttribute(string name, string namespaceURI) + { + return this.innerReader.GetAttribute(name, namespaceURI); + } + + public override string LookupNamespace(string prefix) + { + return this.innerReader.LookupNamespace(prefix); + } + + public override bool MoveToAttribute(string name) + { + return this.innerReader.MoveToAttribute(name); + } + + public override bool MoveToAttribute(string name, string ns) + { + return this.innerReader.MoveToAttribute(name, ns); + } + + public override bool MoveToElement() + { + return this.innerReader.MoveToElement(); + } + + public override bool MoveToFirstAttribute() + { + return this.innerReader.MoveToFirstAttribute(); + } + + public override bool MoveToNextAttribute() + { + return this.innerReader.MoveToNextAttribute(); + } + + public override bool Read() + { + return this.innerReader.Read(); + } + + public override bool ReadAttributeValue() + { + return this.innerReader.ReadAttributeValue(); + } + + public override void ResolveEntity() + { + this.innerReader.ResolveEntity(); + } + } +} diff --git a/Test/TestFabricApplication/TestStatefulService/TestStatefulService.cs b/Test/TestFabricApplication/TestStatefulService/TestStatefulService.cs index 72c1e0029..76018fd63 100644 --- a/Test/TestFabricApplication/TestStatefulService/TestStatefulService.cs +++ b/Test/TestFabricApplication/TestStatefulService/TestStatefulService.cs @@ -18,6 +18,7 @@ using System.Threading; using System.Threading.Tasks; using DurableTask; +using DurableTask.History; using DurableTask.ServiceFabric; using DurableTask.Test.Orchestrations.Perf; using Microsoft.ServiceFabric.Services.Communication.Runtime; @@ -49,6 +50,12 @@ public TestStatefulService(StatefulServiceContext context) : base(context) this.fabricProviderFactory = new FabricOrchestrationProviderFactory(this.StateManager, settings); this.fabricProvider = this.fabricProviderFactory.CreateProvider(); this.client = new TaskHubClient(fabricProvider.OrchestrationServiceClient); + + this.StateManager.TryAddStateSerializer(new CustomDataContractStateSerializer()); + this.StateManager.TryAddStateSerializer(new CustomDataContractStateSerializer()); + this.StateManager.TryAddStateSerializer(new CustomDataContractStateSerializer()); + this.StateManager.TryAddStateSerializer(new CustomDataContractStateSerializer()); + this.StateManager.TryAddStateSerializer(new CustomDataContractStateSerializer()); } /// diff --git a/Test/TestFabricApplication/TestStatefulService/TestStatefulService.csproj b/Test/TestFabricApplication/TestStatefulService/TestStatefulService.csproj index 70bcc5acb..6f8672019 100644 --- a/Test/TestFabricApplication/TestStatefulService/TestStatefulService.csproj +++ b/Test/TestFabricApplication/TestStatefulService/TestStatefulService.csproj @@ -13,6 +13,7 @@ 512 true True + true @@ -113,8 +114,11 @@ ..\..\..\packages\Microsoft.AspNet.WebApi.Owin.5.2.3\lib\net45\System.Web.Http.Owin.dll True + + + diff --git a/src/DurableTask.ServiceFabric/TaskMessageItem.cs b/src/DurableTask.ServiceFabric/TaskMessageItem.cs index 55ff3d129..896f5be87 100644 --- a/src/DurableTask.ServiceFabric/TaskMessageItem.cs +++ b/src/DurableTask.ServiceFabric/TaskMessageItem.cs @@ -16,17 +16,29 @@ namespace DurableTask.ServiceFabric using System; using System.Runtime.Serialization; + /// + /// This wrapper is used with Service Fabric collections to persist TaskMessage. + /// [DataContract] - sealed class TaskMessageItem : IExtensibleDataObject + public sealed class TaskMessageItem : IExtensibleDataObject { + /// + /// This wrapper is used with Service Fabric collections to persist TaskMessage. + /// public TaskMessageItem(TaskMessage taskMessage) { this.TaskMessage = taskMessage ?? throw new ArgumentNullException(nameof(taskMessage)); } + /// + /// TaskMessage defined in TaskHub Core. + /// [DataMember] public TaskMessage TaskMessage { get; private set; } + /// + /// TaskMessage ExtensionData. + /// public ExtensionDataObject ExtensionData { get; set; } } } From 03d4b7f0218ab5174ea923a10638ce505bda90f6 Mon Sep 17 00:00:00 2001 From: Shankar Reddy Sama Date: Tue, 2 Jan 2024 17:17:47 -0800 Subject: [PATCH 2/2] Fix namespace manipulation issues --- .../CustomDataContractStateSerializer.cs | 2 +- .../CustomerXmlDictionaryReader.cs | 40 +++++++++++++++---- .../TestStatefulService.cs | 4 -- 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/Test/TestFabricApplication/TestStatefulService/CustomDataContractStateSerializer.cs b/Test/TestFabricApplication/TestStatefulService/CustomDataContractStateSerializer.cs index e387010f0..1a92ded0d 100644 --- a/Test/TestFabricApplication/TestStatefulService/CustomDataContractStateSerializer.cs +++ b/Test/TestFabricApplication/TestStatefulService/CustomDataContractStateSerializer.cs @@ -40,7 +40,7 @@ public T Read(BinaryReader binaryReader) using (var memoryStream = new MemoryStream(bytes)) { using (var reader = XmlDictionaryReader.CreateBinaryReader(memoryStream, XmlDictionaryReaderQuotas.Max)) - using (var customReader = new CustomerXmlDictionaryReader(reader, typeof(T).Namespace)) + using (var customReader = new CustomerXmlDictionaryReader(reader, typeof(T))) { return (T) this.serializer.ReadObject(customReader); } diff --git a/Test/TestFabricApplication/TestStatefulService/CustomerXmlDictionaryReader.cs b/Test/TestFabricApplication/TestStatefulService/CustomerXmlDictionaryReader.cs index f3c81c659..3774d59de 100644 --- a/Test/TestFabricApplication/TestStatefulService/CustomerXmlDictionaryReader.cs +++ b/Test/TestFabricApplication/TestStatefulService/CustomerXmlDictionaryReader.cs @@ -13,10 +13,10 @@ using System; +using System.Collections; using System.Collections.Generic; +using System.IO; using System.Linq; -using System.Text; -using System.Threading.Tasks; using System.Xml; namespace TestStatefulService @@ -24,12 +24,29 @@ namespace TestStatefulService internal class CustomerXmlDictionaryReader : XmlDictionaryReader { XmlDictionaryReader innerReader; - string targetNamespace; - public CustomerXmlDictionaryReader(XmlDictionaryReader innerReader, string targetNamespace) + Type targetType; + bool isTargetV2; + + static IDictionary namespaceMapV1toV2 = new Dictionary + { + { "DurableTask.ServiceFabric", "DurableTask.AzureServiceFabric" }, + { "DurableTask", "DurableTask.Core" }, + { "DurableTask.History", "DurableTask.Core.History" } + }; + + static IDictionary namespaceMapV2toV1 = new Dictionary + { + { "DurableTask.AzureServiceFabric", "DurableTask.ServiceFabric" }, + { "DurableTask.Core", "DurableTask" }, + { "DurableTask.Core.History", "DurableTask.History" } + }; + + public CustomerXmlDictionaryReader(XmlDictionaryReader innerReader, Type type) { this.innerReader = innerReader; - this.targetNamespace = targetNamespace; + this.targetType = type; + this.isTargetV2 = this.targetType.Namespace == "DurableTask.AzureServiceFabric" ? true : false; } public override int AttributeCount => this.innerReader.AttributeCount; @@ -50,11 +67,20 @@ public override string NamespaceURI { // Alter the old namespace UriBuilder builder = new UriBuilder(innerReader.NamespaceURI); + var segments = builder.Uri.Segments; - if (!segments.LastOrDefault().EndsWith("/") && segments.LastOrDefault() != this.targetNamespace) + var mapToUse = namespaceMapV2toV1; + string last = segments.LastOrDefault(); + if (isTargetV2) + { + mapToUse = namespaceMapV1toV2; + } + + if (mapToUse.ContainsKey(last)) { - segments[segments.Length - 1] = this.targetNamespace; + segments[segments.Length - 1] = mapToUse[last]; builder.Path = String.Join("", segments); + //File.AppendAllText(@"D:\git\durabletask\main\Test\TestFabricApplication\TestFabricApplication\SerializationInfo2.txt", $"OLD: {innerReader.NamespaceURI} NEW: {builder.Uri} {this.isTargetV2} {Environment.NewLine}"); return builder.Uri.ToString(); } diff --git a/Test/TestFabricApplication/TestStatefulService/TestStatefulService.cs b/Test/TestFabricApplication/TestStatefulService/TestStatefulService.cs index 76018fd63..27c3cc857 100644 --- a/Test/TestFabricApplication/TestStatefulService/TestStatefulService.cs +++ b/Test/TestFabricApplication/TestStatefulService/TestStatefulService.cs @@ -52,10 +52,6 @@ public TestStatefulService(StatefulServiceContext context) : base(context) this.client = new TaskHubClient(fabricProvider.OrchestrationServiceClient); this.StateManager.TryAddStateSerializer(new CustomDataContractStateSerializer()); - this.StateManager.TryAddStateSerializer(new CustomDataContractStateSerializer()); - this.StateManager.TryAddStateSerializer(new CustomDataContractStateSerializer()); - this.StateManager.TryAddStateSerializer(new CustomDataContractStateSerializer()); - this.StateManager.TryAddStateSerializer(new CustomDataContractStateSerializer()); } ///