From 513f3c6bf5d69d11a9ff4f329453eb3aa9b57351 Mon Sep 17 00:00:00 2001 From: Ali Robertson Date: Thu, 4 Sep 2025 13:49:26 +1000 Subject: [PATCH 1/3] Add Copy-PnPFileMetadata cmdlet and docs --- documentation/Copy-PnPFileMetadata.md | 187 +++++++ src/Commands/Files/CopyFileMetadata.cs | 562 ++++++++++++++++++++ src/Tests/Files/CopyPnPFileMetadataTests.cs | 30 ++ 3 files changed, 779 insertions(+) create mode 100644 documentation/Copy-PnPFileMetadata.md create mode 100644 src/Commands/Files/CopyFileMetadata.cs create mode 100644 src/Tests/Files/CopyPnPFileMetadataTests.cs diff --git a/documentation/Copy-PnPFileMetadata.md b/documentation/Copy-PnPFileMetadata.md new file mode 100644 index 000000000..dac568544 --- /dev/null +++ b/documentation/Copy-PnPFileMetadata.md @@ -0,0 +1,187 @@ +--- +Module Name: PnP.PowerShell +schema: 2.0.0 +applicable: SharePoint Online +online version: https://pnp.github.io/powershell/cmdlets/Copy-PnPFileMetadata.html +external help file: PnP.PowerShell.dll-Help.xml +title: Copy-PnPFileMetadata +--- + +# Copy-PnPFileMetadata + +## SYNOPSIS + +Synchronizes metadata between files and folders in SharePoint + +## SYNTAX + +```powershell +Copy-PnPFileMetadata [-SourceUrl] [-TargetUrl] [-Fields ] [-Recursive] [-Force] [-Connection ] [-SourceConnection ] [-TargetConnection ] +``` + +## DESCRIPTION + +Synchronizes metadata (Created, Modified, Author, Editor) from source files and folders to their corresponding targets without copying the actual content. This cmdlet is particularly useful for restoring lost metadata after migration operations where system fields may have been reset. + +When updating items, the cmdlet uses `UpdateOverwriteVersion()` to allow setting system fields while avoiding new user-facing versions. + +For folder contents, files are processed one-by-one. + +## EXAMPLES + +### EXAMPLE 1 + +```powershell +Copy-PnPFileMetadata -SourceUrl "Shared Documents/MyProject" -TargetUrl "/sites/target/Shared Documents/MyProject" +``` + +Synchronizes metadata for the MyProject folder and all its contents recursively from the source to the target location, preserving original creation dates, modification dates, and author information. + +### EXAMPLE 2 + +```powershell +Copy-PnPFileMetadata -SourceUrl "Shared Documents/report.docx" -TargetUrl "/sites/archive/Documents/report.docx" +``` + +Synchronizes metadata for a single file from the source to the target, restoring the original system fields. + +### EXAMPLE 3 + +```powershell +Copy-PnPFileMetadata -SourceUrl "Shared Documents/Projects" -TargetUrl "/sites/backup/Documents/Projects" -Fields @("Created", "Modified") -Force +``` + +Synchronizes only the Created and Modified dates for the Projects folder and its contents, without prompting for confirmation. + +### EXAMPLE 4 + +```powershell +Copy-PnPFileMetadata -SourceUrl "Shared Documents/Archives" -TargetUrl "/sites/newsite/Documents/Archives" -Recursive:$false +``` + +Synchronizes metadata only for the Archives folder itself, without processing its subfolders and files. + +### EXAMPLE 5 + +```powershell +$src = Connect-PnPOnline -Url https://contoso.sharepoint.com/sites/archives -ReturnConnection +$dst = Connect-PnPOnline -Url https://contoso.sharepoint.com/sites/projects -ReturnConnection +Copy-PnPFileMetadata -SourceUrl "Shared Documents/MyProject" -TargetUrl "Shared Documents/MyProject" -SourceConnection $src -TargetConnection $dst -Verbose +``` + +Synchronizes metadata across two different site connections. + +## PARAMETERS + +### -Force + +If provided, no confirmation will be requested and the action will be performed + +```yaml +Type: SwitchParameter +Parameter Sets: (All) + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Fields + +Specifies which metadata fields to synchronize. Default fields are Created, Modified, Author, and Editor. + +```yaml +Type: String[] +Parameter Sets: (All) + +Required: False +Position: Named +Default value: @("Created", "Modified", "Author", "Editor") +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Recursive + +If provided, processes folders recursively including all subfolders and files. This is enabled by default. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) + +Required: False +Position: Named +Default value: $true +Accept pipeline input: False +Accept wildcard characters: False +``` + + + +### -SourceUrl + +Site or server relative URL specifying the file or folder to copy metadata from. Must include the file name if it is a file or the entire path to the folder if it is a folder. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: ServerRelativeUrl + +Required: True +Position: 0 +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -TargetUrl + +Site or server relative URL specifying the file or folder to copy metadata to. Must include the file name if it is a file or the entire path to the folder if it is a folder. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: TargetServerRelativeUrl + +Required: True +Position: 1 +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -SourceConnection + +Optional connection to be used for accessing the source file or folder. If not provided, the current connection is used. + +```yaml +Type: PnPConnection +Parameter Sets: (All) + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -TargetConnection + +Optional connection to be used for accessing the target file or folder. If not provided, the current connection is used. + +```yaml +Type: PnPConnection +Parameter Sets: (All) + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +## RELATED LINKS + +[Microsoft 365 Patterns and Practices](https://aka.ms/m365pnp) diff --git a/src/Commands/Files/CopyFileMetadata.cs b/src/Commands/Files/CopyFileMetadata.cs new file mode 100644 index 000000000..4add141e0 --- /dev/null +++ b/src/Commands/Files/CopyFileMetadata.cs @@ -0,0 +1,562 @@ +using System; +using System.Management.Automation; +using Microsoft.SharePoint.Client; +using Resources = PnP.PowerShell.Commands.Properties.Resources; +using PnP.Framework.Utilities; +using PnP.PowerShell.Commands.Base; + +namespace PnP.PowerShell.Commands.Files +{ + [Cmdlet(VerbsCommon.Copy, "PnPFileMetadata")] + public class CopyFileMetadata : PnPWebCmdlet + { + // Cache for resolved users on the target to minimize EnsureUser calls + private readonly System.Collections.Generic.Dictionary _targetUserCache = new System.Collections.Generic.Dictionary(StringComparer.OrdinalIgnoreCase); + + [Parameter(Mandatory = true, Position = 0, ValueFromPipeline = true)] + [Alias("ServerRelativeUrl")] + public string SourceUrl = string.Empty; + + [Parameter(Mandatory = true, Position = 1)] + [Alias("TargetServerRelativeUrl")] + public string TargetUrl = string.Empty; + + [Parameter(Mandatory = false)] + public SwitchParameter Force; + + [Parameter(Mandatory = false)] + public string[] Fields = new[] { "Created", "Modified", "Author", "Editor" }; + + [Parameter(Mandatory = false)] + public SwitchParameter Recursive = SwitchParameter.Present; + + [Parameter(Mandatory = false, HelpMessage = "Optional connection to be used for accessing the source file. If not provided, uses the current connection.")] + public PnPConnection SourceConnection = null; + + [Parameter(Mandatory = false, HelpMessage = "Optional connection to be used for accessing the target file. If not provided, uses the current connection.")] + public PnPConnection TargetConnection = null; + + protected override void ExecuteCmdlet() + { + // Get the contexts for source and target operations + var sourceContext = GetSourceContext(); + var targetContext = GetTargetContext(); + + // Ensure we have valid contexts + if (sourceContext == null) + { + throw new InvalidOperationException("Source connection is not available. Please provide a valid SourceConnection or ensure you are connected."); + } + + if (targetContext == null) + { + throw new InvalidOperationException("Target connection is not available. Please provide a valid TargetConnection or ensure you are connected."); + } + + var sourceWebServerRelativeUrl = sourceContext.Web.EnsureProperty(w => w.ServerRelativeUrl); + var targetWebServerRelativeUrl = targetContext.Web.EnsureProperty(w => w.ServerRelativeUrl); + + WriteVerbose($"Source context web URL: {sourceWebServerRelativeUrl}"); + WriteVerbose($"Target context web URL: {targetWebServerRelativeUrl}"); + + // Process URLs relative to their respective contexts + SourceUrl = ProcessUrl(SourceUrl, sourceWebServerRelativeUrl); + TargetUrl = ProcessUrl(TargetUrl, targetWebServerRelativeUrl); + + WriteVerbose($"Processed SourceUrl: {SourceUrl}"); + WriteVerbose($"Processed TargetUrl: {TargetUrl}"); + + if (Force || ShouldContinue(string.Format("Synchronize metadata from '{0}' to '{1}'", SourceUrl, TargetUrl), Resources.Confirm)) + { + try + { + WriteVerbose($"Starting metadata synchronization from '{SourceUrl}' to '{TargetUrl}'"); + WriteVerbose($"Fields to sync: {string.Join(", ", Fields)}"); + + // Determine if source is a file or folder (using source context) + var sourceItem = GetFileOrFolderInfo(SourceUrl, sourceContext); + if (sourceItem.Name == null) + { + WriteError(new ErrorRecord( + new PSArgumentException($"Source path '{SourceUrl}' not found."), + "SourceNotFound", + ErrorCategory.ObjectNotFound, + SourceUrl)); + return; + } + + var itemsProcessed = 0; + var itemsSkipped = 0; + var itemsErrored = 0; + + if (sourceItem.IsFile) + { + var result = SyncFileMetadata(SourceUrl, TargetUrl, sourceContext, targetContext); + if (result == SyncResult.Success) itemsProcessed++; + else if (result == SyncResult.Skipped) itemsSkipped++; + else itemsErrored++; + } + else + { + var results = SyncFolderMetadataRecursive(SourceUrl, TargetUrl, sourceContext, targetContext); + itemsProcessed = results.Processed; + itemsSkipped = results.Skipped; + itemsErrored = results.Errored; + } + + WriteObject($"Metadata synchronization completed. Processed: {itemsProcessed}, Skipped: {itemsSkipped}, Errors: {itemsErrored}"); + } + catch (Exception ex) + { + WriteError(new ErrorRecord(ex, "MetadataSyncError", ErrorCategory.InvalidOperation, SourceUrl)); + } + } + } + + private (int Processed, int Skipped, int Errored) SyncFolderMetadataRecursive(string sourceFolderUrl, string targetFolderUrl) + { + return SyncFolderMetadataRecursive(sourceFolderUrl, targetFolderUrl, ClientContext, ClientContext); + } + + private (int Processed, int Skipped, int Errored) SyncFolderMetadataRecursive(string sourceFolderUrl, string targetFolderUrl, ClientContext sourceContext, ClientContext targetContext) + { + var processed = 0; + var skipped = 0; + var errored = 0; + + try + { + WriteVerbose($"Processing folder: {sourceFolderUrl}"); + + // Get source and target folder via ResourcePath and validate existence + var sourceFolder = sourceContext.Web.GetFolderByServerRelativePath(ResourcePath.FromDecodedUrl(Uri.UnescapeDataString(sourceFolderUrl))); + var targetFolder = targetContext.Web.GetFolderByServerRelativePath(ResourcePath.FromDecodedUrl(Uri.UnescapeDataString(targetFolderUrl))); + + // Step 1: check existence first to avoid exceptions when loading list item of a non-existing folder + sourceContext.Load(sourceFolder, f => f.Name, f => f.Exists); + targetContext.Load(targetFolder, f => f.Name, f => f.Exists); + sourceContext.ExecuteQueryRetry(); + targetContext.ExecuteQueryRetry(); + + if (!sourceFolder.Exists) + { + WriteWarning($"Source folder not found: '{sourceFolderUrl}'"); + errored++; + return (processed, skipped, errored); + } + + if (!targetFolder.Exists) + { + WriteWarning($"Target folder not found, skipping subtree: '{targetFolderUrl}'"); + skipped++; + return (processed, skipped, errored); + } + + // Step 2: load heavy properties once existence is confirmed + sourceContext.Load(sourceFolder, f => f.ListItemAllFields, f => f.Files, f => f.Folders); + targetContext.Load(targetFolder, f => f.ListItemAllFields); + sourceContext.ExecuteQueryRetry(); + targetContext.ExecuteQueryRetry(); + + // Sync folder metadata if both have list items + if (!sourceFolder.ListItemAllFields.ServerObjectIsNull.GetValueOrDefault() && + !targetFolder.ListItemAllFields.ServerObjectIsNull.GetValueOrDefault()) + { + var result = SyncListItemMetadata(sourceFolder.ListItemAllFields, targetFolder.ListItemAllFields, sourceContext, targetContext); + if (result == SyncResult.Success) processed++; + else if (result == SyncResult.Skipped) skipped++; + else errored++; + } + + // Process files + if (sourceFolder.Files != null && sourceFolder.Files.Count > 0) + { + foreach (var sourceFile in sourceFolder.Files) + { + var src = UrlUtility.Combine(sourceFolderUrl, sourceFile.Name); + var dst = UrlUtility.Combine(targetFolderUrl, sourceFile.Name); + var r = SyncFileMetadata(src, dst, sourceContext, targetContext); + if (r == SyncResult.Success) processed++; + else if (r == SyncResult.Skipped) skipped++; + else errored++; + } + } + + // Process subfolders recursively if Recursive is enabled + if (Recursive) + { + foreach (var sourceSubfolder in sourceFolder.Folders) + { + if (sourceSubfolder.Name.StartsWith("_")) continue; // Skip system folders + + var sourceSubfolderUrlCombined = UrlUtility.Combine(sourceFolderUrl, sourceSubfolder.Name); + var targetSubfolderUrl = UrlUtility.Combine(targetFolderUrl, sourceSubfolder.Name); + var subResults = SyncFolderMetadataRecursive(sourceSubfolderUrlCombined, targetSubfolderUrl, sourceContext, targetContext); + + processed += subResults.Processed; + skipped += subResults.Skipped; + errored += subResults.Errored; + } + } + } + catch (Exception ex) + { + WriteWarning($"Failed to process folder '{sourceFolderUrl}': {ex.Message}"); + errored++; + } + + return (processed, skipped, errored); + } + + private SyncResult SyncFileMetadata(string sourceFileUrl, string targetFileUrl) + { + return SyncFileMetadata(sourceFileUrl, targetFileUrl, ClientContext, ClientContext); + } + + private SyncResult SyncFileMetadata(string sourceFileUrl, string targetFileUrl, ClientContext sourceContext, ClientContext targetContext) + { + try + { + WriteVerbose($"Syncing file metadata: {sourceFileUrl} -> {targetFileUrl}"); + // Decode and use server-relative ResourcePath to reliably check existence + var decodedSourceUrl = Uri.UnescapeDataString(sourceFileUrl); + var decodedTargetUrl = Uri.UnescapeDataString(targetFileUrl); + + var sourceFile = sourceContext.Web.GetFileByServerRelativePath(ResourcePath.FromDecodedUrl(decodedSourceUrl)); + var targetFile = targetContext.Web.GetFileByServerRelativePath(ResourcePath.FromDecodedUrl(decodedTargetUrl)); + + sourceContext.Load(sourceFile, f => f.Exists, f => f.Name, f => f.ListItemAllFields); + targetContext.Load(targetFile, f => f.Exists, f => f.Name, f => f.ListItemAllFields); + sourceContext.ExecuteQueryRetry(); + targetContext.ExecuteQueryRetry(); + + if (!sourceFile.Exists) + { + WriteWarning($"Source file not found: '{decodedSourceUrl}'"); + return SyncResult.Error; + } + + if (!targetFile.Exists) + { + WriteWarning($"Target file not found, skipping: '{decodedTargetUrl}'"); + return SyncResult.Skipped; + } + + if (!sourceFile.ListItemAllFields.ServerObjectIsNull.GetValueOrDefault() && + !targetFile.ListItemAllFields.ServerObjectIsNull.GetValueOrDefault()) + { + return SyncListItemMetadata(sourceFile.ListItemAllFields, targetFile.ListItemAllFields, sourceContext, targetContext); + } + else + { + WriteWarning($"Cannot access list item for file '{sourceFileUrl}' or '{targetFileUrl}'"); + return SyncResult.Skipped; + } + } + catch (Exception ex) + { + WriteWarning($"Failed to sync file metadata from '{sourceFileUrl}' to '{targetFileUrl}': {ex.Message}"); + return SyncResult.Error; + } + } + + private SyncResult SyncListItemMetadata(ListItem sourceItem, ListItem targetItem, ClientContext sourceContext, ClientContext targetContext) + { + try + { + // Load necessary fields from source using the source context + sourceContext.Load(sourceItem); + sourceContext.ExecuteQueryRetry(); + + var metadataUpdated = false; // track if we changed anything + + // Always set Created/Modified from source when available + try + { + if (sourceItem.FieldValues.ContainsKey("Created") && sourceItem.FieldValues["Created"] is DateTime cdt) + { + targetItem["Created"] = cdt; + metadataUpdated = true; + WriteVerbose($"Updated Created: {cdt}"); + } + if (sourceItem.FieldValues.ContainsKey("Modified") && sourceItem.FieldValues["Modified"] is DateTime mdt) + { + targetItem["Modified"] = mdt; + metadataUpdated = true; + WriteVerbose($"Updated Modified: {mdt}"); + } + } + catch (Exception ex) + { + WriteWarning($"Failed to set Created/Modified: {ex.Message}"); + } + + // Process the rest of the requested fields + foreach (var fieldName in Fields) + { + if (fieldName == "Created" || fieldName == "Modified") + { + continue; // already handled + } + + try + { + if (sourceItem.FieldValues.ContainsKey(fieldName) && sourceItem.FieldValues[fieldName] != null) + { + var sourceValue = sourceItem.FieldValues[fieldName]; + + if (fieldName == "Author" || fieldName == "Editor") + { + if (sourceValue is FieldUserValue userValue && userValue.LookupId > 0) + { + var mapped = MapUserToTarget(userValue, sourceContext, targetContext); + if (mapped != null) + { + targetItem[fieldName] = mapped; + metadataUpdated = true; + WriteVerbose($"Updated {fieldName} (mapped to target)"); + } + } + } + else + { + targetItem[fieldName] = sourceValue; + metadataUpdated = true; + WriteVerbose($"Updated {fieldName}: {sourceValue}"); + } + } + } + catch (Exception ex) + { + WriteWarning($"Failed to sync field '{fieldName}': {ex.Message}"); + } + } + + if (metadataUpdated) + { + targetItem.UpdateOverwriteVersion(); + targetContext.ExecuteQueryRetry(); + return SyncResult.Success; + } + else + { + return SyncResult.Skipped; + } + } + catch (Exception ex) + { + WriteWarning($"Failed to sync list item metadata: {ex.Message}"); + return SyncResult.Error; + } + } + + private (bool IsFile, string Name) GetFileOrFolderInfo(string url, ClientContext context) + { + var webServerRelativeUrl = context.Web.EnsureProperty(w => w.ServerRelativeUrl); + WriteVerbose($"GetFileOrFolderInfo: Checking URL '{url}' using context for '{webServerRelativeUrl}'"); + + // Ensure we have a server-relative URL (starting with '/') + string serverRelativeUrl = url; + if (!serverRelativeUrl.StartsWith("/", StringComparison.OrdinalIgnoreCase)) + { + serverRelativeUrl = UrlUtility.Combine(webServerRelativeUrl, serverRelativeUrl); + WriteVerbose($"Normalized to server-relative URL: '{serverRelativeUrl}'"); + } + + // CSOM ResourcePath.FromDecodedUrl expects a decoded server-relative path + var decodedServerRelative = Uri.UnescapeDataString(serverRelativeUrl); + WriteVerbose($"Decoded server-relative URL for lookup: '{decodedServerRelative}'"); + + try + { + // Try as file first using GetFileByServerRelativePath + WriteVerbose($"Trying as file (server-relative): {decodedServerRelative}"); + var file = context.Web.GetFileByServerRelativePath(ResourcePath.FromDecodedUrl(decodedServerRelative)); + context.Load(file, f => f.Name, f => f.Exists); + context.ExecuteQueryRetry(); + + WriteVerbose($"File exists check result: {file.Exists}"); + if (file.Exists) + { + WriteVerbose($"Found file: {file.Name}"); + return (true, file.Name); + } + } + catch (Exception ex) + { + WriteVerbose($"Exception when checking as file: {ex.Message}"); + } + + try + { + WriteVerbose($"Trying as folder (server-relative): {decodedServerRelative}"); + var folder = context.Web.GetFolderByServerRelativePath(ResourcePath.FromDecodedUrl(decodedServerRelative)); + context.Load(folder, f => f.Name, f => f.Exists); + context.ExecuteQueryRetry(); + + WriteVerbose($"Folder exists check result: {folder.Exists}"); + if (folder.Exists) + { + WriteVerbose($"Found folder: {folder.Name}"); + return (false, folder.Name); + } + } + catch (Exception ex) + { + WriteVerbose($"Exception when checking as folder: {ex.Message}"); + } + + // Fallback: if the URL didn't work due to trailing slash difference, try toggling it + if (!decodedServerRelative.EndsWith("/", StringComparison.Ordinal)) + { + var alt = decodedServerRelative + "/"; + try + { + WriteVerbose($"Retry as folder with trailing slash: {alt}"); + var altFolder = context.Web.GetFolderByServerRelativePath(ResourcePath.FromDecodedUrl(alt)); + context.Load(altFolder, f => f.Name, f => f.Exists); + context.ExecuteQueryRetry(); + if (altFolder.Exists) + { + WriteVerbose($"Found folder (alt): {altFolder.Name}"); + return (false, altFolder.Name); + } + } + catch (Exception ex) + { + WriteVerbose($"Alt folder check failed: {ex.Message}"); + } + } + + WriteVerbose("Neither file nor folder found"); + return (false, null); + } + + private string ProcessUrl(string url, string webServerRelativeUrl) + { + WriteVerbose($"ProcessUrl input: url='{url}', webServerRelativeUrl='{webServerRelativeUrl}'"); + + // Handle absolute URLs first + if (Uri.IsWellFormedUriString(url, UriKind.Absolute)) + { + var uri = new Uri(url); + // Extract just the path and query from the absolute URL + url = uri.AbsolutePath + uri.Query; + WriteVerbose($"After absolute URL conversion: '{url}'"); + } + + // If we're connected at tenant level (webServerRelativeUrl = '/') or the URL is already server-relative, + // just return the server-relative URL as-is + if (webServerRelativeUrl == "/" || url.StartsWith("/")) + { + WriteVerbose($"Returning server-relative URL: '{url}'"); + return url; + } + + // If it doesn't start with /, make it relative to current web + var result = UrlUtility.Combine(webServerRelativeUrl, url); + WriteVerbose($"Made relative to current web: '{result}'"); + return result; + } + + private ClientContext GetSourceContext() + { + if (SourceConnection != null) + { + WriteVerbose("Using provided SourceConnection"); + return SourceConnection.Context; + } + WriteVerbose("Using current connection for source"); + + // Check if we have a current connection + try + { + return ClientContext; + } + catch + { + return null; + } + } + + private ClientContext GetTargetContext() + { + if (TargetConnection != null) + { + WriteVerbose("Using provided TargetConnection"); + return TargetConnection.Context; + } + WriteVerbose("Using current connection for target"); + + // Check if we have a current connection + try + { + return ClientContext; + } + catch + { + return null; + } + } + + private enum SyncResult + { + Success, + Skipped, + Error + } + + private FieldUserValue MapUserToTarget(FieldUserValue sourceUser, ClientContext sourceContext, ClientContext targetContext) + { + if (sourceUser == null || sourceUser.LookupId <= 0) return null; + + string email = null; + string login = null; + try + { + var su = sourceContext.Web.GetUserById(sourceUser.LookupId); + sourceContext.Load(su, u => u.Email, u => u.LoginName); + sourceContext.ExecuteQueryRetry(); + email = su.Email; + login = su.LoginName; + } + catch (Exception ex) + { + WriteVerbose($"MapUserToTarget: failed to load source user {sourceUser.LookupId}: {ex.Message}"); + } + + // Prefer email if available; fallback to login name + string[] identities = string.IsNullOrEmpty(email) + ? new[] { login } + : (string.IsNullOrEmpty(login) ? new[] { email } : new[] { email, login }); + + foreach (var identity in identities) + { + if (string.IsNullOrWhiteSpace(identity)) continue; + try + { + if (!_targetUserCache.TryGetValue(identity, out int id)) + { + var ensured = targetContext.Web.EnsureUser(identity); + targetContext.Load(ensured, u => u.Id); + targetContext.ExecuteQueryRetry(); + id = ensured.Id; + _targetUserCache[identity] = id; + } + + if (id > 0) + { + return new FieldUserValue { LookupId = id }; + } + } + catch (Exception ex) + { + WriteVerbose($"MapUserToTarget: EnsureUser failed for '{identity}': {ex.Message}"); + // Try next identity + } + } + WriteWarning($"MapUserToTarget: could not map source user id {sourceUser.LookupId}. Leaving target value unchanged."); + return null; + } + } +} diff --git a/src/Tests/Files/CopyPnPFileMetadataTests.cs b/src/Tests/Files/CopyPnPFileMetadataTests.cs new file mode 100644 index 000000000..125229fa5 --- /dev/null +++ b/src/Tests/Files/CopyPnPFileMetadataTests.cs @@ -0,0 +1,30 @@ +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Management.Automation.Runspaces; + +namespace PnP.PowerShell.Tests.Files +{ + [TestClass] + public class CopyFileMetadataTests + { + [TestMethod] + public void CopyPnPFileMetadataOnlyTest() + { + using (var scope = new PSTestScope(true)) + { + // Test that MetadataOnly parameter is recognized + var results = scope.ExecuteCommand("Get-Command", + new CommandParameter("Name", "Copy-PnPFileMetadata")); + + Assert.IsNotNull(results); + + // Verify that MetadataOnly parameter exists + var cmdlet = results[0]; + Assert.IsNotNull(cmdlet); + + // This test validates that the MetadataOnly parameter is properly added to the cmdlet + // Full functional testing would require SharePoint connection and test content + } + } + } +} From dfa563e4d4cefca99855ccd960cc4a38372c0e19 Mon Sep 17 00:00:00 2001 From: Ali Robertson Date: Thu, 4 Sep 2025 14:02:27 +1000 Subject: [PATCH 2/3] implement fix recursive flag --- src/Commands/Files/CopyFileMetadata.cs | 29 +++++++++++++------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/Commands/Files/CopyFileMetadata.cs b/src/Commands/Files/CopyFileMetadata.cs index 4add141e0..33f092760 100644 --- a/src/Commands/Files/CopyFileMetadata.cs +++ b/src/Commands/Files/CopyFileMetadata.cs @@ -168,23 +168,24 @@ protected override void ExecuteCmdlet() else errored++; } - // Process files - if (sourceFolder.Files != null && sourceFolder.Files.Count > 0) + // When Recursive is enabled, process files and subfolders; otherwise only the folder item itself + if (Recursive) { - foreach (var sourceFile in sourceFolder.Files) + // Process files in the current folder + if (sourceFolder.Files != null && sourceFolder.Files.Count > 0) { - var src = UrlUtility.Combine(sourceFolderUrl, sourceFile.Name); - var dst = UrlUtility.Combine(targetFolderUrl, sourceFile.Name); - var r = SyncFileMetadata(src, dst, sourceContext, targetContext); - if (r == SyncResult.Success) processed++; - else if (r == SyncResult.Skipped) skipped++; - else errored++; + foreach (var sourceFile in sourceFolder.Files) + { + var src = UrlUtility.Combine(sourceFolderUrl, sourceFile.Name); + var dst = UrlUtility.Combine(targetFolderUrl, sourceFile.Name); + var r = SyncFileMetadata(src, dst, sourceContext, targetContext); + if (r == SyncResult.Success) processed++; + else if (r == SyncResult.Skipped) skipped++; + else errored++; + } } - } - // Process subfolders recursively if Recursive is enabled - if (Recursive) - { + // Process subfolders recursively foreach (var sourceSubfolder in sourceFolder.Folders) { if (sourceSubfolder.Name.StartsWith("_")) continue; // Skip system folders @@ -192,7 +193,7 @@ protected override void ExecuteCmdlet() var sourceSubfolderUrlCombined = UrlUtility.Combine(sourceFolderUrl, sourceSubfolder.Name); var targetSubfolderUrl = UrlUtility.Combine(targetFolderUrl, sourceSubfolder.Name); var subResults = SyncFolderMetadataRecursive(sourceSubfolderUrlCombined, targetSubfolderUrl, sourceContext, targetContext); - + processed += subResults.Processed; skipped += subResults.Skipped; errored += subResults.Errored; From 88e25d82bff74fe3cdd6c7f7b0eea8061dd4dcee Mon Sep 17 00:00:00 2001 From: Ali Robertson Date: Fri, 5 Sep 2025 02:32:39 +1000 Subject: [PATCH 3/3] CopyFileMetadata: improve performance, and reduce server load --- documentation/Copy-PnPFileMetadata.md | 26 +- src/Commands/Files/CopyFileMetadata.cs | 679 ++++++++++++++----------- 2 files changed, 396 insertions(+), 309 deletions(-) diff --git a/documentation/Copy-PnPFileMetadata.md b/documentation/Copy-PnPFileMetadata.md index dac568544..a3a85204c 100644 --- a/documentation/Copy-PnPFileMetadata.md +++ b/documentation/Copy-PnPFileMetadata.md @@ -21,47 +21,49 @@ Copy-PnPFileMetadata [-SourceUrl] [-TargetUrl] [-Fields _targetUserCache = new System.Collections.Generic.Dictionary(StringComparer.OrdinalIgnoreCase); + // Batch + cache management + private const int FileLoadBatchSize = 200; + private const int UpdateFlushSize = 100; + private readonly Dictionary _sourceUserIdentityCache = new(); + private readonly Dictionary _targetUserIdByIdentity = new(StringComparer.OrdinalIgnoreCase); + private readonly HashSet _unmappedSourceUsersWarned = new(); + + private static readonly HashSet SystemDateFieldsSet = new(["Created", "Modified"]); [Parameter(Mandatory = true, Position = 0, ValueFromPipeline = true)] [Alias("ServerRelativeUrl")] @@ -25,10 +35,10 @@ public class CopyFileMetadata : PnPWebCmdlet public SwitchParameter Force; [Parameter(Mandatory = false)] - public string[] Fields = new[] { "Created", "Modified", "Author", "Editor" }; + public string[] Fields = ["Author", "Editor", "Created", "Modified"]; - [Parameter(Mandatory = false)] - public SwitchParameter Recursive = SwitchParameter.Present; + [Parameter(Mandatory = false)] + public SwitchParameter Recursive = SwitchParameter.Present; [Parameter(Mandatory = false, HelpMessage = "Optional connection to be used for accessing the source file. If not provided, uses the current connection.")] public PnPConnection SourceConnection = null; @@ -38,41 +48,26 @@ public class CopyFileMetadata : PnPWebCmdlet protected override void ExecuteCmdlet() { + // Marshal inputs + Fields = OrganizeFields(); + // Get the contexts for source and target operations - var sourceContext = GetSourceContext(); - var targetContext = GetTargetContext(); - - // Ensure we have valid contexts - if (sourceContext == null) - { - throw new InvalidOperationException("Source connection is not available. Please provide a valid SourceConnection or ensure you are connected."); - } - - if (targetContext == null) - { - throw new InvalidOperationException("Target connection is not available. Please provide a valid TargetConnection or ensure you are connected."); - } - + var sourceContext = SourceConnection?.Context ?? ClientContext; + var targetContext = TargetConnection?.Context ?? ClientContext; + SyncResultCount resultTotals; + + // Ensure web URLs are loaded before using them var sourceWebServerRelativeUrl = sourceContext.Web.EnsureProperty(w => w.ServerRelativeUrl); var targetWebServerRelativeUrl = targetContext.Web.EnsureProperty(w => w.ServerRelativeUrl); - WriteVerbose($"Source context web URL: {sourceWebServerRelativeUrl}"); - WriteVerbose($"Target context web URL: {targetWebServerRelativeUrl}"); - - // Process URLs relative to their respective contexts - SourceUrl = ProcessUrl(SourceUrl, sourceWebServerRelativeUrl); - TargetUrl = ProcessUrl(TargetUrl, targetWebServerRelativeUrl); + SourceUrl = GetServerRelativePath(SourceUrl, sourceWebServerRelativeUrl); + TargetUrl = GetServerRelativePath(TargetUrl, targetWebServerRelativeUrl); - WriteVerbose($"Processed SourceUrl: {SourceUrl}"); - WriteVerbose($"Processed TargetUrl: {TargetUrl}"); - - if (Force || ShouldContinue(string.Format("Synchronize metadata from '{0}' to '{1}'", SourceUrl, TargetUrl), Resources.Confirm)) + if (Force || ShouldContinue(string.Format("Synchronize metadata from '{0}' to '{1}'. Recursion: {2}", SourceUrl, TargetUrl, Recursive), Resources.Confirm)) { try { - WriteVerbose($"Starting metadata synchronization from '{SourceUrl}' to '{TargetUrl}'"); - WriteVerbose($"Fields to sync: {string.Join(", ", Fields)}"); - + WriteVerbose($"Syncing."); // Determine if source is a file or folder (using source context) var sourceItem = GetFileOrFolderInfo(SourceUrl, sourceContext); if (sourceItem.Name == null) @@ -85,54 +80,48 @@ protected override void ExecuteCmdlet() return; } - var itemsProcessed = 0; - var itemsSkipped = 0; - var itemsErrored = 0; - if (sourceItem.IsFile) { var result = SyncFileMetadata(SourceUrl, TargetUrl, sourceContext, targetContext); - if (result == SyncResult.Success) itemsProcessed++; - else if (result == SyncResult.Skipped) itemsSkipped++; - else itemsErrored++; + resultTotals = new SyncResultCount() { processed = 0, skipped = 0, errored = 0 }; + addResultToCount(result, ref resultTotals); } else { - var results = SyncFolderMetadataRecursive(SourceUrl, TargetUrl, sourceContext, targetContext); - itemsProcessed = results.Processed; - itemsSkipped = results.Skipped; - itemsErrored = results.Errored; + resultTotals = SyncFolderMetadata(SourceUrl, TargetUrl, sourceContext, targetContext); } - WriteObject($"Metadata synchronization completed. Processed: {itemsProcessed}, Skipped: {itemsSkipped}, Errors: {itemsErrored}"); + WriteObject($"Metadata synchronization completed. Processed: {resultTotals.processed}, Skipped: {resultTotals.skipped}, Errors: {resultTotals.errored}"); } - catch (Exception ex) + catch (Exception) { - WriteError(new ErrorRecord(ex, "MetadataSyncError", ErrorCategory.InvalidOperation, SourceUrl)); + WriteError(new ErrorRecord(new InvalidOperationException("Metadata synchronization failed."), "MetadataSyncError", ErrorCategory.InvalidOperation, SourceUrl)); } } } - private (int Processed, int Skipped, int Errored) SyncFolderMetadataRecursive(string sourceFolderUrl, string targetFolderUrl) + private string[] OrganizeFields() { - return SyncFolderMetadataRecursive(sourceFolderUrl, targetFolderUrl, ClientContext, ClientContext); + // Normalize, distinct (case-insensitive), and order system date fields last + var normalized = (Fields ?? Array.Empty()) + .Where(f => !string.IsNullOrWhiteSpace(f)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(f => SystemDateFieldsSet.Contains(f) ? 1 : 0) + .ToArray(); + return normalized; } - private (int Processed, int Skipped, int Errored) SyncFolderMetadataRecursive(string sourceFolderUrl, string targetFolderUrl, ClientContext sourceContext, ClientContext targetContext) + private SyncResultCount SyncFolderMetadata(string sourceFolderUrl, string targetFolderUrl, ClientContext sourceContext, ClientContext targetContext) { - var processed = 0; - var skipped = 0; - var errored = 0; + var resultCount = new SyncResultCount() { processed = 0, skipped = 0, errored = 0 }; try { - WriteVerbose($"Processing folder: {sourceFolderUrl}"); - // Get source and target folder via ResourcePath and validate existence var sourceFolder = sourceContext.Web.GetFolderByServerRelativePath(ResourcePath.FromDecodedUrl(Uri.UnescapeDataString(sourceFolderUrl))); var targetFolder = targetContext.Web.GetFolderByServerRelativePath(ResourcePath.FromDecodedUrl(Uri.UnescapeDataString(targetFolderUrl))); - // Step 1: check existence first to avoid exceptions when loading list item of a non-existing folder + // Confirm existence sourceContext.Load(sourceFolder, f => f.Name, f => f.Exists); targetContext.Load(targetFolder, f => f.Name, f => f.Exists); sourceContext.ExecuteQueryRetry(); @@ -141,47 +130,92 @@ protected override void ExecuteCmdlet() if (!sourceFolder.Exists) { WriteWarning($"Source folder not found: '{sourceFolderUrl}'"); - errored++; - return (processed, skipped, errored); + resultCount.errored++; + return resultCount; } if (!targetFolder.Exists) { WriteWarning($"Target folder not found, skipping subtree: '{targetFolderUrl}'"); - skipped++; - return (processed, skipped, errored); + resultCount.skipped++; + return resultCount; } - // Step 2: load heavy properties once existence is confirmed - sourceContext.Load(sourceFolder, f => f.ListItemAllFields, f => f.Files, f => f.Folders); - targetContext.Load(targetFolder, f => f.ListItemAllFields); + // Progress: folder being processed + WriteVerbose($"Folder: '{sourceFolderUrl}' -> '{targetFolderUrl}'"); + + // Load folder children (avoid loading all list item fields) + sourceContext.Load(sourceFolder, f => f.Files, f => f.Folders); + targetContext.Load(targetFolder, f => f.Files, f => f.Folders); sourceContext.ExecuteQueryRetry(); targetContext.ExecuteQueryRetry(); - // Sync folder metadata if both have list items - if (!sourceFolder.ListItemAllFields.ServerObjectIsNull.GetValueOrDefault() && - !targetFolder.ListItemAllFields.ServerObjectIsNull.GetValueOrDefault()) - { - var result = SyncListItemMetadata(sourceFolder.ListItemAllFields, targetFolder.ListItemAllFields, sourceContext, targetContext); - if (result == SyncResult.Success) processed++; - else if (result == SyncResult.Skipped) skipped++; - else errored++; - } + // Progress: child counts + WriteVerbose($" - Contains: {sourceFolder.Files?.Count ?? 0} files, {sourceFolder.Folders?.Count ?? 0} folders"); - // When Recursive is enabled, process files and subfolders; otherwise only the folder item itself + // For the folder list items, preload only the needed fields (fetch the list items explicitly) + var folderNeededFields = new HashSet(Fields, StringComparer.OrdinalIgnoreCase); + LoadListItemFields(sourceContext, sourceFolder.ListItemAllFields, folderNeededFields); + LoadListItemFields(targetContext, targetFolder.ListItemAllFields, folderNeededFields); + sourceContext.ExecuteQueryRetry(); + targetContext.ExecuteQueryRetry(); + + // Sync folder metadata (targeted fields already loaded) + WriteVerbose($" - Syncing folder metadata for '{sourceFolder.Name}'"); + var folderSync = SyncListItemMetadata(sourceFolder.ListItemAllFields, targetFolder.ListItemAllFields, sourceContext, targetContext, skipSourceLoad: true, itemLabel: sourceFolder.Name); + addResultToCount(folderSync, ref resultCount); + + // When Recursive is enabled, process files and subfolders. if (Recursive) { + // Determine required fields: always Created/Modified + var neededFields = new HashSet(Fields, StringComparer.OrdinalIgnoreCase); + // Batch-load only needed fields for source and target files + PreloadFilesWithFields(sourceContext, sourceFolder.Files, neededFields); + PreloadFilesWithFields(targetContext, targetFolder.Files, neededFields); + // Process files in the current folder if (sourceFolder.Files != null && sourceFolder.Files.Count > 0) { + Dictionary targetByName = targetFolder.Files.ToDictionary(f => f.Name, StringComparer.OrdinalIgnoreCase); + + var pendingChanges = false; + var pendingChangesCount = 0; foreach (var sourceFile in sourceFolder.Files) { - var src = UrlUtility.Combine(sourceFolderUrl, sourceFile.Name); - var dst = UrlUtility.Combine(targetFolderUrl, sourceFile.Name); - var r = SyncFileMetadata(src, dst, sourceContext, targetContext); - if (r == SyncResult.Success) processed++; - else if (r == SyncResult.Skipped) skipped++; - else errored++; + WriteVerbose($" File: {sourceFile.Name}"); + if (!targetByName.TryGetValue(sourceFile.Name, out var targetFileForName)) + { + var missing = UrlUtility.Combine(targetFolderUrl, sourceFile.Name); + WriteWarning($"Target file not found, skipping: '{missing}'"); + resultCount.skipped++; + continue; + } + + var sItem = sourceFile.ListItemAllFields; + var tItem = targetFileForName.ListItemAllFields; + var r = SyncListItemMetadata(sItem, tItem, sourceContext, targetContext, skipSourceLoad: true, executeImmediately: false, itemLabel: sourceFile.Name); + addResultToCount(r, ref resultCount); + if (r == SyncResult.Success) + { + pendingChanges = true; + pendingChangesCount++; + if (pendingChangesCount >= UpdateFlushSize) + { + targetContext.ExecuteQueryRetry(); + WriteVerbose($"Flushed {pendingChangesCount} pending updates in folder '{targetFolderUrl}'."); + pendingChanges = false; + pendingChangesCount = 0; + } + } + } + + if (pendingChanges) + { + targetContext.ExecuteQueryRetry(); + WriteVerbose($"Flushed {pendingChangesCount} pending updates in folder '{targetFolderUrl}'."); + pendingChanges = false; + pendingChangesCount = 0; } } @@ -189,45 +223,51 @@ protected override void ExecuteCmdlet() foreach (var sourceSubfolder in sourceFolder.Folders) { if (sourceSubfolder.Name.StartsWith("_")) continue; // Skip system folders - + WriteVerbose($" Folder: {sourceSubfolder.Name}"); var sourceSubfolderUrlCombined = UrlUtility.Combine(sourceFolderUrl, sourceSubfolder.Name); var targetSubfolderUrl = UrlUtility.Combine(targetFolderUrl, sourceSubfolder.Name); - var subResults = SyncFolderMetadataRecursive(sourceSubfolderUrlCombined, targetSubfolderUrl, sourceContext, targetContext); + var subResults = SyncFolderMetadata(sourceSubfolderUrlCombined, targetSubfolderUrl, sourceContext, targetContext); - processed += subResults.Processed; - skipped += subResults.Skipped; - errored += subResults.Errored; + resultCount.processed += subResults.processed; + resultCount.skipped += subResults.skipped; + resultCount.errored += subResults.errored; } } } catch (Exception ex) { WriteWarning($"Failed to process folder '{sourceFolderUrl}': {ex.Message}"); - errored++; + resultCount.errored++; } - return (processed, skipped, errored); + return resultCount; } - private SyncResult SyncFileMetadata(string sourceFileUrl, string targetFileUrl) - { - return SyncFileMetadata(sourceFileUrl, targetFileUrl, ClientContext, ClientContext); - } private SyncResult SyncFileMetadata(string sourceFileUrl, string targetFileUrl, ClientContext sourceContext, ClientContext targetContext) { try { - WriteVerbose($"Syncing file metadata: {sourceFileUrl} -> {targetFileUrl}"); + // Sync file metadata // Decode and use server-relative ResourcePath to reliably check existence var decodedSourceUrl = Uri.UnescapeDataString(sourceFileUrl); var decodedTargetUrl = Uri.UnescapeDataString(targetFileUrl); + WriteVerbose($"File: '{decodedSourceUrl}' -> '{decodedTargetUrl}'"); + var sourceFile = sourceContext.Web.GetFileByServerRelativePath(ResourcePath.FromDecodedUrl(decodedSourceUrl)); var targetFile = targetContext.Web.GetFileByServerRelativePath(ResourcePath.FromDecodedUrl(decodedTargetUrl)); - sourceContext.Load(sourceFile, f => f.Exists, f => f.Name, f => f.ListItemAllFields); - targetContext.Load(targetFile, f => f.Exists, f => f.Name, f => f.ListItemAllFields); + sourceContext.Load(sourceFile, f => f.Exists, f => f.Name); + targetContext.Load(targetFile, f => f.Exists, f => f.Name); + + // Preload only the required fields on both list items prior to reading values + var neededFields = new HashSet(Fields, StringComparer.OrdinalIgnoreCase); + var sourceFileItem = sourceFile.ListItemAllFields; + var targetFileItem = targetFile.ListItemAllFields; + LoadListItemFields(sourceContext, sourceFileItem, neededFields); + LoadListItemFields(targetContext, targetFileItem, neededFields); + sourceContext.ExecuteQueryRetry(); targetContext.ExecuteQueryRetry(); @@ -243,16 +283,7 @@ private SyncResult SyncFileMetadata(string sourceFileUrl, string targetFileUrl, return SyncResult.Skipped; } - if (!sourceFile.ListItemAllFields.ServerObjectIsNull.GetValueOrDefault() && - !targetFile.ListItemAllFields.ServerObjectIsNull.GetValueOrDefault()) - { - return SyncListItemMetadata(sourceFile.ListItemAllFields, targetFile.ListItemAllFields, sourceContext, targetContext); - } - else - { - WriteWarning($"Cannot access list item for file '{sourceFileUrl}' or '{targetFileUrl}'"); - return SyncResult.Skipped; - } + return SyncListItemMetadata(sourceFileItem, targetFileItem, sourceContext, targetContext, skipSourceLoad: true); } catch (Exception ex) { @@ -261,303 +292,357 @@ private SyncResult SyncFileMetadata(string sourceFileUrl, string targetFileUrl, } } - private SyncResult SyncListItemMetadata(ListItem sourceItem, ListItem targetItem, ClientContext sourceContext, ClientContext targetContext) + private SyncResult SyncListItemMetadata(ListItem sourceItem, ListItem targetItem, ClientContext sourceContext, ClientContext targetContext, bool skipSourceLoad = false, bool executeImmediately = true, string itemLabel = null) { try { - // Load necessary fields from source using the source context - sourceContext.Load(sourceItem); - sourceContext.ExecuteQueryRetry(); + if (!skipSourceLoad) + { + sourceContext.Load(sourceItem); + sourceContext.ExecuteQueryRetry(); + } var metadataUpdated = false; // track if we changed anything + var labelPrefix = string.IsNullOrEmpty(itemLabel) ? string.Empty : ($"[{itemLabel}] "); + var valuesToSet = PrepareItemValues(sourceItem, targetItem, sourceContext, targetContext, Fields, labelPrefix, ref metadataUpdated); + + if (!metadataUpdated) + { + return SyncResult.Skipped; + } - // Always set Created/Modified from source when available try { - if (sourceItem.FieldValues.ContainsKey("Created") && sourceItem.FieldValues["Created"] is DateTime cdt) - { - targetItem["Created"] = cdt; - metadataUpdated = true; - WriteVerbose($"Updated Created: {cdt}"); - } - if (sourceItem.FieldValues.ContainsKey("Modified") && sourceItem.FieldValues["Modified"] is DateTime mdt) + targetItem.SetFieldValues(valuesToSet, this); + targetItem.UpdateOverwriteVersion(); + if (executeImmediately) { - targetItem["Modified"] = mdt; - metadataUpdated = true; - WriteVerbose($"Updated Modified: {mdt}"); + targetContext.ExecuteQueryRetry(); } + return SyncResult.Success; } - catch (Exception ex) + catch (Exception ex) when (ex.Message.IndexOf("user", StringComparison.OrdinalIgnoreCase) >= 0) { - WriteWarning($"Failed to set Created/Modified: {ex.Message}"); - } + // Fallback: if setting user fields caused an error, retry without Author/Editor + var hadAuthor = valuesToSet.ContainsKey("Author"); + var hadEditor = valuesToSet.ContainsKey("Editor"); + if (hadAuthor) valuesToSet.Remove("Author"); + if (hadEditor) valuesToSet.Remove("Editor"); - // Process the rest of the requested fields - foreach (var fieldName in Fields) - { - if (fieldName == "Created" || fieldName == "Modified") + if (valuesToSet.Count == 0) { - continue; // already handled + WriteWarning($"{labelPrefix}Failed to sync list item metadata due to user mapping; no other changes to apply. {ex.Message}"); + return SyncResult.Skipped; } try { - if (sourceItem.FieldValues.ContainsKey(fieldName) && sourceItem.FieldValues[fieldName] != null) + targetItem.SetFieldValues(valuesToSet, this); + targetItem.UpdateOverwriteVersion(); + if (executeImmediately) { - var sourceValue = sourceItem.FieldValues[fieldName]; + targetContext.ExecuteQueryRetry(); + } + WriteVerbose($"{labelPrefix}Applied non-user fields after user mapping failure."); + return SyncResult.Success; + } + catch (Exception ex2) + { + WriteWarning($"{labelPrefix}Failed to sync list item metadata after removing user fields: {ex2.Message}"); + return SyncResult.Error; + } + } + } + catch (Exception ex) + { + var labelPrefix = string.IsNullOrEmpty(itemLabel) ? string.Empty : ($"[{itemLabel}] "); + WriteWarning($"{labelPrefix}Failed to sync list item metadata: {ex.Message}"); + return SyncResult.Error; + } + } + + + private enum SyncResult { Success, Skipped, Error } + + protected struct SyncResultCount { public int processed, skipped, errored; } + + private void addResultToCount(SyncResult result, ref SyncResultCount resultCount) + { + switch (result) + { + case SyncResult.Success: resultCount.processed++; break; + case SyncResult.Skipped: resultCount.skipped++; break; + case SyncResult.Error: resultCount.errored++; break; + } + } + + + private Hashtable PrepareItemValues(ListItem sourceItem, ListItem targetItem, ClientContext sourceContext, ClientContext targetContext, string[] fields, string labelPrefix, ref bool metadataUpdated) + { + var valuesToSet = new Hashtable(StringComparer.OrdinalIgnoreCase); - if (fieldName == "Author" || fieldName == "Editor") + foreach (var fieldName in fields) + { + try + { + if (sourceItem.FieldValues.ContainsKey(fieldName) && sourceItem.FieldValues[fieldName] != null) + { + var sourceValue = sourceItem.FieldValues[fieldName]; + + if (string.Equals(fieldName, "Author", StringComparison.OrdinalIgnoreCase) || + string.Equals(fieldName, "Editor", StringComparison.OrdinalIgnoreCase)) + { + if (sourceValue is FieldUserValue userValue && userValue.LookupId > 0) { - if (sourceValue is FieldUserValue userValue && userValue.LookupId > 0) + var mapped = MapUserToTarget(userValue, sourceContext, targetContext); + if (mapped != null) { - var mapped = MapUserToTarget(userValue, sourceContext, targetContext); - if (mapped != null) + var current = targetItem[fieldName] as FieldUserValue; + if (current == null || current.LookupId != mapped.LookupId) { - targetItem[fieldName] = mapped; + valuesToSet[fieldName] = mapped; // set FieldUserValue directly metadataUpdated = true; - WriteVerbose($"Updated {fieldName} (mapped to target)"); } } } - else + } + else + { + var current = targetItem.FieldValues.ContainsKey(fieldName) ? targetItem[fieldName] : null; + if (!ValuesEqual(current, sourceValue)) { - targetItem[fieldName] = sourceValue; + valuesToSet[fieldName] = sourceValue; metadataUpdated = true; - WriteVerbose($"Updated {fieldName}: {sourceValue}"); } } } - catch (Exception ex) - { - WriteWarning($"Failed to sync field '{fieldName}': {ex.Message}"); - } } + catch (Exception) + { + WriteWarning($"{labelPrefix}Failed to prepare field '{fieldName}'."); + } + } + + return valuesToSet; + } + - if (metadataUpdated) + private void SetSystemDateFields(ListItem sourceItem, ListItem targetItem, string labelPrefix, ref bool metadataUpdated) + { + try + { + if (sourceItem.FieldValues.ContainsKey("Created") && sourceItem.FieldValues["Created"] is DateTime cdt2) { - targetItem.UpdateOverwriteVersion(); - targetContext.ExecuteQueryRetry(); - return SyncResult.Success; + var curCreated = targetItem.FieldValues.ContainsKey("Created") ? targetItem["Created"] as DateTime? : null; + if (!curCreated.HasValue || curCreated.Value != cdt2) + { + targetItem["Created"] = cdt2; + metadataUpdated = true; + } } - else + if (sourceItem.FieldValues.ContainsKey("Modified") && sourceItem.FieldValues["Modified"] is DateTime mdt2) { - return SyncResult.Skipped; + var curModified = targetItem.FieldValues.ContainsKey("Modified") ? targetItem["Modified"] as DateTime? : null; + if (!curModified.HasValue || curModified.Value != mdt2) + { + targetItem["Modified"] = mdt2; + metadataUpdated = true; + } } } - catch (Exception ex) + catch (Exception) { - WriteWarning($"Failed to sync list item metadata: {ex.Message}"); - return SyncResult.Error; + WriteWarning($"{labelPrefix}Failed to prepare Created/Modified."); } } - private (bool IsFile, string Name) GetFileOrFolderInfo(string url, ClientContext context) + private static bool ValuesEqual(object a, object b) { - var webServerRelativeUrl = context.Web.EnsureProperty(w => w.ServerRelativeUrl); - WriteVerbose($"GetFileOrFolderInfo: Checking URL '{url}' using context for '{webServerRelativeUrl}'"); + if (ReferenceEquals(a, b)) return true; + if (a == null || b == null) return false; + + if (a is DateTime adt && b is DateTime bdt) + { + return adt == bdt; + } - // Ensure we have a server-relative URL (starting with '/') - string serverRelativeUrl = url; - if (!serverRelativeUrl.StartsWith("/", StringComparison.OrdinalIgnoreCase)) + if (a is FieldUserValue au && b is FieldUserValue bu) { - serverRelativeUrl = UrlUtility.Combine(webServerRelativeUrl, serverRelativeUrl); - WriteVerbose($"Normalized to server-relative URL: '{serverRelativeUrl}'"); + return au.LookupId == bu.LookupId; } - // CSOM ResourcePath.FromDecodedUrl expects a decoded server-relative path + return object.Equals(a, b); + } + + private (bool IsFile, string Name) GetFileOrFolderInfo(string url, ClientContext context) + { + var webServerRelativeUrl = context.Web.EnsureProperty(w => w.ServerRelativeUrl); + var serverRelativeUrl = GetServerRelativePath(url, webServerRelativeUrl); var decodedServerRelative = Uri.UnescapeDataString(serverRelativeUrl); - WriteVerbose($"Decoded server-relative URL for lookup: '{decodedServerRelative}'"); + // Try as file first using GetFileByServerRelativePath try { - // Try as file first using GetFileByServerRelativePath - WriteVerbose($"Trying as file (server-relative): {decodedServerRelative}"); var file = context.Web.GetFileByServerRelativePath(ResourcePath.FromDecodedUrl(decodedServerRelative)); context.Load(file, f => f.Name, f => f.Exists); context.ExecuteQueryRetry(); - - WriteVerbose($"File exists check result: {file.Exists}"); - if (file.Exists) - { - WriteVerbose($"Found file: {file.Name}"); - return (true, file.Name); - } - } - catch (Exception ex) - { - WriteVerbose($"Exception when checking as file: {ex.Message}"); + if (file.Exists) return (true, file.Name); } + catch (Exception) { } // keep trying + // try as folder try { - WriteVerbose($"Trying as folder (server-relative): {decodedServerRelative}"); var folder = context.Web.GetFolderByServerRelativePath(ResourcePath.FromDecodedUrl(decodedServerRelative)); context.Load(folder, f => f.Name, f => f.Exists); context.ExecuteQueryRetry(); - - WriteVerbose($"Folder exists check result: {folder.Exists}"); - if (folder.Exists) - { - WriteVerbose($"Found folder: {folder.Name}"); - return (false, folder.Name); - } - } - catch (Exception ex) - { - WriteVerbose($"Exception when checking as folder: {ex.Message}"); + if (folder.Exists) return (false, folder.Name); } + catch (Exception) { } // keep trying - // Fallback: if the URL didn't work due to trailing slash difference, try toggling it + // add a trailing slash and try as folder again if (!decodedServerRelative.EndsWith("/", StringComparison.Ordinal)) { var alt = decodedServerRelative + "/"; try { - WriteVerbose($"Retry as folder with trailing slash: {alt}"); + // retry as folder with trailing slash var altFolder = context.Web.GetFolderByServerRelativePath(ResourcePath.FromDecodedUrl(alt)); context.Load(altFolder, f => f.Name, f => f.Exists); context.ExecuteQueryRetry(); - if (altFolder.Exists) - { - WriteVerbose($"Found folder (alt): {altFolder.Name}"); - return (false, altFolder.Name); - } - } - catch (Exception ex) - { - WriteVerbose($"Alt folder check failed: {ex.Message}"); + if (altFolder.Exists) return (false, altFolder.Name); } + catch (Exception) { } // failover } - WriteVerbose("Neither file nor folder found"); - return (false, null); + return (false, null); // neither file nor folder found } - private string ProcessUrl(string url, string webServerRelativeUrl) + private static string GetServerRelativePath(string url, string webServerRelativeUrl) { - WriteVerbose($"ProcessUrl input: url='{url}', webServerRelativeUrl='{webServerRelativeUrl}'"); - - // Handle absolute URLs first + if (url.StartsWith('/')) return url; if (Uri.IsWellFormedUriString(url, UriKind.Absolute)) { var uri = new Uri(url); - // Extract just the path and query from the absolute URL - url = uri.AbsolutePath + uri.Query; - WriteVerbose($"After absolute URL conversion: '{url}'"); - } - - // If we're connected at tenant level (webServerRelativeUrl = '/') or the URL is already server-relative, - // just return the server-relative URL as-is - if (webServerRelativeUrl == "/" || url.StartsWith("/")) - { - WriteVerbose($"Returning server-relative URL: '{url}'"); - return url; + return uri.AbsolutePath + uri.Query; } - - // If it doesn't start with /, make it relative to current web - var result = UrlUtility.Combine(webServerRelativeUrl, url); - WriteVerbose($"Made relative to current web: '{result}'"); - return result; + return UrlUtility.Combine(webServerRelativeUrl, url); } - private ClientContext GetSourceContext() - { - if (SourceConnection != null) - { - WriteVerbose("Using provided SourceConnection"); - return SourceConnection.Context; - } - WriteVerbose("Using current connection for source"); - - // Check if we have a current connection - try - { - return ClientContext; - } - catch - { - return null; - } - } - private ClientContext GetTargetContext() - { - if (TargetConnection != null) - { - WriteVerbose("Using provided TargetConnection"); - return TargetConnection.Context; - } - WriteVerbose("Using current connection for target"); - - // Check if we have a current connection - try - { - return ClientContext; - } - catch - { - return null; - } - } - - private enum SyncResult - { - Success, - Skipped, - Error - } private FieldUserValue MapUserToTarget(FieldUserValue sourceUser, ClientContext sourceContext, ClientContext targetContext) { - if (sourceUser == null || sourceUser.LookupId <= 0) return null; - - string email = null; - string login = null; try { - var su = sourceContext.Web.GetUserById(sourceUser.LookupId); - sourceContext.Load(su, u => u.Email, u => u.LoginName); - sourceContext.ExecuteQueryRetry(); - email = su.Email; - login = su.LoginName; - } - catch (Exception ex) - { - WriteVerbose($"MapUserToTarget: failed to load source user {sourceUser.LookupId}: {ex.Message}"); - } + if (sourceUser == null || sourceUser.LookupId <= 0) return null; - // Prefer email if available; fallback to login name - string[] identities = string.IsNullOrEmpty(email) - ? new[] { login } - : (string.IsNullOrEmpty(login) ? new[] { email } : new[] { email, login }); + if (ReferenceEquals(sourceContext, targetContext)) + { + // when both contexts refer to the same site, LookupId will match + return new FieldUserValue { LookupId = sourceUser.LookupId }; + } - foreach (var identity in identities) - { - if (string.IsNullOrWhiteSpace(identity)) continue; - try + // Obtain source identities (email/login) from cache or load + string email = null; + string login = null; + if (!_sourceUserIdentityCache.TryGetValue(sourceUser.LookupId, out var identities)) { - if (!_targetUserCache.TryGetValue(identity, out int id)) + try { - var ensured = targetContext.Web.EnsureUser(identity); - targetContext.Load(ensured, u => u.Id); - targetContext.ExecuteQueryRetry(); - id = ensured.Id; - _targetUserCache[identity] = id; + var su = sourceContext.Web.GetUserById(sourceUser.LookupId); + sourceContext.Load(su, u => u.Email, u => u.LoginName); + sourceContext.ExecuteQueryRetry(); + email = su.Email; + login = su.LoginName; + _sourceUserIdentityCache[sourceUser.LookupId] = [email, login]; } + catch (Exception) + { + // source user load failed + } + } + else + { + if (identities != null) + { + if (identities.Length > 0) email = identities[0]; + if (identities.Length > 1) login = identities[1]; + } + } + + // Try email first, then login + var candidates = new System.Collections.Generic.List(); + if (!string.IsNullOrEmpty(email)) candidates.Add(email); + if (!string.IsNullOrEmpty(login)) candidates.Add(login); - if (id > 0) + foreach (var identity in candidates) + { + if (string.IsNullOrWhiteSpace(identity)) continue; + try + { + if (!_targetUserIdByIdentity.TryGetValue(identity, out int id)) + { + var ensured = targetContext.Web.EnsureUser(identity); + targetContext.Load(ensured, u => u.Id); + targetContext.ExecuteQueryRetry(); + id = ensured.Id; + _targetUserIdByIdentity[identity] = id; + } + if (id > 0) + { + return new FieldUserValue { LookupId = id }; + } + // id <= 0 indicates previous failure; skip retry + continue; + } + catch (Exception) { - return new FieldUserValue { LookupId = id }; + // EnsureUser failed for identity; store negative cache to avoid repeated retries + _targetUserIdByIdentity[identity] = -1; } } - catch (Exception ex) + + if (_unmappedSourceUsersWarned.Add(sourceUser.LookupId)) + { + WriteWarning($"MapUserToTarget: could not map source user id {sourceUser.LookupId}. Leaving target value unchanged."); + } + return null; + } + catch (Exception) + { + // map user unexpected error + return null; + } + } + + private void PreloadFilesWithFields(ClientContext context, Microsoft.SharePoint.Client.FileCollection files, ISet fieldNames) + { + if (files == null || files.Count == 0) return; + + int idx = 0; + foreach (var f in files) + { + context.Load(f, x => x.Name); + foreach (var nf in fieldNames) { - WriteVerbose($"MapUserToTarget: EnsureUser failed for '{identity}': {ex.Message}"); - // Try next identity + context.Load(f.ListItemAllFields, i => i[nf]); } + idx++; + if (idx % FileLoadBatchSize == 0) context.ExecuteQueryRetry(); + } + + if (idx % FileLoadBatchSize != 0) context.ExecuteQueryRetry(); + } + + // Helper: queue loads for specific fields on a ListItem to avoid loading all fields + private static void LoadListItemFields(ClientContext context, ListItem item, ISet fieldNames) + { + if (item == null || fieldNames == null || fieldNames.Count == 0) return; + foreach (var nf in fieldNames) + { + context.Load(item, i => i[nf]); } - WriteWarning($"MapUserToTarget: could not map source user id {sourceUser.LookupId}. Leaving target value unchanged."); - return null; } } }