From 31f97be60e51bb0c8d20f6b9d4f4340ce32a4fbb Mon Sep 17 00:00:00 2001 From: Kevin Jump Date: Thu, 26 Mar 2026 20:45:02 +0000 Subject: [PATCH 1/5] A few IO improvements based on suggestions from AI prompts. --- .../Services/ISyncActionService.cs | 9 ++- uSync.BackOffice/Services/ISyncFileService.cs | 12 ++- .../Services/SyncActionService.cs | 4 +- uSync.BackOffice/Services/SyncFileService.cs | 47 ++++++----- .../SyncHandlers/SyncHandlerBase.cs | 2 +- .../SyncHandlers/SyncHandlerRoot.cs | 80 +++++++++++-------- .../SyncContainerSerializerBase.cs | 18 +++-- 7 files changed, 105 insertions(+), 67 deletions(-) diff --git a/uSync.BackOffice/Services/ISyncActionService.cs b/uSync.BackOffice/Services/ISyncActionService.cs index 3742f8134..f1a80bfdf 100644 --- a/uSync.BackOffice/Services/ISyncActionService.cs +++ b/uSync.BackOffice/Services/ISyncActionService.cs @@ -68,5 +68,12 @@ public interface ISyncActionService /// /// unpacks a zip archive (stream) to disk, checks it and copies it over the existing uSync folder. /// - UploadImportResult UnpackImportFromStream(Stream stream); + [Obsolete("Use UnpackImportFromStreamAsync(Stream stream) will be removed in v19")] + UploadImportResult UnpackImportFromStream(Stream stream) + => UnpackImportFromStreamAsync(stream).GetAwaiter().GetResult(); + + /// + /// unpacks a zip archive (stream) to disk, checks it and copies it over the existing uSync folder. + /// + Task UnpackImportFromStreamAsync(Stream stream); } \ No newline at end of file diff --git a/uSync.BackOffice/Services/ISyncFileService.cs b/uSync.BackOffice/Services/ISyncFileService.cs index 839886777..1cafae4bd 100644 --- a/uSync.BackOffice/Services/ISyncFileService.cs +++ b/uSync.BackOffice/Services/ISyncFileService.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.IO; using System.Threading.Tasks; using System.Xml.Linq; @@ -200,5 +201,12 @@ public interface ISyncFileService /// /// run some basic checks on a folder to see if it looks ok. /// - List VerifyFolder(string folder, string extension); + [Obsolete("Use VerifyFolderAsync instead will be removed in v19")] + List VerifyFolder(string folder, string extension) + => VerifyFolderAsync(folder, extension).GetAwaiter().GetResult(); + + /// + /// run some basic checks on a folder to see if it looks ok. + /// + Task> VerifyFolderAsync(string folder, string extension); } \ No newline at end of file diff --git a/uSync.BackOffice/Services/SyncActionService.cs b/uSync.BackOffice/Services/SyncActionService.cs index 07fc5d00f..3afbc06e1 100644 --- a/uSync.BackOffice/Services/SyncActionService.cs +++ b/uSync.BackOffice/Services/SyncActionService.cs @@ -231,7 +231,7 @@ public Stream GetExportFolderAsStream() return _uSyncService.CompressFolder(_uSyncConfig.GetWorkingFolder()); } - public UploadImportResult UnpackImportFromStream(Stream stream) + public async Task UnpackImportFromStreamAsync(Stream stream) { var tempFolder = Path.Combine(_uSyncTempPath, Path.GetFileNameWithoutExtension(Path.GetRandomFileName())) ?? $"{_uSyncTempPath}{Path.DirectorySeparatorChar}{Path.GetFileNameWithoutExtension(Path.GetRandomFileName()) ?? Guid.NewGuid().ToString()}"; @@ -242,7 +242,7 @@ public UploadImportResult UnpackImportFromStream(Stream stream) { _uSyncService.DeCompressFile(stream, tempFolder); - var errors = _syncFileService.VerifyFolder(tempFolder, + var errors = await _syncFileService.VerifyFolderAsync(tempFolder, _uSyncConfig.Settings.DefaultExtension); if (errors.Count > 0) diff --git a/uSync.BackOffice/Services/SyncFileService.cs b/uSync.BackOffice/Services/SyncFileService.cs index 6a0c96c9c..38bd4f980 100644 --- a/uSync.BackOffice/Services/SyncFileService.cs +++ b/uSync.BackOffice/Services/SyncFileService.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Logging; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -348,7 +349,7 @@ public async Task MakeSingleExportFromFolders(string[] folders, string item } /// - public List VerifyFolder(string folder, string extension) + public async Task> VerifyFolderAsync(string folder, string extension) { var resolvedFolder = GetAbsPath(folder); if (!DirectoryExists(resolvedFolder)) @@ -371,7 +372,7 @@ public List VerifyFolder(string folder, string extension) { try { - var node = LoadXElementAsync(file).Result; + var node = await LoadXElementAsync(file); if (!node.IsEmptyItem()) { @@ -489,29 +490,33 @@ public async Task> MergeFoldersAsync(string[] folde private static XElement MergeNodes(XElement source, XElement target, ISyncTrackerBase? trackerBase) => trackerBase is null ? target : trackerBase.MergeFiles(source, target) ?? target; - private async Task>> GetFolderItemsAsync(string folder, string extension) + private async Task>> GetFolderItemsAsync( + string folder, string extension) { - var items = new List>(); + var filePaths = GetFilePaths(folder, extension); + var results = new ConcurrentBag>(); - foreach (var file in GetFilePaths(folder, extension)) - { - var element = await LoadXElementSafeAsync(file); - if (element != null) + await Parallel.ForEachAsync( + filePaths, + new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }, + async (file, _) => { - var path = file.Substring(folder.Length); - - items.Add(new KeyValuePair( - key: path, - value: new OrderedNodeInfo( - filename: file, - node: element, - level: (element.GetLevel() * 1000) + element.GetItemSortOrder(), - path: path, - isRoot: true))); - } - } + var element = await LoadXElementSafeAsync(file); + if (element is not null) + { + var path = file.Substring(folder.Length); + results.Add(new KeyValuePair( + key: path, + value: new OrderedNodeInfo( + filename: file, + node: element, + level: (element.GetLevel() * 1000) + element.GetItemSortOrder(), + path: path, + isRoot: true))); + } + }); - return items; + return results; } /// diff --git a/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs b/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs index 0f37fa09b..764ab9b9c 100644 --- a/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs +++ b/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs @@ -74,7 +74,7 @@ protected override async Task> CleanFolderAsync(string // be a little slower (not much though) // we cache this, (it is cleared on an ImportAll) - var keys = GetFolderKeys(folder, flat); + var keys = await GetFolderKeysAsync(folder, flat); if (keys.Count > 0) { // move parent to here, we only need to check it if there are files. diff --git a/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs b/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs index 6eadf6b42..c5834c684 100644 --- a/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs +++ b/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs @@ -617,7 +617,7 @@ protected virtual async Task> CleanFolderAsync(string c // be a little slower (not much though) // we cache this, (it is cleared on an ImportAll) - var keys = GetFolderKeys(folder, flat); + var keys = await GetFolderKeysAsync(folder, flat); if (keys.Count > 0) { // move parent to here, we only need to check it if there are files. @@ -664,39 +664,50 @@ public Task PreCacheFolderKeysAsync(string folder, IList folderKeys) /// so we cache it, and if we are using the flat folder structure, then /// we only do it once, so its quicker. /// + [Obsolete("Use GetFolderKeysAsync instead, will be removed in v19")] protected IList GetFolderKeys(string folder, bool flat) + => GetFolderKeysAsync(folder, flat).GetAwaiter().GetResult(); + + /// + /// Get the GUIDs for all items in a folder + /// + /// + /// This is disk intensive, (checking the .config files all the time) + /// so we cache it, and if we are using the flat folder structure, then + /// we only do it once, so its quicker. + /// + protected async Task> GetFolderKeysAsync(string folder, bool flat) { // We only need to load all the keys once per handler (if all items are in a folder that key will be used). var folderKey = folder.GetHashCode(); var cacheKey = $"{GetCacheKeyBase()}_{folderKey}"; + var cached = runtimeCache.GetCacheItem>(cacheKey); + if (cached is not null) return cached; - return runtimeCache.GetCacheItem(cacheKey, () => - { - if (logger.IsEnabled(LogLevel.Debug)) - logger.LogDebug("Getting Folder Keys : {cacheKey}", cacheKey); + if (logger.IsEnabled(LogLevel.Debug)) + logger.LogDebug("Getting Folder Keys : {cacheKey}", cacheKey); - // when it's not flat structure we also get the sub folders. (extra defensive get them all) - var keys = new List(); - var files = syncFileService.GetFiles(folder, $"*.{this.uSyncConfig.Settings.DefaultExtension}", !flat).ToList(); + // when it's not flat structure we also get the sub folders. (extra defensive get them all) + var keys = new List(); + var files = syncFileService.GetFiles(folder, $"*.{this.uSyncConfig.Settings.DefaultExtension}", !flat); - foreach (var file in files) + foreach (var file in files) + { + var node = await syncFileService.LoadXElementAsync(file); + var key = node.GetKey(); + if (key != Guid.Empty && !keys.Contains(key)) { - var node = syncFileService.LoadXElementAsync(file).Result; - var key = node.GetKey(); - if (key != Guid.Empty && !keys.Contains(key)) - { - keys.Add(key); - } + keys.Add(key); } + } - if (logger.IsEnabled(LogLevel.Debug)) - logger.LogDebug("Loaded {count} keys from {folder} [{cacheKey}]", keys.Count, folder, cacheKey); - - return keys; + if (logger.IsEnabled(LogLevel.Debug)) + logger.LogDebug("Loaded {count} keys from {folder} [{cacheKey}]", keys.Count, folder, cacheKey); - }, null) ?? []; + runtimeCache.GetCacheItem(cacheKey, () => keys); + return keys; } /// @@ -872,7 +883,7 @@ public async Task> ExportAsync(Udi udi, string[] folder return [uSyncAction.Fail(nameof(udi), this.handlerType, this.ItemType, ChangeType.Fail, $"Item not found {udi}", new KeyNotFoundException(nameof(udi)))]; - + } /// @@ -1325,7 +1336,7 @@ public virtual async Task> ReportElementSingleAsync(XEl { return [uSyncActionHelper .ReportActionFail(Path.GetFileName(node.GetAlias()), $"format error {fex.Message}")]; - + } } @@ -1501,17 +1512,16 @@ protected virtual async Task ExportDeletedItemAsync(TObject item, string[] folde if (item == null) return; var targetFolder = folders.Last(); - var filename = (await GetPathAsync(targetFolder, item, config.GuidNames, config.UseFlatStructure)) .ToAppSafeFileName(); if (IsLockedAtRoot(folders, filename.Substring(targetFolder.Length + 1))) - { - // don't do anything this thing exists at a higher level. ! return; - } - if (await ShouldExportDeletedFileAsync(item, config) is false) return; + // Only perform the expensive full-serialize check if this handler + // actually overrides ShouldExportAsync (i.e. has filtering logic). + if (HandlerHasShouldExportOverride() && await ShouldExportDeletedFileAsync(item, config) is false) + return; var attempt = await serializer.SerializeEmptyAsync(item, SyncActionType.Delete, string.Empty); if (attempt.Item is not null && await ShouldExportAsync(attempt.Item, config) is true) @@ -1520,17 +1530,20 @@ protected virtual async Task ExportDeletedItemAsync(TObject item, string[] folde { await syncFileService.SaveXElementAsync(attempt.Item, filename); - // so check - it shouldn't (under normal operation) - // be possible for a clash to exist at delete, because nothing else - // will have changed (like name or location) - - // we only then do this if we are not using flat structure. if (!DefaultConfig.UseFlatStructure) await this.CleanUpAsync(item, filename, Path.Combine(folders.Last(), this.DefaultFolder)); } } } + // Cached reflection result — only computed once per handler type. + private bool? _hasShouldExportOverride; + private bool HandlerHasShouldExportOverride() + => _hasShouldExportOverride ??= GetType() + .GetMethod(nameof(ShouldExportAsync), + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance) + ?.DeclaringType != typeof(SyncHandlerRoot); + private async Task ShouldExportDeletedFileAsync(TObject item, HandlerSettings config) { try @@ -1541,7 +1554,7 @@ private async Task ShouldExportDeletedFileAsync(TObject item, HandlerSetti } catch (Exception ex) { - logger.LogWarning(ex, "Failed to calculate if this item should be exported when deleted, the common option is yes, so we will"); + logger.LogWarning("Error while checking if should export deleted file: {message}", ex.Message); return true; } } @@ -1608,7 +1621,6 @@ protected virtual async Task CleanUpAsync(TObject item, string newFile, string f await CleanUpAsync(item, newFile, children); } } - #endregion // 98% of the time the serializer can do all these calls for us, diff --git a/uSync.Core/Serialization/SyncContainerSerializerBase.cs b/uSync.Core/Serialization/SyncContainerSerializerBase.cs index be67b39f5..66f97d73a 100644 --- a/uSync.Core/Serialization/SyncContainerSerializerBase.cs +++ b/uSync.Core/Serialization/SyncContainerSerializerBase.cs @@ -194,18 +194,20 @@ public virtual async Task> GetContainersAsync(TObje { if (entityTypeContainerTypeService is null) return []; - var parent = await entityTypeContainerTypeService.GetParentAsync(item); - if (parent is null) return []; + if (_containersCache.TryGetValue(item.ParentId, out var cached) && cached is not null) + return cached; + + var containers = new List(); - var containers = new List() { parent }; + var parent = await entityTypeContainerTypeService.GetParentAsync(item); while (parent is not null) { + containers.Add(parent); parent = await entityTypeContainerTypeService.GetParentAsync(parent); - if (parent is not null) - containers.Add(parent); } + _containersCache.TryAdd(item.ParentId, containers); return containers; } @@ -302,9 +304,13 @@ public virtual async Task SaveContainerAsync(Guid parent, EntityContainer contai /// only used on serialization, allows us to only build the folder path for a set of containers once. /// private ConcurrentDictionary _folderCache = []; + private ConcurrentDictionary> _containersCache = []; private void ClearFolderCache() - => _folderCache = []; + { + _folderCache = []; + _containersCache = []; + } public void InitializeCache() { From 5f729f952f7bb2f42eca926782a8273fd73d403b Mon Sep 17 00:00:00 2001 From: Kevin Jump Date: Fri, 27 Mar 2026 13:47:49 +0000 Subject: [PATCH 2/5] Update uSync.Core/Serialization/SyncContainerSerializerBase.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- uSync.Core/Serialization/SyncContainerSerializerBase.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uSync.Core/Serialization/SyncContainerSerializerBase.cs b/uSync.Core/Serialization/SyncContainerSerializerBase.cs index 66f97d73a..d083cfc63 100644 --- a/uSync.Core/Serialization/SyncContainerSerializerBase.cs +++ b/uSync.Core/Serialization/SyncContainerSerializerBase.cs @@ -207,8 +207,9 @@ public virtual async Task> GetContainersAsync(TObje parent = await entityTypeContainerTypeService.GetParentAsync(parent); } - _containersCache.TryAdd(item.ParentId, containers); - return containers; + var containersArray = containers.ToArray(); + _containersCache.TryAdd(item.ParentId, containersArray); + return containersArray; } From ef143614c177a42a261608fa184d92143874fba6 Mon Sep 17 00:00:00 2001 From: Kevin Jump Date: Fri, 27 Mar 2026 13:48:14 +0000 Subject: [PATCH 3/5] Update uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs b/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs index c5834c684..c7076f293 100644 --- a/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs +++ b/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs @@ -1554,7 +1554,7 @@ private async Task ShouldExportDeletedFileAsync(TObject item, HandlerSetti } catch (Exception ex) { - logger.LogWarning("Error while checking if should export deleted file: {message}", ex.Message); + logger.LogWarning(ex, "Error while checking if should export deleted file."); return true; } } From ad928ee6e408e63e2ca6a98c019672dd747c6cf3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 27 Mar 2026 13:49:39 +0000 Subject: [PATCH 4/5] Use HashSet for O(1) deduplication in GetFolderKeysAsync and fix containersCache type mismatch Agent-Logs-Url: https://github.com/KevinJump/uSync/sessions/1bfc228e-e9a6-4247-a9b2-89154e279425 Co-authored-by: KevinJump <431231+KevinJump@users.noreply.github.com> --- uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs | 8 +++++--- uSync.Core/Serialization/SyncContainerSerializerBase.cs | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs b/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs index c7076f293..ea7b62f8f 100644 --- a/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs +++ b/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs @@ -690,19 +690,21 @@ protected async Task> GetFolderKeysAsync(string folder, bool flat) logger.LogDebug("Getting Folder Keys : {cacheKey}", cacheKey); // when it's not flat structure we also get the sub folders. (extra defensive get them all) - var keys = new List(); + var keySet = new HashSet(); var files = syncFileService.GetFiles(folder, $"*.{this.uSyncConfig.Settings.DefaultExtension}", !flat); foreach (var file in files) { var node = await syncFileService.LoadXElementAsync(file); var key = node.GetKey(); - if (key != Guid.Empty && !keys.Contains(key)) + if (key != Guid.Empty) { - keys.Add(key); + keySet.Add(key); } } + var keys = keySet.ToList(); + if (logger.IsEnabled(LogLevel.Debug)) logger.LogDebug("Loaded {count} keys from {folder} [{cacheKey}]", keys.Count, folder, cacheKey); diff --git a/uSync.Core/Serialization/SyncContainerSerializerBase.cs b/uSync.Core/Serialization/SyncContainerSerializerBase.cs index d083cfc63..f7265f783 100644 --- a/uSync.Core/Serialization/SyncContainerSerializerBase.cs +++ b/uSync.Core/Serialization/SyncContainerSerializerBase.cs @@ -305,7 +305,7 @@ public virtual async Task SaveContainerAsync(Guid parent, EntityContainer contai /// only used on serialization, allows us to only build the folder path for a set of containers once. /// private ConcurrentDictionary _folderCache = []; - private ConcurrentDictionary> _containersCache = []; + private ConcurrentDictionary _containersCache = []; private void ClearFolderCache() { From 344a8fe922047fdde3ecd826f8fea2c6bee448b8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 27 Mar 2026 13:59:08 +0000 Subject: [PATCH 5/5] Clone XmlReaderSettings per call to ensure thread safety in parallel XML loading Agent-Logs-Url: https://github.com/KevinJump/uSync/sessions/408ea6a5-d136-43a2-b422-9855153456da Co-authored-by: KevinJump <431231+KevinJump@users.noreply.github.com> --- uSync.BackOffice/Services/SyncFileService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uSync.BackOffice/Services/SyncFileService.cs b/uSync.BackOffice/Services/SyncFileService.cs index 38bd4f980..4b20b4e46 100644 --- a/uSync.BackOffice/Services/SyncFileService.cs +++ b/uSync.BackOffice/Services/SyncFileService.cs @@ -198,7 +198,7 @@ public async Task LoadXElementAsync(string file) if (stream is null) throw new FileNotFoundException($"Cannot create stream for {file}"); ; - using (var xmlReader = XmlReader.Create(stream, _readerSettings)) + using (var xmlReader = XmlReader.Create(stream, _readerSettings.Clone())) { return await XElement.LoadAsync(xmlReader, LoadOptions.PreserveWhitespace, CancellationToken.None); }