diff --git a/uSync.BackOffice/Services/ISyncActionService.cs b/uSync.BackOffice/Services/ISyncActionService.cs index 3742f813..f1a80bfd 100644 --- a/uSync.BackOffice/Services/ISyncActionService.cs +++ b/uSync.BackOffice/Services/ISyncActionService.cs @@ -68,5 +68,12 @@ public interface ISyncActionService /// /// unpacks a zip archive (stream) to disk, checks it and copies it over the existing uSync folder. /// - UploadImportResult UnpackImportFromStream(Stream stream); + [Obsolete("Use UnpackImportFromStreamAsync(Stream stream) will be removed in v19")] + UploadImportResult UnpackImportFromStream(Stream stream) + => UnpackImportFromStreamAsync(stream).GetAwaiter().GetResult(); + + /// + /// unpacks a zip archive (stream) to disk, checks it and copies it over the existing uSync folder. + /// + Task UnpackImportFromStreamAsync(Stream stream); } \ No newline at end of file diff --git a/uSync.BackOffice/Services/ISyncFileService.cs b/uSync.BackOffice/Services/ISyncFileService.cs index 83988677..1cafae4b 100644 --- a/uSync.BackOffice/Services/ISyncFileService.cs +++ b/uSync.BackOffice/Services/ISyncFileService.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.IO; using System.Threading.Tasks; using System.Xml.Linq; @@ -200,5 +201,12 @@ public interface ISyncFileService /// /// run some basic checks on a folder to see if it looks ok. /// - List VerifyFolder(string folder, string extension); + [Obsolete("Use VerifyFolderAsync instead will be removed in v19")] + List VerifyFolder(string folder, string extension) + => VerifyFolderAsync(folder, extension).GetAwaiter().GetResult(); + + /// + /// run some basic checks on a folder to see if it looks ok. + /// + Task> VerifyFolderAsync(string folder, string extension); } \ No newline at end of file diff --git a/uSync.BackOffice/Services/SyncActionService.cs b/uSync.BackOffice/Services/SyncActionService.cs index 07fc5d00..3afbc06e 100644 --- a/uSync.BackOffice/Services/SyncActionService.cs +++ b/uSync.BackOffice/Services/SyncActionService.cs @@ -231,7 +231,7 @@ public Stream GetExportFolderAsStream() return _uSyncService.CompressFolder(_uSyncConfig.GetWorkingFolder()); } - public UploadImportResult UnpackImportFromStream(Stream stream) + public async Task UnpackImportFromStreamAsync(Stream stream) { var tempFolder = Path.Combine(_uSyncTempPath, Path.GetFileNameWithoutExtension(Path.GetRandomFileName())) ?? $"{_uSyncTempPath}{Path.DirectorySeparatorChar}{Path.GetFileNameWithoutExtension(Path.GetRandomFileName()) ?? Guid.NewGuid().ToString()}"; @@ -242,7 +242,7 @@ public UploadImportResult UnpackImportFromStream(Stream stream) { _uSyncService.DeCompressFile(stream, tempFolder); - var errors = _syncFileService.VerifyFolder(tempFolder, + var errors = await _syncFileService.VerifyFolderAsync(tempFolder, _uSyncConfig.Settings.DefaultExtension); if (errors.Count > 0) diff --git a/uSync.BackOffice/Services/SyncFileService.cs b/uSync.BackOffice/Services/SyncFileService.cs index 6a0c96c9..4b20b4e4 100644 --- a/uSync.BackOffice/Services/SyncFileService.cs +++ b/uSync.BackOffice/Services/SyncFileService.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Logging; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -197,7 +198,7 @@ public async Task LoadXElementAsync(string file) if (stream is null) throw new FileNotFoundException($"Cannot create stream for {file}"); ; - using (var xmlReader = XmlReader.Create(stream, _readerSettings)) + using (var xmlReader = XmlReader.Create(stream, _readerSettings.Clone())) { return await XElement.LoadAsync(xmlReader, LoadOptions.PreserveWhitespace, CancellationToken.None); } @@ -348,7 +349,7 @@ public async Task MakeSingleExportFromFolders(string[] folders, string item } /// - public List VerifyFolder(string folder, string extension) + public async Task> VerifyFolderAsync(string folder, string extension) { var resolvedFolder = GetAbsPath(folder); if (!DirectoryExists(resolvedFolder)) @@ -371,7 +372,7 @@ public List VerifyFolder(string folder, string extension) { try { - var node = LoadXElementAsync(file).Result; + var node = await LoadXElementAsync(file); if (!node.IsEmptyItem()) { @@ -489,29 +490,33 @@ public async Task> MergeFoldersAsync(string[] folde private static XElement MergeNodes(XElement source, XElement target, ISyncTrackerBase? trackerBase) => trackerBase is null ? target : trackerBase.MergeFiles(source, target) ?? target; - private async Task>> GetFolderItemsAsync(string folder, string extension) + private async Task>> GetFolderItemsAsync( + string folder, string extension) { - var items = new List>(); + var filePaths = GetFilePaths(folder, extension); + var results = new ConcurrentBag>(); - foreach (var file in GetFilePaths(folder, extension)) - { - var element = await LoadXElementSafeAsync(file); - if (element != null) + await Parallel.ForEachAsync( + filePaths, + new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }, + async (file, _) => { - var path = file.Substring(folder.Length); - - items.Add(new KeyValuePair( - key: path, - value: new OrderedNodeInfo( - filename: file, - node: element, - level: (element.GetLevel() * 1000) + element.GetItemSortOrder(), - path: path, - isRoot: true))); - } - } + var element = await LoadXElementSafeAsync(file); + if (element is not null) + { + var path = file.Substring(folder.Length); + results.Add(new KeyValuePair( + key: path, + value: new OrderedNodeInfo( + filename: file, + node: element, + level: (element.GetLevel() * 1000) + element.GetItemSortOrder(), + path: path, + isRoot: true))); + } + }); - return items; + return results; } /// diff --git a/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs b/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs index 0f37fa09..764ab9b9 100644 --- a/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs +++ b/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs @@ -74,7 +74,7 @@ protected override async Task> CleanFolderAsync(string // be a little slower (not much though) // we cache this, (it is cleared on an ImportAll) - var keys = GetFolderKeys(folder, flat); + var keys = await GetFolderKeysAsync(folder, flat); if (keys.Count > 0) { // move parent to here, we only need to check it if there are files. diff --git a/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs b/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs index 6eadf6b4..ea7b62f8 100644 --- a/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs +++ b/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs @@ -617,7 +617,7 @@ protected virtual async Task> CleanFolderAsync(string c // be a little slower (not much though) // we cache this, (it is cleared on an ImportAll) - var keys = GetFolderKeys(folder, flat); + var keys = await GetFolderKeysAsync(folder, flat); if (keys.Count > 0) { // move parent to here, we only need to check it if there are files. @@ -664,39 +664,52 @@ public Task PreCacheFolderKeysAsync(string folder, IList folderKeys) /// so we cache it, and if we are using the flat folder structure, then /// we only do it once, so its quicker. /// + [Obsolete("Use GetFolderKeysAsync instead, will be removed in v19")] protected IList GetFolderKeys(string folder, bool flat) + => GetFolderKeysAsync(folder, flat).GetAwaiter().GetResult(); + + /// + /// Get the GUIDs for all items in a folder + /// + /// + /// This is disk intensive, (checking the .config files all the time) + /// so we cache it, and if we are using the flat folder structure, then + /// we only do it once, so its quicker. + /// + protected async Task> GetFolderKeysAsync(string folder, bool flat) { // We only need to load all the keys once per handler (if all items are in a folder that key will be used). var folderKey = folder.GetHashCode(); var cacheKey = $"{GetCacheKeyBase()}_{folderKey}"; + var cached = runtimeCache.GetCacheItem>(cacheKey); + if (cached is not null) return cached; - return runtimeCache.GetCacheItem(cacheKey, () => - { - if (logger.IsEnabled(LogLevel.Debug)) - logger.LogDebug("Getting Folder Keys : {cacheKey}", cacheKey); + if (logger.IsEnabled(LogLevel.Debug)) + logger.LogDebug("Getting Folder Keys : {cacheKey}", cacheKey); - // when it's not flat structure we also get the sub folders. (extra defensive get them all) - var keys = new List(); - var files = syncFileService.GetFiles(folder, $"*.{this.uSyncConfig.Settings.DefaultExtension}", !flat).ToList(); + // when it's not flat structure we also get the sub folders. (extra defensive get them all) + var keySet = new HashSet(); + var files = syncFileService.GetFiles(folder, $"*.{this.uSyncConfig.Settings.DefaultExtension}", !flat); - foreach (var file in files) + foreach (var file in files) + { + var node = await syncFileService.LoadXElementAsync(file); + var key = node.GetKey(); + if (key != Guid.Empty) { - var node = syncFileService.LoadXElementAsync(file).Result; - var key = node.GetKey(); - if (key != Guid.Empty && !keys.Contains(key)) - { - keys.Add(key); - } + keySet.Add(key); } + } - if (logger.IsEnabled(LogLevel.Debug)) - logger.LogDebug("Loaded {count} keys from {folder} [{cacheKey}]", keys.Count, folder, cacheKey); + var keys = keySet.ToList(); - return keys; + if (logger.IsEnabled(LogLevel.Debug)) + logger.LogDebug("Loaded {count} keys from {folder} [{cacheKey}]", keys.Count, folder, cacheKey); - }, null) ?? []; + runtimeCache.GetCacheItem(cacheKey, () => keys); + return keys; } /// @@ -872,7 +885,7 @@ public async Task> ExportAsync(Udi udi, string[] folder return [uSyncAction.Fail(nameof(udi), this.handlerType, this.ItemType, ChangeType.Fail, $"Item not found {udi}", new KeyNotFoundException(nameof(udi)))]; - + } /// @@ -1325,7 +1338,7 @@ public virtual async Task> ReportElementSingleAsync(XEl { return [uSyncActionHelper .ReportActionFail(Path.GetFileName(node.GetAlias()), $"format error {fex.Message}")]; - + } } @@ -1501,17 +1514,16 @@ protected virtual async Task ExportDeletedItemAsync(TObject item, string[] folde if (item == null) return; var targetFolder = folders.Last(); - var filename = (await GetPathAsync(targetFolder, item, config.GuidNames, config.UseFlatStructure)) .ToAppSafeFileName(); if (IsLockedAtRoot(folders, filename.Substring(targetFolder.Length + 1))) - { - // don't do anything this thing exists at a higher level. ! return; - } - if (await ShouldExportDeletedFileAsync(item, config) is false) return; + // Only perform the expensive full-serialize check if this handler + // actually overrides ShouldExportAsync (i.e. has filtering logic). + if (HandlerHasShouldExportOverride() && await ShouldExportDeletedFileAsync(item, config) is false) + return; var attempt = await serializer.SerializeEmptyAsync(item, SyncActionType.Delete, string.Empty); if (attempt.Item is not null && await ShouldExportAsync(attempt.Item, config) is true) @@ -1520,17 +1532,20 @@ protected virtual async Task ExportDeletedItemAsync(TObject item, string[] folde { await syncFileService.SaveXElementAsync(attempt.Item, filename); - // so check - it shouldn't (under normal operation) - // be possible for a clash to exist at delete, because nothing else - // will have changed (like name or location) - - // we only then do this if we are not using flat structure. if (!DefaultConfig.UseFlatStructure) await this.CleanUpAsync(item, filename, Path.Combine(folders.Last(), this.DefaultFolder)); } } } + // Cached reflection result — only computed once per handler type. + private bool? _hasShouldExportOverride; + private bool HandlerHasShouldExportOverride() + => _hasShouldExportOverride ??= GetType() + .GetMethod(nameof(ShouldExportAsync), + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance) + ?.DeclaringType != typeof(SyncHandlerRoot); + private async Task ShouldExportDeletedFileAsync(TObject item, HandlerSettings config) { try @@ -1541,7 +1556,7 @@ private async Task ShouldExportDeletedFileAsync(TObject item, HandlerSetti } catch (Exception ex) { - logger.LogWarning(ex, "Failed to calculate if this item should be exported when deleted, the common option is yes, so we will"); + logger.LogWarning(ex, "Error while checking if should export deleted file."); return true; } } @@ -1608,7 +1623,6 @@ protected virtual async Task CleanUpAsync(TObject item, string newFile, string f await CleanUpAsync(item, newFile, children); } } - #endregion // 98% of the time the serializer can do all these calls for us, diff --git a/uSync.Core/Serialization/SyncContainerSerializerBase.cs b/uSync.Core/Serialization/SyncContainerSerializerBase.cs index be67b39f..f7265f78 100644 --- a/uSync.Core/Serialization/SyncContainerSerializerBase.cs +++ b/uSync.Core/Serialization/SyncContainerSerializerBase.cs @@ -194,19 +194,22 @@ public virtual async Task> GetContainersAsync(TObje { if (entityTypeContainerTypeService is null) return []; - var parent = await entityTypeContainerTypeService.GetParentAsync(item); - if (parent is null) return []; + if (_containersCache.TryGetValue(item.ParentId, out var cached) && cached is not null) + return cached; + + var containers = new List(); - var containers = new List() { parent }; + var parent = await entityTypeContainerTypeService.GetParentAsync(item); while (parent is not null) { + containers.Add(parent); parent = await entityTypeContainerTypeService.GetParentAsync(parent); - if (parent is not null) - containers.Add(parent); } - return containers; + var containersArray = containers.ToArray(); + _containersCache.TryAdd(item.ParentId, containersArray); + return containersArray; } @@ -302,9 +305,13 @@ public virtual async Task SaveContainerAsync(Guid parent, EntityContainer contai /// only used on serialization, allows us to only build the folder path for a set of containers once. /// private ConcurrentDictionary _folderCache = []; + private ConcurrentDictionary _containersCache = []; private void ClearFolderCache() - => _folderCache = []; + { + _folderCache = []; + _containersCache = []; + } public void InitializeCache() {