VCI-80: Fix missing dlls in app_data

Build.cs

@@ -47,6 +47,9 @@ partial class Build : NukeBuild
     //    ToolPathResolver.ExecutingAssemblyDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
     //}
 
+
+    private static bool ClearTempBeforeExit { get; set; } = false;
+
     public static int Main()
     {
         var nukeFile = Directory.GetFiles(Directory.GetCurrentDirectory(), ".nuke");
@@ -64,7 +67,13 @@ public static int Main()
                 File.CreateText(Path.Combine(Directory.GetCurrentDirectory(), ".nuke")).Close();
             }
         }
+
         var exitCode = Execute<Build>(x => x.Compile);
+        if (ClearTempBeforeExit)
+        {
+            FileSystemTasks.DeleteDirectory(TemporaryDirectory);
+        }
+
         return ExitCode ?? exitCode;
     }
 
@@ -951,6 +960,7 @@ void FinishReleaseOrHotfix(string tag)
     Target ClearTemp => _ => _
         .Executes(() =>
         {
-            FileSystemTasks.DeleteDirectory(TemporaryDirectory);
+            ClearTempBeforeExit = true;
         });
+
 }

PlatformTools/Build.PackageManager.cs

@@ -176,24 +176,24 @@ private bool NeedToInstallPlatform(string version)
             {
                 Id = moduleInstall.Id,
                 Version = moduleInstall.Version,
-                Dependencies = SkipDependencySolving ? null : externalModule.Dependencies.Select(d => new ManifestDependency()
+                Dependencies = SkipDependencySolving ? null : externalModule?.Dependencies?.Select(d => new ManifestDependency()
                 {
                     Id = d.Id,
                     Version = d.Version.ToString()
                 }).ToArray(),
-                PackageUrl = externalModule.Ref.Replace(externalModule.Version.ToString(), moduleInstall.Version),
-                Authors = externalModule.Authors.ToArray(),
-                PlatformVersion = externalModule.PlatformVersion.ToString(),
-                Incompatibilities = externalModule.Incompatibilities.Select(d => new ManifestDependency()
+                PackageUrl = externalModule.Ref?.Replace(externalModule.Version.ToString(), moduleInstall.Version),
+                Authors = externalModule.Authors?.ToArray(),
+                PlatformVersion = externalModule.PlatformVersion?.ToString(),
+                Incompatibilities = externalModule.Incompatibilities?.Select(d => new ManifestDependency()
                 {
                     Id = d.Id,
                     Version = d.Version.ToString()
                 }).ToArray(),
-                Groups = externalModule.Groups.Select(g => g).ToArray(),
+                Groups = externalModule.Groups?.Select(g => g).ToArray(),
                 Copyright = externalModule.Copyright,
                 Description = externalModule.Description,
                 IconUrl = externalModule.IconUrl,
-                Owners = externalModule.Owners.ToArray(),
+                Owners = externalModule.Owners?.ToArray(),
                 ProjectUrl = externalModule.ProjectUrl,
                 ReleaseNotes = externalModule.ReleaseNotes,
                 Tags = externalModule.Tags,
@@ -214,6 +214,9 @@ private bool NeedToInstallPlatform(string version)
             modulesToInstall.AddRange(missingModules);
         }
         moduleInstaller.Install(modulesToInstall.Where(m => !m.IsInstalled), progress);
+        AbsolutePath discoveryAbsolutePath = (AbsolutePath) Path.GetFullPath(discoveryPath);
+        var zipFiles = discoveryAbsolutePath.GlobFiles("*/*.zip");
+        zipFiles.ForEach(path => FileSystemTasks.DeleteFile(path));
         localModuleCatalog.Reload();
     });
 
@@ -228,7 +231,7 @@ private bool NeedToInstallPlatform(string version)
         Module.ForEach(m => FileSystemTasks.DeleteDirectory(Path.Combine(discoveryPath, m)));
         packageManifest.Modules.RemoveAll(m => Module.Contains(m.Id));
         PackageManager.ToFile(packageManifest);
-        localModulesCatalog.Load();
+        localModulesCatalog.Reload();
     });
 
     Target Update => _ => _

PlatformTools/ExtModuleCatalog.cs

@@ -15,13 +15,13 @@ class ExtModuleCatalog
     {
         private static ExternalModuleCatalog _catalog;
 
-        public static ExternalModuleCatalog GetCatalog(string authToken, LocalStorageModuleCatalog localCatalog, IEnumerable<string> manifestUrls)
+        public static ExternalModuleCatalog GetCatalog(string authToken, ILocalModuleCatalog localCatalog, IEnumerable<string> manifestUrls)
         {
             var options = GetOptions(authToken, manifestUrls);
             return GetCatalog(options, localCatalog);
         }
 
-        public static ExternalModuleCatalog GetCatalog(IOptions<ExternalModuleCatalogOptions> options, LocalStorageModuleCatalog localCatalog)
+        public static ExternalModuleCatalog GetCatalog(IOptions<ExternalModuleCatalogOptions> options, ILocalModuleCatalog localCatalog)
         {
             if (_catalog == null)
             {

PlatformTools/LocalCatalog.cs

@@ -0,0 +1,240 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using VirtoCommerce.Platform.Core.Common;
+using VirtoCommerce.Platform.Core.Exceptions;
+using VirtoCommerce.Platform.Core.Modularity;
+using VirtoCommerce.Platform.DistributedLock;
+using VirtoCommerce.Platform.Modules;
+
+namespace PlatformTools
+{
+    public class LocalCatalog : LocalStorageModuleCatalog
+    {
+
+        private readonly LocalStorageModuleCatalogOptions _options;
+        private readonly ILogger<LocalStorageModuleCatalog> _logger;
+        private readonly IDistributedLockProvider _distributedLockProvider;
+        private readonly string _discoveryPath;
+        public LocalCatalog(IOptions<LocalStorageModuleCatalogOptions> options, IDistributedLockProvider distributedLockProvider, ILogger<LocalStorageModuleCatalog> logger):
+            base(options, distributedLockProvider, logger)
+        {
+            _options = options.Value;
+            _logger = logger;
+            _distributedLockProvider = distributedLockProvider;
+            _discoveryPath = _options.DiscoveryPath;
+        }
+
+        protected override void InnerLoad()
+        {
+            if (string.IsNullOrEmpty(_options.ProbingPath))
+                throw new InvalidOperationException("The ProbingPath cannot contain a null value or be empty");
+            if (string.IsNullOrEmpty(_options.DiscoveryPath))
+                throw new InvalidOperationException("The DiscoveryPath cannot contain a null value or be empty");
+
+
+            var manifests = GetModuleManifests();
+
+            var needToCopyAssemblies = _options.RefreshProbingFolderOnStart;
+
+            if (!Directory.Exists(_options.ProbingPath))
+            {
+                needToCopyAssemblies = true; // Force to refresh assemblies anyway, even if RefreshProbeFolderOnStart set to false, because the probing path is absent
+                Directory.CreateDirectory(_options.ProbingPath);
+            }
+
+            if (needToCopyAssemblies)
+            {
+                CopyAssembliesSynchronized(manifests);
+            }
+
+            foreach (var pair in manifests)
+            {
+                var manifest = pair.Value;
+
+                var moduleInfo = AbstractTypeFactory<ManifestModuleInfo>.TryCreateInstance();
+                moduleInfo.LoadFromManifest(manifest);
+                moduleInfo.FullPhysicalPath = Path.GetDirectoryName(pair.Key);
+
+                // Modules without assembly file don't need initialization
+                if (string.IsNullOrEmpty(manifest.AssemblyFile))
+                {
+                    moduleInfo.State = ModuleState.Initialized;
+                }
+                else
+                {
+                    //Set module assembly physical path for future loading by IModuleTypeLoader instance
+                    moduleInfo.Ref = GetFileAbsoluteUri(_options.ProbingPath, manifest.AssemblyFile);
+                }
+
+                moduleInfo.IsInstalled = true;
+                AddModule(moduleInfo);
+            }
+        }
+
+        private void CopyAssembliesSynchronized(IDictionary<string, ModuleManifest> manifests)
+        {
+            _distributedLockProvider.ExecuteSynhronized(GetSourceMark(), (x) =>
+            {
+                if (x != DistributedLockCondition.Delayed)
+                {
+                    CopyAssemblies(_discoveryPath, _options.ProbingPath); // Copy platform files if needed
+                    foreach (var pair in manifests)
+                    {
+                        var modulePath = Path.GetDirectoryName(pair.Key);
+                        CopyAssemblies(modulePath, _options.ProbingPath); // Copy module files if needed
+                    }
+                }
+                else // Delayed lock acquire, do nothing here with a notice logging
+                {
+                    _logger.LogInformation("Skip copy assemblies to ProbingPath for local storage (another instance made it)");
+                }
+            });
+        }
+
+        private string GetSourceMark()
+        {
+            var markerFilePath = Path.Combine(_options.ProbingPath, "storage.mark");
+            var marker = Guid.NewGuid().ToString();
+            try
+            {
+                if (File.Exists(markerFilePath))
+                {
+                    using (var stream = File.OpenText(markerFilePath))
+                    {
+                        marker = stream.ReadToEnd();
+                    }
+                }
+                else
+                {
+                    // Non-marked storage, mark by placing a file with resource id.                    
+                    using (var stream = File.CreateText(markerFilePath))
+                    {
+                        stream.Write(marker);
+                    }
+                }
+            }
+            catch (IOException exc)
+            {
+                throw new PlatformException($"An IO error occurred while marking local modules storage.", exc);
+            }
+            return $@"{nameof(LocalStorageModuleCatalog)}-{marker}";
+        }
+
+        private static string GetFileAbsoluteUri(string rootPath, string relativePath)
+        {
+            var builder = new UriBuilder
+            {
+                Host = string.Empty,
+                Scheme = Uri.UriSchemeFile,
+                Path = Path.GetFullPath(Path.Combine(rootPath, relativePath))
+            };
+
+            return builder.Uri.ToString();
+        }
+
+        private IDictionary<string, ModuleManifest> GetModuleManifests()
+        {
+            var result = new Dictionary<string, ModuleManifest>();
+
+            if (Directory.Exists(_options.DiscoveryPath))
+            {
+                foreach (var manifestFile in Directory.EnumerateFiles(_options.DiscoveryPath, "module.manifest", SearchOption.AllDirectories))
+                {
+                    if (!manifestFile.Contains("artifacts"))
+                    {
+                        var manifest = ManifestReader.Read(manifestFile);
+                        result.Add(manifestFile, manifest);
+                    }
+                }
+            }
+            return result;
+        }
+        private void CopyAssemblies(string sourceParentPath, string targetDirectoryPath)
+        {
+            if (sourceParentPath != null)
+            {
+                var sourceDirectoryPath = Path.Combine(sourceParentPath, "bin");
+
+                if (Directory.Exists(sourceDirectoryPath))
+                {
+                    foreach (var sourceFilePath in Directory.EnumerateFiles(sourceDirectoryPath, "*.*", SearchOption.AllDirectories))
+                    {
+                        // Copy all assembly related files except assemblies that are inlcuded in TPA list
+                        if (IsAssemblyRelatedFile(sourceFilePath))
+                        {
+                            // Copy localization resource files to related subfolders
+                            var targetFilePath = Path.Combine(
+                                IsLocalizationFile(sourceFilePath) ? Path.Combine(targetDirectoryPath, Path.GetFileName(Path.GetDirectoryName(sourceFilePath)))
+                                : targetDirectoryPath,
+                                Path.GetFileName(sourceFilePath));
+                            CopyFile(sourceFilePath, targetFilePath);
+                        }
+                    }
+                }
+            }
+        }
+
+        private bool IsAssemblyRelatedFile(string path)
+        {
+            return _options.AssemblyFileExtensions.Union(_options.AssemblyServiceFileExtensions).Any(x => path.EndsWith(x, StringComparison.OrdinalIgnoreCase));
+        }
+
+        private bool IsAssemblyFile(string path)
+        {
+            return _options.AssemblyFileExtensions.Any(x => path.EndsWith(x, StringComparison.OrdinalIgnoreCase));
+        }
+
+        private bool IsLocalizationFile(string path)
+        {
+            return _options.LocalizationFileExtensions.Any(x => path.EndsWith(x, StringComparison.OrdinalIgnoreCase));
+        }
+
+        private void CopyFile(string sourceFilePath, string targetFilePath)
+        {
+            var sourceFileInfo = new FileInfo(sourceFilePath);
+            var targetFileInfo = new FileInfo(targetFilePath);
+
+            var sourceFileVersionInfo = FileVersionInfo.GetVersionInfo(sourceFilePath);
+            var sourceVersion = new Version(sourceFileVersionInfo.FileMajorPart, sourceFileVersionInfo.FileMinorPart, sourceFileVersionInfo.FileBuildPart, sourceFileVersionInfo.FilePrivatePart);
+            var targetVersion = sourceVersion;
+
+            if (targetFileInfo.Exists)
+            {
+                var targetFileVersionInfo = FileVersionInfo.GetVersionInfo(targetFilePath);
+                targetVersion = new Version(targetFileVersionInfo.FileMajorPart, targetFileVersionInfo.FileMinorPart, targetFileVersionInfo.FileBuildPart, targetFileVersionInfo.FilePrivatePart);
+            }
+
+            var versionsAreSameButLaterDate = (sourceVersion == targetVersion && targetFileInfo.Exists && sourceFileInfo.Exists && targetFileInfo.LastWriteTimeUtc < sourceFileInfo.LastWriteTimeUtc);
+            if (!targetFileInfo.Exists || sourceVersion > targetVersion || versionsAreSameButLaterDate)
+            {
+                var targetDirectoryPath = Path.GetDirectoryName(targetFilePath);
+                Directory.CreateDirectory(targetDirectoryPath);
+
+                try
+                {
+                    File.Copy(sourceFilePath, targetFilePath, true);
+                }
+                catch (IOException)
+                {
+                    // VP-3719: Need to catch to avoid possible problem when different instances are trying to update the same file with the same version but different dates in the probing folder.
+                    // We should not fail platform start in that case - just add warning into the log. In case of unability to place newer version - should fail platform start.
+                    if (versionsAreSameButLaterDate)
+                    {
+                        _logger.LogWarning($"File '{targetFilePath}' was not updated by '{sourceFilePath}' of the same version but later modified date, because probably it was used by another process");
+                    }
+                    else
+                    {
+                        throw;
+                    }
+                }
+            }
+        }
+    }
+
+
+}

PlatformTools/LocalModuleCatalog.cs

@@ -11,21 +11,21 @@ namespace PlatformTools
 {
     class LocalModuleCatalog
     {
-        private static LocalStorageModuleCatalog _catalog;
+        private static LocalCatalog _catalog;
 
-        public static LocalStorageModuleCatalog GetCatalog(string discoveryPath, string probingPath)
+        public static ILocalModuleCatalog GetCatalog(string discoveryPath, string probingPath)
         {
             var options = GetOptions(discoveryPath, probingPath);
             return GetCatalog(options);
         }
 
-        public static LocalStorageModuleCatalog GetCatalog(IOptions<LocalStorageModuleCatalogOptions> options)
+        public static ILocalModuleCatalog GetCatalog(IOptions<LocalStorageModuleCatalogOptions> options)
         {
             if (_catalog == null)
             {
                 var logger = new LoggerFactory().CreateLogger<LocalStorageModuleCatalog>();
                 var distributedLock = new NoLockDistributedLockProvider(new LoggerFactory().CreateLogger<NoLockDistributedLockProvider>());
-                _catalog = new LocalStorageModuleCatalog(options, distributedLock, logger);
+                _catalog = new LocalCatalog(options, distributedLock, logger);
                 _catalog.Load();
             }
             else _catalog.Reload();

_build.csproj

@@ -31,7 +31,7 @@
         <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
         <PackageReference Include="Octokit" Version="0.50.0" />
         <PackageReference Include="System.Data.SqlClient" Version="4.8.2" />
-        <PackageReference Include="Virtocommerce.Platform.Core" Version="3.66.0" />
+        <PackageReference Include="Virtocommerce.Platform.Core" Version="3.70.0" />
         <PackageReference Include="Nuke.Common" Version="5.0.2" />
     </ItemGroup>
 
@@ -42,9 +42,9 @@
         <PackageReference Include="Swashbuckle.AspNetCore.Cli" Version="5.6.3">
             <ExcludeAssets>all</ExcludeAssets>
         </PackageReference>
-        <PackageReference Include="VirtoCommerce.Platform.Data" Version="3.66.0" />
-        <PackageReference Include="VirtoCommerce.Platform.Modules" Version="3.66.0" />
-		<PackageReference Include="VirtoCommerce.Platform.DistributedLock" Version="3.66.0" />
+        <PackageReference Include="VirtoCommerce.Platform.Data" Version="3.70.0" />
+        <PackageReference Include="VirtoCommerce.Platform.Modules" Version="3.70.0" />
+		<PackageReference Include="VirtoCommerce.Platform.DistributedLock" Version="3.70.0" />
     </ItemGroup>
 
     <ItemGroup>