diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index fde8ac4b0..6285204c1 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -6,6 +6,7 @@ on: branches: - dev - future + - rc/* tags: - v[0-9]+.[0-9]+.[0-9]+ # Matches all semantic versioning tags with major, minor, patch pull_request: @@ -22,6 +23,10 @@ env: MORYX_PACKAGE_TARGET_V3_FUTURE: 'https://www.myget.org/F/moryx-oss-ci/api/v3/index.json' MORYX_PACKAGE_TARGET_RELEASE: 'https://api.nuget.org/v3/index.json' MORYX_PACKAGE_TARGET_V3_RELEASE: 'https://api.nuget.org/v3/index.json' + REF_NAME: ${{ github.ref_name}} + BASE_REF: ${{ github.base_ref}} + REF: ${{ github.ref }} + HEAD_REF: ${{ github.head_ref }} jobs: EnvVar: @@ -30,17 +35,18 @@ jobs: - run: echo "" outputs: dotnet_sdk_version: ${{ env.dotnet_sdk_version }} - REPOSITORY_NAME: ${{ env.REPOSITORY_NAME }} + REPOSITORY_NAME: ${{ github.event.repository.name }} MORYX_PACKAGE_TARGET_DEV: ${{ env.MORYX_PACKAGE_TARGET_DEV }} MORYX_PACKAGE_TARGET_V3_DEV: ${{ env.MORYX_PACKAGE_TARGET_V3_DEV }} MORYX_PACKAGE_TARGET_FUTURE: ${{ env.MORYX_PACKAGE_TARGET_FUTURE }} MORYX_PACKAGE_TARGET_V3_FUTURE: ${{ env.MORYX_PACKAGE_TARGET_V3_FUTURE }} MORYX_PACKAGE_TARGET_RELEASE: ${{ env.MORYX_PACKAGE_TARGET_RELEASE }} MORYX_PACKAGE_TARGET_V3_RELEASE: ${{ env.MORYX_PACKAGE_TARGET_V3_RELEASE }} + Build: needs: [EnvVar] - uses: phoenixcontact/tools/.github/workflows/build-tool.yml@main + uses: phoenixcontact/tools/.github/workflows/build-tool.yml@rc-builds with: dotnet_sdk_version: ${{ needs.EnvVar.outputs.dotnet_sdk_version }} REPOSITORY_NAME: ${{ needs.EnvVar.outputs.REPOSITORY_NAME }} @@ -83,7 +89,7 @@ jobs: Publish: needs: [EnvVar, UnitTests] - uses: phoenixcontact/tools/.github/workflows/publish-tool.yml@main + uses: phoenixcontact/tools/.github/workflows/publish-tool.yml@rc-builds with: dotnet_sdk_version: ${{ needs.EnvVar.outputs.dotnet_sdk_version }} REPOSITORY_NAME: ${{ needs.EnvVar.outputs.REPOSITORY_NAME }} diff --git a/VERSION b/VERSION index e7fdef7e2..6d2890793 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -8.4.2 +8.5.0 diff --git a/docs/articles/Core/FileSystem.md b/docs/articles/Core/FileSystem.md new file mode 100644 index 000000000..b81974172 --- /dev/null +++ b/docs/articles/Core/FileSystem.md @@ -0,0 +1,28 @@ +--- +uid: FileSystem +--- +# FileSystem + +## Inspiration + +The MORYX file system was inspired by the GIT tree and obj structure. + +## Architecture + +````mermaid +classDiagram + MoryxFile <|-- Blob + MoryxFile <|-- Tree + Tree --> MoryxFile + OwnerFile --> Tree + class MoryxFileSystem{ + -string Directory + +WriteBlob() + +WriteTree() + +ReadBlob() + +ReadTree() + } + class MoryxFile { + +String Hash + } +```` \ No newline at end of file diff --git a/src/Moryx.Runtime.Kernel/FileSystem/HashPath.cs b/src/Moryx.Runtime.Kernel/FileSystem/HashPath.cs new file mode 100644 index 000000000..2a5351622 --- /dev/null +++ b/src/Moryx.Runtime.Kernel/FileSystem/HashPath.cs @@ -0,0 +1,62 @@ +using Microsoft.Extensions.Logging; +using Moryx.Logging; +using System; +using System.Collections.Generic; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; + +namespace Moryx.Runtime.Kernel.FileSystem +{ + internal class HashPath + { + public string Hash { get; private set; } + + public string DirectoryName { get; private set; } + + public string FileName { get; private set; } + + private HashPath() + { + } + + public static HashPath FromStream(Stream stream) => + BuildPath(HashFromStream(stream)); + + public static HashPath FromHash(string hash) => + BuildPath(hash); + + public string FilePath(string storagePath) => + Path.Combine(storagePath, DirectoryName, FileName); + + public string DirectoryPath(string storagePath) => + Path.Combine(storagePath, DirectoryName); + + private static HashPath BuildPath(string hash) + { + return new HashPath + { + Hash = hash, + DirectoryName = hash.Substring(0, 2), + FileName = hash.Substring(2) + }; + } + + private static string HashFromStream(Stream stream) + { + string name; + using (var hashing = SHA256.Create()) + { + stream.Position = 0; + + var hash = hashing.ComputeHash(stream); + name = BitConverter.ToString(hash).Replace("-", string.Empty); + + stream.Position = 0; + } + + return name; + } + } +} diff --git a/src/Moryx.Runtime.Kernel/FileSystem/LocalFileSystem.cs b/src/Moryx.Runtime.Kernel/FileSystem/LocalFileSystem.cs new file mode 100644 index 000000000..69ae463f5 --- /dev/null +++ b/src/Moryx.Runtime.Kernel/FileSystem/LocalFileSystem.cs @@ -0,0 +1,277 @@ +using Castle.MicroKernel.Registration; +using Microsoft.Extensions.Logging; +using Moryx.FileSystem; +using Moryx.Logging; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices.ComTypes; +using System.Text; +using System.Threading.Tasks; + +namespace Moryx.Runtime.Kernel.FileSystem; + +internal class LocalFileSystem : IMoryxFileSystem +{ + private string _fsDirectory; + private string _ownerFilesDirectory; + private readonly ILogger _logger; + + private readonly List _ownerTrees = new List(); + + public LocalFileSystem(ILoggerFactory loggerFactory) + { + _logger = loggerFactory.CreateLogger(nameof(LocalFileSystem)); + } + + public void SetBasePath(string basePath = "fs") + { + _fsDirectory = Path.Combine(Directory.GetCurrentDirectory(), basePath); + _ownerFilesDirectory = Path.Combine(_fsDirectory, "owners"); + } + + public void LoadTrees() + { + // Load all trees from the owner directory + var ownerFiles = Directory.EnumerateFiles(_ownerFilesDirectory); + foreach (var ownerFile in ownerFiles) + { + var treeHash = File.ReadAllText(ownerFile); + var ownerTree = new MoryxFileTree + { + FileName = ownerFile, + Hash = treeHash, + }; + ReadExtensibleTree(ownerTree); + } + } + + public MoryxFile GetFile(string hash) + { + return _ownerTrees.FindFile(hash); + } + + public async Task WriteAsync(MoryxFile file, Stream content) + { + if (file is MoryxFileTree fileTree) + return await WriteTreeAsync(fileTree); + else if (content != null) + return await WriteBlobAsync(file, content); + + throw new ArgumentException("For all files except trees the content stream must be given"); + } + + private async Task WriteTreeAsync(MoryxFileTree tree) + { + // Convert metadata to lines + var stream = new MemoryStream(); + using (var sw = new StreamWriter(stream)) + { + foreach (var line in tree.Files.Select(FileToLine)) + sw.WriteLine(line); + await sw.FlushAsync(); + } + + tree.Hash = await StreamToDiskAsync(stream); + + // Update parent or owner file + if (tree.ParentTree == null) + { + var ownerFile = Path.Combine(_ownerFilesDirectory, tree.FileName); + File.WriteAllText(ownerFile, tree.Hash); + } + else + { + await WriteTreeAsync(tree.ParentTree); + } + + return tree.Hash; + } + + private async Task WriteBlobAsync(MoryxFile file, Stream fileStream) + { + // Create file first + var hash = await StreamToDiskAsync(fileStream); + file.Hash = hash; + + // Now write the tree recursively + await WriteTreeAsync(file.ParentTree); + + return hash; + } + + private async Task StreamToDiskAsync(Stream stream) + { + var hashPath = HashPath.FromStream(stream); + + // Create directory if necessary + var targetPath = hashPath.DirectoryPath(_fsDirectory); + try + { + if (!Directory.Exists(targetPath)) + Directory.CreateDirectory(targetPath); + } + catch (Exception e) + { + throw LoggedException(e, _logger, _fsDirectory); + } + + var fileName = hashPath.FilePath(_fsDirectory); + if (File.Exists(fileName)) + return hashPath.Hash; + + // Write file + try + { + using var fileStream = new FileStream(fileName, FileMode.Create); + await stream.CopyToAsync(fileStream); + await fileStream.FlushAsync(); + stream.Position = 0; + } + catch (Exception e) + { + throw LoggedException(e, _logger, fileName); + } + + return hashPath.Hash; + } + + public Stream OpenStream(string hash) + { + var path = HashPath.FromHash(hash).FilePath(_fsDirectory); + return File.Exists(path) ? new FileStream(path, FileMode.Open, FileAccess.Read) : null; + } + + public MoryxFileTree ReadTreeByOwner(string ownerKey) + { + var existingTree = _ownerTrees.FirstOrDefault(ot => ot.FileName == ownerKey); + if (existingTree == null) + { + existingTree = new MoryxFileTree { FileName = ownerKey }; + _ownerTrees.Add(existingTree); + } + return existingTree; + } + + private MoryxFileTree ReadExtensibleTree(MoryxFileTree tree) + { + // Read tree from hash + var path = HashPath.FromHash(tree.Hash).FilePath(_fsDirectory); + if (!File.Exists(path)) + throw new FileNotFoundException(path); + + var lines = File.ReadAllLines(path); + foreach (var line in lines) + { + var file = FileFromLine(line); + tree.AddFile(file); + + if (file is MoryxFileTree subTree) + ReadExtensibleTree(subTree); + } + + return tree; + } + + public bool RemoveFile(MoryxFile file) + { + // Delete file if found + var hashPath = HashPath.FromHash(file.Hash); + var filePath = hashPath.FilePath(_fsDirectory); + if (!File.Exists(filePath)) + return false; + + RemoveFile(filePath, _logger); + + // Check if subdirectory is empty and remove + var directory = hashPath.DirectoryPath(_fsDirectory); + CleanUpDirectory(directory, _logger); + + // Remove file from tree and rewrite + var parentTree = file.ParentTree; + parentTree.RemoveFile(file); + WriteTreeAsync(parentTree).Wait(); + + return true; + } + + private bool IsOwner(string hash, string ownerFile) + { + var ownerFilePath = Path.Combine(_ownerFilesDirectory, ownerFile); + using (var reader = new StreamReader(ownerFilePath)) + { + string line; + while ((line = reader.ReadLine()) != null) + { + if (line.Contains(hash)) + return true; + } + } + return false; + } + + private void RemoveFile(string filePath, ILogger logger) + { + try + { + File.Delete(filePath); + } + catch (Exception e) + { + throw LoggedException(e, logger, filePath); + } + } + + private void CleanUpDirectory(string directoryPath, ILogger logger) + + { + try + { + if (Directory.GetFiles(directoryPath).Length == 0) + Directory.Delete(directoryPath); + } + catch (Exception e) + { + throw LoggedException(e, logger, directoryPath); + } + } + + private Exception LoggedException(Exception e, ILogger logger, string cause) + { + switch (e) + { + case UnauthorizedAccessException unauthorizedAccessException: + logger.LogError("Error: {0}. You do not have the required permission to manipulate the file {1}.", e.Message, cause); // ToDo + return unauthorizedAccessException; + case ArgumentException argumentException: + logger.LogError("Error: {0}. The path {1} contains invalid characters such as \", <, >, or |.", e.Message, cause); + return argumentException; + case IOException iOException: + logger.LogError("Error: {0}. An I/O error occurred while opening the file {1}.", e.Message, cause); + return iOException; + default: + logger.LogError("Unspecified error on file system access: {0}", e.Message); + return e; + } + } + + private static string FileToLine(MoryxFile file) + { + return $"{file.Mode} {file.FileType.ToString().ToLower()} {file.MimeType} {file.Hash} {file.FileName}"; + } + + private static MoryxFile FileFromLine(string line) + { + var parts = line.Split(' '); + + var file = parts[1] == FileType.Blob.ToString().ToLower() + ? new MoryxFile() : new MoryxFileTree(); + file.Mode = int.Parse(parts[0]); + file.MimeType = parts[2]; + file.Hash = parts[3]; + file.FileName = string.Join(" ", parts.Skip(4)); + + return file; + } +} diff --git a/src/Moryx.Runtime.Kernel/KernelServiceCollectionExtensions.cs b/src/Moryx.Runtime.Kernel/KernelServiceCollectionExtensions.cs index 83bc5b833..d5e4861b3 100644 --- a/src/Moryx.Runtime.Kernel/KernelServiceCollectionExtensions.cs +++ b/src/Moryx.Runtime.Kernel/KernelServiceCollectionExtensions.cs @@ -4,6 +4,8 @@ using Microsoft.Extensions.DependencyInjection; using Moryx.Configuration; using Moryx.Container; +using Moryx.FileSystem; +using Moryx.Runtime.Kernel.FileSystem; using Moryx.Runtime.Modules; using Moryx.Threading; using System; @@ -30,6 +32,10 @@ public static void AddMoryxKernel(this IServiceCollection serviceCollection) serviceCollection.AddSingleton(); serviceCollection.AddSingleton(x => x.GetRequiredService()); + // Register local file system + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(x => x.GetRequiredService()); + // Register parallel operations serviceCollection.AddTransient(); @@ -87,6 +93,19 @@ public static IConfigManager UseMoryxConfigurations(this IServiceProvider servic return configManager; } + /// + /// Use moryx file system and configure base directory + /// + /// + public static IMoryxFileSystem UseLocalFileSystem(this IServiceProvider serviceProvider, string path) + { + var fileSystem = serviceProvider.GetRequiredService(); + if (!Directory.Exists(path)) + Directory.CreateDirectory(path); + fileSystem.SetBasePath(path); + return fileSystem; + } + private static IModuleManager _moduleManager; /// /// Boot system and start all modules diff --git a/src/Moryx/FileSystem/FileType.cs b/src/Moryx/FileSystem/FileType.cs new file mode 100644 index 000000000..ea9a09528 --- /dev/null +++ b/src/Moryx/FileSystem/FileType.cs @@ -0,0 +1,25 @@ +// Copyright (c) 2025, Phoenix Contact GmbH & Co. KG +// Licensed under the Apache License, Version 2.0 + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Moryx.FileSystem; + +/// +/// Moryx file types in owner tree file +/// +public enum FileType +{ + /// + /// Binary file, unspecified + /// + Blob = 0, + + /// + /// Tree file with references to files + /// + Tree = 1, +} diff --git a/src/Moryx/FileSystem/IMoryxFileSystem.cs b/src/Moryx/FileSystem/IMoryxFileSystem.cs new file mode 100644 index 000000000..2035d14e5 --- /dev/null +++ b/src/Moryx/FileSystem/IMoryxFileSystem.cs @@ -0,0 +1,44 @@ +// Copyright (c) 2025, Phoenix Contact GmbH & Co. KG +// Licensed under the Apache License, Version 2.0 + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading.Tasks; + +namespace Moryx.FileSystem; + +/// +/// Common file system interface across MORYX modules and components. +/// +public interface IMoryxFileSystem +{ + /// + /// Write a stream to the file system and represent by the given file + /// + /// The hash of the written file on the disk + Task WriteAsync(MoryxFile file, Stream content); + + /// + /// Resolve MORYX file by hash + /// + MoryxFile GetFile(string hash); + + /// + /// Read the file content for the MORYX file + /// + Stream OpenStream(string fileHash); + + /// + /// Return all files stored under a given owner + /// + /// Arbitrary string value used to identify the tree later, can be module or assembly name. + MoryxFileTree ReadTreeByOwner(string ownerKey); + + /// + /// Remove a file by hash and provided owner key. + /// Files without owner key can not be removed + /// + bool RemoveFile(MoryxFile file); +} diff --git a/src/Moryx/FileSystem/MoryxFile.cs b/src/Moryx/FileSystem/MoryxFile.cs new file mode 100644 index 000000000..147f4ebeb --- /dev/null +++ b/src/Moryx/FileSystem/MoryxFile.cs @@ -0,0 +1,22 @@ +// Copyright (c) 2025, Phoenix Contact GmbH & Co. KG +// Licensed under the Apache License, Version 2.0 + +using System; +using System.Linq; + +namespace Moryx.FileSystem; + +public class MoryxFile +{ + public int Mode { get; set; } = (int)new MoryxFileMode(); + + public virtual FileType FileType { get; } = FileType.Blob; + + public string MimeType { get; set; } + + public string Hash { get; set; } + + public string FileName { get; set; } + + public MoryxFileTree ParentTree { get; set; } +} diff --git a/src/Moryx/FileSystem/MoryxFileMode.cs b/src/Moryx/FileSystem/MoryxFileMode.cs new file mode 100644 index 000000000..087493aed --- /dev/null +++ b/src/Moryx/FileSystem/MoryxFileMode.cs @@ -0,0 +1,31 @@ +// Copyright (c) 2025, Phoenix Contact GmbH & Co. KG +// Licensed under the Apache License, Version 2.0 + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Moryx.FileSystem; + +public class MoryxFileMode +{ + private int _value = FileBase + 666; + + public static int FileBase = 100000; + + public static int Admin(int access) => access * 100; + + public static int Owner(int access) => access * 10; + + public static int Public(int access) => access; + + public static int Read = 4; + + public static int Write = 6; + + public static int Execute = 5; + + public static explicit operator int(MoryxFileMode mode) => mode._value; + + public static explicit operator MoryxFileMode(int value) => new MoryxFileMode { _value = value }; +} diff --git a/src/Moryx/FileSystem/MoryxFileSystemExtensions.cs b/src/Moryx/FileSystem/MoryxFileSystemExtensions.cs new file mode 100644 index 000000000..489b23895 --- /dev/null +++ b/src/Moryx/FileSystem/MoryxFileSystemExtensions.cs @@ -0,0 +1,55 @@ +// Copyright (c) 2025, Phoenix Contact GmbH & Co. KG +// Licensed under the Apache License, Version 2.0 + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading.Tasks; + +namespace Moryx.FileSystem; + +public static class MoryxFileSystemExtensions +{ + public static MoryxFile FindFile(this MoryxFileTree fileTree, string hash) + { + if(fileTree.Hash == hash) + return fileTree; + + return FindFile(fileTree.Files, hash); + } + + public static MoryxFile FindFile(this IReadOnlyList files, string hash) + { + foreach (var file in files) + { + if (file.Hash == hash) + return file; + + if (file is not MoryxFileTree subTree) + continue; + + var match = FindFile(subTree.Files, hash); + if (match != null) + return match; + } + + return null; + } + + public static Task WriteBlobAsync(this IMoryxFileSystem fileSystem, Stream stream) + { + return fileSystem.WriteAsync(new MoryxFile + { + FileName = Guid.NewGuid().ToString(), + MimeType = "application/octet-stream" + }, stream); + } + + public static Task WriteTreeAsync(this IMoryxFileSystem fileSystem, MoryxFileTree tree) + { + return fileSystem.WriteAsync(tree, null); + } + + public static Stream OpenStream(this IMoryxFileSystem fileSystem, MoryxFile file) => fileSystem.OpenStream(file.Hash); +} diff --git a/src/Moryx/FileSystem/MoryxFileTree.cs b/src/Moryx/FileSystem/MoryxFileTree.cs new file mode 100644 index 000000000..f17a54516 --- /dev/null +++ b/src/Moryx/FileSystem/MoryxFileTree.cs @@ -0,0 +1,34 @@ +// Copyright (c) 2025, Phoenix Contact GmbH & Co. KG +// Licensed under the Apache License, Version 2.0 + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Moryx.FileSystem; + +/// +/// File that contains a list of files and is the foundation for the recursive tree +/// +public class MoryxFileTree : MoryxFile +{ + private readonly List _files = new List(); + + public override FileType FileType => FileType.Tree; + + public IReadOnlyList Files => _files; + + public void AddFile(MoryxFile file) + { + file.ParentTree = this; + _files.Add(file); + } + + public bool RemoveFile(MoryxFile file) + { + var found = _files.Remove(file); + if (found) + file.ParentTree = null; + return found; + } +} diff --git a/src/Moryx/Moryx.csproj b/src/Moryx/Moryx.csproj index 9a2ea99dc..5585e660f 100644 --- a/src/Moryx/Moryx.csproj +++ b/src/Moryx/Moryx.csproj @@ -15,7 +15,7 @@ - + diff --git a/src/StartProject.Asp/Program.cs b/src/StartProject.Asp/Program.cs index bc308a172..2ced090c6 100644 --- a/src/StartProject.Asp/Program.cs +++ b/src/StartProject.Asp/Program.cs @@ -27,6 +27,7 @@ public static void Main(string[] args) webBuilder.UseStartup(); }).Build(); + host.Services.UseLocalFileSystem("fs"); host.Services.UseMoryxConfigurations("Config"); var moduleManager = host.Services.GetRequiredService();