diff --git a/Sdk/Sdk.csproj b/Sdk/Sdk.csproj index dbf8ff6..5117400 100644 --- a/Sdk/Sdk.csproj +++ b/Sdk/Sdk.csproj @@ -15,9 +15,10 @@ True SkylineCommunications Skyline Communications - The Skyline.DataMiner.Sdk is a development kit designed to streamline the creation and management of DataMiner Installation Packages (.dmapp). -By integrating this SDK into your build process, you can easily generate installation packages for DataMiner through a simple project build or compile step. Additionally, it provides tools to publish these packages directly to the DataMiner Catalog, ensuring a smooth and efficient development pipeline. - + + The Skyline.DataMiner.Sdk is a development kit designed to streamline the creation and management of DataMiner Installation Packages (.dmapp). + By integrating this SDK into your build process, you can easily generate installation packages for DataMiner through a simple project build or compile step. Additionally, it provides tools to publish these packages directly to the DataMiner Catalog, ensuring a smooth and efficient development pipeline. + https://github.com/SkylineCommunications/Skyline.DataMiner.Sdk git 1701;1702;NU5100 @@ -25,7 +26,7 @@ By integrating this SDK into your build process, you can easily generate install Sdk true $(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage - 0.1.0 + 0.0.149 @@ -41,12 +42,15 @@ By integrating this SDK into your build process, you can easily generate install + + + + - - - - - + + + + diff --git a/Sdk/Sdk/Sdk.targets b/Sdk/Sdk/Sdk.targets index e9cf3e9..8e6cfbe 100644 --- a/Sdk/Sdk/Sdk.targets +++ b/Sdk/Sdk/Sdk.targets @@ -26,7 +26,7 @@ Debug="$(SkylineDataMinerSdkDebug)" /> - + + /// Allows running commands on the shell. + /// + internal interface IShell + { + /// + /// Runs the given command on the shell. + /// + /// The command to run. + /// Any output from running the command. + /// Any error from the command. + /// that controls the cancellation of the command. + /// Optional working directory in which the command should be run. + /// if there were no errors with the command. + bool RunCommand(string command, out string output, out string errors, CancellationToken cancellationToken, string workingDirectory = ""); + } + + /// + /// Helper methods for . + /// + internal static class ShellFactory + { + /// + /// Get your shiny shells here! Tailored specifically to your OS! + /// + /// Shiny shell. + /// Can't create a shell for this OS. + public static IShell GetShell() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return new WindowsShell(); + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + return new UnixShell(); + } + + throw new NotSupportedException($"The current operating system ({System.Runtime.InteropServices.RuntimeInformation.OSDescription}) is not supported."); + } + } +} diff --git a/Sdk/Shell/UnixShell.cs b/Sdk/Shell/UnixShell.cs new file mode 100644 index 0000000..7085803 --- /dev/null +++ b/Sdk/Shell/UnixShell.cs @@ -0,0 +1,54 @@ +namespace Skyline.DataMiner.Sdk.Shell +{ + using System.Diagnostics; + using System.Text; + using System.Threading; + + /// + /// Allows running commands on the unix shell. + /// + internal class UnixShell : IShell + { + /// + public bool RunCommand(string command, out string output, out string errors, CancellationToken cancellationToken, string workingDirectory = "") + { + StringBuilder outputStream = new StringBuilder(); + StringBuilder errorStream = new StringBuilder(); + string escapedArgs = command.Replace("\"", "\\\""); + bool success = true; + using (Process cmd = new Process + { + StartInfo = new ProcessStartInfo + { + WindowStyle = ProcessWindowStyle.Hidden, + FileName = "/bin/bash", + Arguments = $"-c \"{escapedArgs}\"", + CreateNoWindow = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + WorkingDirectory = workingDirectory + } + }) + { + cmd.OutputDataReceived += (_, args) => { outputStream.AppendLine(args.Data); }; + cmd.ErrorDataReceived += (_, args) => { errorStream.AppendLine(args.Data); }; + cmd.Start(); + cmd.BeginOutputReadLine(); + cmd.BeginErrorReadLine(); + cmd.WaitForExit(300000);// 5 min max wait + if (!cmd.HasExited) + { + success = false; + cmd.Kill(); + } + + output = outputStream.ToString(); + errors = errorStream.ToString(); + + success &= cmd.ExitCode == 0; + return success; + } + } + } +} diff --git a/Sdk/Shell/WindowsShell.cs b/Sdk/Shell/WindowsShell.cs new file mode 100644 index 0000000..06060a5 --- /dev/null +++ b/Sdk/Shell/WindowsShell.cs @@ -0,0 +1,60 @@ +namespace Skyline.DataMiner.Sdk.Shell +{ + using System.Diagnostics; + using System.Text; + using System.Threading; + + /// + /// Allows running commands on the windows shell. + /// + internal class WindowsShell : IShell + { + /// + public bool RunCommand(string command, out string output, out string errors, CancellationToken cancellationToken, string workingDirectory = "") + { + StringBuilder outputStream = new StringBuilder(); + StringBuilder errorStream = new StringBuilder(); + + bool success = true; + using (Process cmd = new Process + { + // You got to put the entire thing in quotes so the WindowsShell can remove the quotes + // and think there's no quotes while there are quotes which get removed. + // If you don't put the quotes, you have to put more quotes which is too confusing. + // See https://ss64.com/nt/cmd.html + StartInfo = new ProcessStartInfo + { + WindowStyle = ProcessWindowStyle.Hidden, + Verb = "runas", + FileName = "cmd.exe", + Arguments = $"/S /C \"{command}\"", + CreateNoWindow = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + WorkingDirectory = workingDirectory + } + }) + { + + cmd.OutputDataReceived += (_, args) => { outputStream.AppendLine(args.Data); }; + cmd.ErrorDataReceived += (_, args) => { errorStream.AppendLine(args.Data); }; + cmd.Start(); + cmd.BeginOutputReadLine(); + cmd.BeginErrorReadLine(); + cmd.WaitForExit(300000);// 5 min max wait + if (!cmd.HasExited) + { + success = false; + cmd.Kill(); + } + + output = outputStream.ToString(); + errors = errorStream.ToString(); + + success &= cmd.ExitCode == 0; + return success; + } + } + } +} diff --git a/Sdk/Tasks/DmappCreation.cs b/Sdk/Tasks/DmappCreation.cs index 9ad10c0..e7aa9d4 100644 --- a/Sdk/Tasks/DmappCreation.cs +++ b/Sdk/Tasks/DmappCreation.cs @@ -8,11 +8,15 @@ namespace Skyline.DataMiner.Sdk.Tasks using System.Collections.Generic; using System.Diagnostics; using System.Linq; + using System.Management.Automation; using System.Net.Http; using System.Runtime.InteropServices; using System.Text.RegularExpressions; + using System.Threading; using Microsoft.Build.Framework; + using Microsoft.Build.Tasks; + using Microsoft.Build.Utilities; using Microsoft.Extensions.Configuration; using Nito.AsyncEx.Synchronous; @@ -21,14 +25,20 @@ namespace Skyline.DataMiner.Sdk.Tasks using Skyline.ArtifactDownloader; using Skyline.ArtifactDownloader.Identifiers; using Skyline.ArtifactDownloader.Services; + using Skyline.DataMiner.CICD.Assemblers.Automation; using Skyline.DataMiner.CICD.Common; using Skyline.DataMiner.CICD.DMApp.Common; using Skyline.DataMiner.CICD.FileSystem; + using Skyline.DataMiner.CICD.FileSystem.DirectoryInfoWrapper; + using Skyline.DataMiner.CICD.FileSystem.FileInfoWrapper; using Skyline.DataMiner.CICD.Loggers; using Skyline.DataMiner.CICD.Parsers.Common.VisualStudio.Projects; using Skyline.DataMiner.Sdk.Helpers; + using Skyline.DataMiner.Sdk.Shell; using Skyline.DataMiner.Sdk.SubTasks; + using static Skyline.AppInstaller.AppPackage; + using Task = Microsoft.Build.Utilities.Task; public class DmappCreation : Task, ICancelableTask @@ -36,7 +46,7 @@ public class DmappCreation : Task, ICancelableTask private readonly Dictionary loadedProjects = new Dictionary(); private bool cancel; - + internal ILogCollector Logger; #region Properties set from targets file @@ -108,8 +118,10 @@ public override bool Execute() return true; } - if (dataMinerProjectType == DataMinerProjectType.Package) + if (dataMinerProjectType == DataMinerProjectType.Package || dataMinerProjectType == DataMinerProjectType.TestPackage) { + // TestPackage is a wrapper around a normal Package + // Package basic files where no special conversion is needed PackageBasicFiles(preparedData, appPackageBuilder); @@ -118,6 +130,11 @@ public override bool Execute() // Catalog references PackageCatalogReferences(preparedData, appPackageBuilder); + + if (dataMinerProjectType == DataMinerProjectType.TestPackage && !PackageTests(preparedData, appPackageBuilder)) + { + return false; + } } else { @@ -134,7 +151,19 @@ public override bool Execute() string destinationFilePath = FileSystem.Instance.Path.Combine(outputDirectory, $"{PackageId}.{PackageVersion}.dmapp"); IAppPackage package = appPackageBuilder.Build(); - string about = package.CreatePackage(destinationFilePath); + string about; + if (dataMinerProjectType == DataMinerProjectType.TestPackage) + { + var byteArray = package.CreatePackage(); + var dmtestFilePath = FileSystem.Instance.Path.ChangeExtension(destinationFilePath, ".dmtest"); + FileSystem.Instance.File.WriteAllBytes(dmtestFilePath, byteArray); + about = $"Test package created at: {dmtestFilePath}"; + } + else + { + about = package.CreatePackage(destinationFilePath); + } + Logger.ReportDebug($"About created package:{Environment.NewLine}{about}"); Log.LogMessage(MessageImportance.High, $"Successfully created package '{destinationFilePath}'."); @@ -155,6 +184,310 @@ public override bool Execute() } } + private bool PackageTests(PackageCreationData preparedData, AppPackageBuilder appPackageBuilder) + { + // Special For Tests. + string testPackageContentPath = FileSystem.Instance.Path.Combine(preparedData.Project.ProjectDirectory, "TestPackageContent"); + string pathToCustomTestHarvesting = FileSystem.Instance.Path.Combine(testPackageContentPath, "TestHarvesting"); + + if (!FileSystem.Instance.Directory.Exists(testPackageContentPath)) + { + // No testpackage content directory found. Skip the rest. + Log.LogError("No testpackage content directory found. Skip the rest."); + return false; + } + + if (!HarvestTests(pathToCustomTestHarvesting)) + { + return false; + } + + // Handles all different special technologies that use non-sdk-project automationscripts for tests. + Logger.ReportWarning("Adding Harvested Automation XML Tests to .dmtest"); + AddHarvestedAutomationTests(pathToCustomTestHarvesting, appPackageBuilder); + + // Handles all other testing technologies + Logger.ReportWarning("Adding Harvested Tests to .dmtest"); + AddHarvestedTests(testPackageContentPath, appPackageBuilder); + + Logger.ReportWarning("Adding Harvested Dependencies to .dmtest"); + AddHarvestedDependencies(pathToCustomTestHarvesting, appPackageBuilder); + + // Add TestPackageExecutionSpecialDependencies + Logger.ReportWarning("Adding Hardcoded Dependencies to .dmtest"); + AddTestsDependencies(testPackageContentPath, appPackageBuilder); + + Logger.ReportWarning("Adding Hardcoded Tests to .dmtest"); + AddTests(testPackageContentPath, appPackageBuilder); + + if (!AddTestsPipelineScripts(appPackageBuilder, testPackageContentPath)) + { + return false; + } + + appPackageBuilder.WithDmTestContent("dmtestversion.txt", "1.0.0", DmTestContentType.Text); + return true; + } + + private bool AddTestsPipelineScripts(AppPackageBuilder appPackageBuilder, string testPackageContentPath) + { + string testPackagePipelinePath = + FileSystem.Instance.Path.Combine(testPackageContentPath, "TestPackagePipeline"); + + if (FileSystem.Instance.Directory.Exists(testPackagePipelinePath)) + { + bool foundAtLeastOne = false; + foreach (var pipelineScript in FileSystem.Instance.Directory.EnumerateFiles(testPackagePipelinePath, "*.ps1")) + { + var fileName = FileSystem.Instance.Path.GetFileName(pipelineScript); + if (Regex.IsMatch(fileName, @"^\d+\.")) + { + foundAtLeastOne = true; + appPackageBuilder.WithDmTestContent("TestPackagePipeline\\" + fileName, pipelineScript, DmTestContentType.FilePath); + } + } + + if (!foundAtLeastOne) + { + Logger.ReportError($"Expected at least a single powershell script that defines how to execute tests within {testPackagePipelinePath}."); + return false; + } + } + else + { + Logger.ReportError($"Expected a directory {testPackagePipelinePath}."); + return false; + } + + return true; + } + + private void AddHarvestedTests(string pathToCustomTestHarvesting, AppPackage.AppPackageBuilder appPackageBuilder) + { + string tests = + FileSystem.Instance.Path.Combine(pathToCustomTestHarvesting, "tests.generated"); + + if (FileSystem.Instance.Directory.Exists(tests)) + { + appPackageBuilder.WithDmTestContent("TestHarvesting\\tests.generated", tests, DmTestContentType.DirectoryPath); + } + } + + private static void AddTestsDependencies(string testPackageContentPath, AppPackage.AppPackageBuilder appPackageBuilder) + { + string dependencies = + FileSystem.Instance.Path.Combine(testPackageContentPath, "Dependencies"); + + if (FileSystem.Instance.Directory.Exists(dependencies)) + { + // Special, these are to be installed by the Bridge or Code that executes tests somewhere the tests can access. + appPackageBuilder.WithDmTestContent("Dependencies", dependencies, DmTestContentType.DirectoryPath); + } + } + + + private static void AddTests(string testPackageContentPath, AppPackage.AppPackageBuilder appPackageBuilder) + { + string tests = + FileSystem.Instance.Path.Combine(testPackageContentPath, "Tests"); + + if (FileSystem.Instance.Directory.Exists(tests)) + { + // Special, these are to be installed by the Bridge or Code that executes tests somewhere the tests can access. + appPackageBuilder.WithDmTestContent("Tests", tests, DmTestContentType.DirectoryPath); + } + } + + private static void AddHarvestedAutomationTests(string pathToCustomTestHarvesting, AppPackage.AppPackageBuilder appPackageBuilder) + { + string testsAsAutomationScripts = + FileSystem.Instance.Path.Combine(pathToCustomTestHarvesting, "xmlautomationtests.generated"); + + if (FileSystem.Instance.Directory.Exists(testsAsAutomationScripts)) + { + foreach (var asFolder in FileSystem.Instance.Directory.EnumerateDirectories(testsAsAutomationScripts)) + { + // Very simple folders. Should contain the XML, the .dll's needed. Name of the folder is name of the AutomationScript + // How that gets added there and created has to be defined through the CollectTests.ps1 based on w/e testing tech used. + // Find the first .xml + string scriptFilePath = FileSystem.Instance.Path.Combine(asFolder, "script.xml"); + string dllsPath = FileSystem.Instance.Path.Combine(asFolder, "dlls"); + DirectoryInfo directoryInfo = new DirectoryInfo(asFolder); + string nameOfTest = directoryInfo.Name; + + var scriptBuilder = new AppPackageAutomationScript.AppPackageAutomationScriptBuilder(nameOfTest, "1.0.0.1", scriptFilePath); + + if (FileSystem.Instance.Directory.Exists(dllsPath)) + { + foreach (var assembly in FileSystem.Instance.Directory.EnumerateFiles(dllsPath)) + { + scriptBuilder.WithAssembly(assembly, "C:\\Skyline DataMiner\\ProtocolScripts\\DllImport"); + } + } + + IAppPackageAutomationScript automationScript = scriptBuilder.Build(); + if (automationScript != null) + { + appPackageBuilder.WithAutomationScript(automationScript); + } + } + } + } + + private static void AddHarvestedDependencies(string pathToCustomTestHarvesting, AppPackage.AppPackageBuilder appPackageBuilder) + { + string dependencies = + FileSystem.Instance.Path.Combine(pathToCustomTestHarvesting, "dependencies.generated"); + + if (FileSystem.Instance.Directory.Exists(dependencies)) + { + // Special, these are to be installed by the Bridge or Code that executes tests somewhere the tests can access. + appPackageBuilder.WithDmTestContent("TestHarvesting\\dependencies.generated", dependencies, DmTestContentType.DirectoryPath); + } + } + + static string FindExecutable(string exeName) + { + // Check PATH + var paths = Environment.GetEnvironmentVariable("PATH")?.Split(FileSystem.Instance.Path.PathSeparator) ?? new string[0]; + return paths.Select(path => FileSystem.Instance.Path.Combine(path, exeName)).FirstOrDefault(FileSystem.Instance.File.Exists); + } + + private bool HarvestTests(string pathToCustomTestHarvesting) + { + if (pathToCustomTestHarvesting == null) + { + throw new InvalidOperationException("pathToCustomTestHarvesting is null"); + } + + Logger.ReportDebug("Starting Test Harvest..."); + + Logger.ReportStatus($"Starting Test Harvest..."); + string collectTestsFile = + FileSystem.Instance.Path.Combine(pathToCustomTestHarvesting, "TestDiscovery.ps1"); + + if (FileSystem.Instance.File.Exists(collectTestsFile)) + { + Logger.ReportStatus($"Collecting Tests with {collectTestsFile}..."); + try + { + using (PowerShell ps = PowerShell.Create()) + { + if (ps == null) + { + // FallBack. In VS editor, PowerShell works and gives a lot of details and logging. + // Falling back to use Shell if the PowerShell.Create() returns null. (slower, but works) + bool isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + + // Determine which PowerShell to use + string shellExe; + if (isWindows) + { + string powerShellPath = FindExecutable("pwsh.exe"); + shellExe = powerShellPath ?? "powershell.exe"; + } + else + { + string powerShellPath = FindExecutable("pwsh"); + shellExe = powerShellPath ?? "pwsh"; + } + + var shell = ShellFactory.GetShell(); + string command = $"\"{shellExe}\" \"{collectTestsFile}\""; + Logger.ReportWarning($"Could not execute with Powershell API, falling back to Shell with less runtime details..."); + + if (!shell.RunCommand(command, out string output, out string errors, CancellationToken.None)) + { + Logger.ReportError($"Failed to run command '{command}' with output: {output} and errors: {errors}"); + return false; + } + else + { + Logger.ReportStatus($"Successfully executed {collectTestsFile} with output: {output}"); + return true; + } + } + + ps.AddCommand(collectTestsFile); + + // Because the powershell HadErrors always returns true. + bool hadError = false; + Logger.ReportStatus($"Invoking {collectTestsFile}..."); + + // Error stream + ps.Streams.Error.DataAdded += (s, e) => + { + var col = (PSDataCollection)s; + var rec = col[e.Index]; + Logger.ReportError($"{collectTestsFile} [Error] {rec}"); + hadError = true; + }; + + // Warning stream + ps.Streams.Warning.DataAdded += (s, e) => + { + var col = (PSDataCollection)s; + var rec = col[e.Index]; + Logger.ReportWarning($"{collectTestsFile} [Warning] {rec}"); + }; + + // Verbose stream + ps.Streams.Verbose.DataAdded += (s, e) => + { + var col = (PSDataCollection)s; + var rec = col[e.Index]; + Logger.ReportDebug($"{collectTestsFile} [Verbose] {rec}"); + }; + + // Debug stream + ps.Streams.Debug.DataAdded += (s, e) => + { + var col = (PSDataCollection)s; + var rec = col[e.Index]; + Logger.ReportDebug($"{collectTestsFile} [Debug] {rec}"); + }; + + // Information stream + ps.Streams.Information.DataAdded += (s, e) => + { + var col = (PSDataCollection)s; + var rec = col[e.Index]; + Logger.ReportStatus($"{collectTestsFile} [Info] {rec}"); + }; + + // Progress stream + ps.Streams.Progress.DataAdded += (s, e) => + { + var col = (PSDataCollection)s; + var rec = col[e.Index]; + Logger.ReportStatus($"{collectTestsFile} [Progress] {rec.Activity} ({rec.PercentComplete}%)"); + }; + + + ps.Invoke(); + Logger.ReportStatus($"Finished {collectTestsFile}..."); + + if (hadError) + { + Logger.ReportError($"{collectTestsFile} indicated it had errors."); + return false; + } + } + } + catch (Exception ex) + { + Logger.ReportError($"Exception occurred, check the powershell script!: {ex}"); + return false; + } + } + else + { + Logger.ReportWarning($"No Test Collection powershell found at {collectTestsFile}..."); + } + + return true; + } + private void AddProjectToPackage(PackageCreationData preparedData, AppPackage.AppPackageBuilder appPackageBuilder) { switch (preparedData.Project.DataMinerProjectType) @@ -219,7 +552,7 @@ private void PackageProjectReferences(PackageCreationData preparedData, AppPacka // Ignore projects that are not recognized as a DataMiner project (unit test projects, class library projects, NuGet package projects, etc.) continue; } - + PackageCreationData newPreparedData = PrepareDataForProject(includedProject, preparedData); AddProjectToPackage(newPreparedData, appPackageBuilder); diff --git a/SdkTests/SdkTests.csproj b/SdkTests/SdkTests.csproj index f4ad545..f5baeae 100644 --- a/SdkTests/SdkTests.csproj +++ b/SdkTests/SdkTests.csproj @@ -17,11 +17,11 @@ - - - - - + + + + +