From 19798ea07b6c584e7d6b40c8b29bd7913d1c8aac Mon Sep 17 00:00:00 2001 From: Arin Ghazarian Date: Thu, 5 Dec 2024 19:35:57 -0800 Subject: [PATCH 01/14] Add progress report for uploading to Azure blob storage --- src/Octoshift/Extensions/NumericExtensions.cs | 19 ++++++++++ src/Octoshift/Services/AzureApi.cs | 35 ++++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 src/Octoshift/Extensions/NumericExtensions.cs diff --git a/src/Octoshift/Extensions/NumericExtensions.cs b/src/Octoshift/Extensions/NumericExtensions.cs new file mode 100644 index 000000000..cf34ee204 --- /dev/null +++ b/src/Octoshift/Extensions/NumericExtensions.cs @@ -0,0 +1,19 @@ +namespace OctoshiftCLI.Extensions; + +public static class NumericExtensions +{ + public static string ToLogFriendlySize(this long size) + { + const int kilobyte = 1024; + const int megabyte = 1024 * kilobyte; + const int gigabyte = 1024 * megabyte; + + return size switch + { + < kilobyte => $"{size:n0} bytes", + < megabyte => $"{size / (double)kilobyte:n0} KB", + < gigabyte => $"{size / (double)megabyte:n0} MB", + _ => $"{size / (double)gigabyte:n2} GB" + }; + } +} diff --git a/src/Octoshift/Services/AzureApi.cs b/src/Octoshift/Services/AzureApi.cs index a1b91b692..31873d58d 100644 --- a/src/Octoshift/Services/AzureApi.cs +++ b/src/Octoshift/Services/AzureApi.cs @@ -6,6 +6,7 @@ using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; using Azure.Storage.Sas; +using OctoshiftCLI.Extensions; namespace OctoshiftCLI.Services; @@ -14,9 +15,12 @@ public class AzureApi private readonly HttpClient _client; private readonly BlobServiceClient _blobServiceClient; private readonly OctoLogger _log; + private readonly object _mutex = new(); private const string CONTAINER_PREFIX = "migration-archives"; private const int AUTHORIZATION_TIMEOUT_IN_HOURS = 48; private const int DEFAULT_BLOCK_SIZE = 4 * 1024 * 1024; + private const int UPLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS = 10; + private DateTime _nextProgressReport; public AzureApi(HttpClient client, BlobServiceClient blobServiceClient, OctoLogger log) { @@ -48,9 +52,17 @@ public virtual async Task UploadToBlob(string fileName, byte[] content) public virtual async Task UploadToBlob(string fileName, Stream content) { + ArgumentNullException.ThrowIfNull(fileName, nameof(fileName)); + ArgumentNullException.ThrowIfNull(content); + var containerClient = await CreateBlobContainerAsync(); var blobClient = containerClient.GetBlobClient(fileName); + _nextProgressReport = DateTime.Now; + var progress = new Progress(); + var archiveSize = content.Length; + progress.ProgressChanged += (_, uploadedBytes) => LogProgress(uploadedBytes, archiveSize); + var options = new BlobUploadOptions { TransferOptions = new Azure.Storage.StorageTransferOptions() @@ -58,8 +70,9 @@ public virtual async Task UploadToBlob(string fileName, Stream content) InitialTransferSize = DEFAULT_BLOCK_SIZE, MaximumTransferSize = DEFAULT_BLOCK_SIZE }, + ProgressHandler = progress }; - + await blobClient.UploadAsync(content, options); return GetServiceSasUriForBlob(blobClient); } @@ -89,4 +102,24 @@ private Uri GetServiceSasUriForBlob(BlobClient blobClient) return blobClient.GenerateSasUri(sasBuilder); } + + private void LogProgress(long uploadedBytes, long totalBytes) + { + lock (_mutex) + { + if (DateTime.Now < _nextProgressReport) + { + return; + } + + _nextProgressReport = _nextProgressReport.AddSeconds(UPLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS); + } + + var percentage = (int)(uploadedBytes * 100L / totalBytes); + var progressMessage = uploadedBytes > 0 + ? $", {uploadedBytes.ToLogFriendlySize()} out of {totalBytes.ToLogFriendlySize()} ({percentage}%) completed" + : ""; + + _log.LogInformation($"Archive upload in progress{progressMessage}..."); + } } From a03cd72c79456d47e0702f6b178f21794374cfa3 Mon Sep 17 00:00:00 2001 From: Arin Ghazarian Date: Fri, 6 Dec 2024 18:12:40 -0800 Subject: [PATCH 02/14] Initialize nextProgressReport on declaration --- src/Octoshift/Services/AzureApi.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Octoshift/Services/AzureApi.cs b/src/Octoshift/Services/AzureApi.cs index 31873d58d..ab4a2bd1f 100644 --- a/src/Octoshift/Services/AzureApi.cs +++ b/src/Octoshift/Services/AzureApi.cs @@ -20,7 +20,7 @@ public class AzureApi private const int AUTHORIZATION_TIMEOUT_IN_HOURS = 48; private const int DEFAULT_BLOCK_SIZE = 4 * 1024 * 1024; private const int UPLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS = 10; - private DateTime _nextProgressReport; + private DateTime _nextProgressReport = DateTime.Now; public AzureApi(HttpClient client, BlobServiceClient blobServiceClient, OctoLogger log) { @@ -58,7 +58,6 @@ public virtual async Task UploadToBlob(string fileName, Stream content) var containerClient = await CreateBlobContainerAsync(); var blobClient = containerClient.GetBlobClient(fileName); - _nextProgressReport = DateTime.Now; var progress = new Progress(); var archiveSize = content.Length; progress.ProgressChanged += (_, uploadedBytes) => LogProgress(uploadedBytes, archiveSize); From 09cbbd6c9fead9923afd07d60dcdfbc33b409189 Mon Sep 17 00:00:00 2001 From: Arin Ghazarian Date: Fri, 6 Dec 2024 21:23:38 -0800 Subject: [PATCH 03/14] Add progress report for uploading to AWS S3 --- src/Octoshift/Services/AwsApi.cs | 60 ++++++++++++++++--- .../Octoshift/Services/AwsApiTests.cs | 35 +++++++---- .../bbs2gh/Factories/AwsApiFactoryTests.cs | 3 +- .../gei/Factories/AwsApiFactoryTests.cs | 3 +- src/bbs2gh/Factories/AwsApiFactory.cs | 6 +- src/gei/Factories/AwsApiFactory.cs | 6 +- 6 files changed, 87 insertions(+), 26 deletions(-) diff --git a/src/Octoshift/Services/AwsApi.cs b/src/Octoshift/Services/AwsApi.cs index 2df028dda..304ebbfdf 100644 --- a/src/Octoshift/Services/AwsApi.cs +++ b/src/Octoshift/Services/AwsApi.cs @@ -13,14 +13,22 @@ namespace OctoshiftCLI.Services; public class AwsApi : IDisposable { private const int AUTHORIZATION_TIMEOUT_IN_HOURS = 48; + private const int UPLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS = 10; private readonly ITransferUtility _transferUtility; + private readonly object _mutex = new(); + private readonly OctoLogger _log; + private DateTime _nextProgressReport = DateTime.Now; - public AwsApi(ITransferUtility transferUtility) => _transferUtility = transferUtility; + public AwsApi(ITransferUtility transferUtility, OctoLogger log) + { + _transferUtility = transferUtility; + _log = log; + } #pragma warning disable CA2000 - public AwsApi(string awsAccessKeyId, string awsSecretAccessKey, string awsRegion = null, string awsSessionToken = null) - : this(new TransferUtility(BuildAmazonS3Client(awsAccessKeyId, awsSecretAccessKey, awsRegion, awsSessionToken))) + public AwsApi(OctoLogger log, string awsAccessKeyId, string awsSecretAccessKey, string awsRegion = null, string awsSessionToken = null) + : this(new TransferUtility(BuildAmazonS3Client(awsAccessKeyId, awsSecretAccessKey, awsRegion, awsSessionToken)), log) #pragma warning restore CA2000 { } @@ -51,20 +59,37 @@ public virtual async Task UploadToBucket(string bucketName, string fileN { try { - await _transferUtility.UploadAsync(fileName, bucketName, keyName); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = bucketName, + Key = keyName, + FilePath = fileName + }; + return await UploadToBucket(uploadRequest); } catch (Exception ex) when (ex is TaskCanceledException or TimeoutException) { throw new OctoshiftCliException($"Upload of archive \"{fileName}\" to AWS timed out", ex); } - - return GetPreSignedUrlForFile(bucketName, keyName); } public virtual async Task UploadToBucket(string bucketName, Stream content, string keyName) { - await _transferUtility.UploadAsync(content, bucketName, keyName); - return GetPreSignedUrlForFile(bucketName, keyName); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = bucketName, + Key = keyName, + InputStream = content + }; + return await UploadToBucket(uploadRequest); + } + + private async Task UploadToBucket(TransferUtilityUploadRequest uploadRequest) + { + uploadRequest.UploadProgressEvent += (_, args) => LogProgress(args.PercentDone, args.TransferredBytes, args.TotalBytes); + await _transferUtility.UploadAsync(uploadRequest); + + return GetPreSignedUrlForFile(uploadRequest.BucketName, uploadRequest.Key); } private string GetPreSignedUrlForFile(string bucketName, string keyName) @@ -81,6 +106,25 @@ private string GetPreSignedUrlForFile(string bucketName, string keyName) return _transferUtility.S3Client.GetPreSignedURL(urlRequest); } + private void LogProgress(int percentDone, long uploadedBytes, long totalBytes) + { + lock (_mutex) + { + if (DateTime.Now < _nextProgressReport) + { + return; + } + + _nextProgressReport = _nextProgressReport.AddSeconds(UPLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS); + } + + var progressMessage = uploadedBytes > 0 + ? $", {uploadedBytes.ToLogFriendlySize()} out of {totalBytes.ToLogFriendlySize()} ({percentDone}%) completed" + : ""; + + _log.LogInformation($"Archive upload in progress{progressMessage}..."); + } + protected virtual void Dispose(bool disposing) { if (disposing) diff --git a/src/OctoshiftCLI.Tests/Octoshift/Services/AwsApiTests.cs b/src/OctoshiftCLI.Tests/Octoshift/Services/AwsApiTests.cs index 7a570da7f..4f849f6da 100644 --- a/src/OctoshiftCLI.Tests/Octoshift/Services/AwsApiTests.cs +++ b/src/OctoshiftCLI.Tests/Octoshift/Services/AwsApiTests.cs @@ -1,6 +1,5 @@ using System; using System.IO; -using System.Text; using System.Threading; using System.Threading.Tasks; using Amazon.S3; @@ -8,6 +7,7 @@ using Amazon.S3.Transfer; using FluentAssertions; using Moq; +using OctoshiftCLI.Extensions; using OctoshiftCLI.Services; using Xunit; @@ -16,6 +16,8 @@ namespace OctoshiftCLI.Tests.Octoshift.Services; public class AwsApiTests { + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + [Fact] public async Task UploadFileToBucket_Should_Succeed() { @@ -30,13 +32,15 @@ public async Task UploadFileToBucket_Should_Succeed() s3Client.Setup(m => m.GetPreSignedURL(It.IsAny())).Returns(url); transferUtility.Setup(m => m.S3Client).Returns(s3Client.Object); - using var awsApi = new AwsApi(transferUtility.Object); + using var awsApi = new AwsApi(transferUtility.Object, _mockOctoLogger.Object); var result = await awsApi.UploadToBucket(bucketName, fileName, keyName); // Assert result.Should().Be(url); - transferUtility.Verify(m => m.UploadAsync(fileName, bucketName, keyName, It.IsAny())); + transferUtility.Verify(m => m.UploadAsync( + It.Is(req => req.BucketName == bucketName && req.Key == keyName && req.FilePath == fileName), + It.IsAny())); } [Fact] @@ -44,9 +48,9 @@ public async Task UploadToBucket_Uploads_FileStream() { // Arrange var bucketName = "bucket"; - var bytes = Encoding.ASCII.GetBytes("here are some bytes"); + var expectedContent = "here are some bytes"; using var stream = new MemoryStream(); - stream.Write(bytes, 0, bytes.Length); + stream.Write(expectedContent.ToBytes()); var keyName = "key"; var url = "http://example.com/file.zip"; @@ -55,13 +59,16 @@ public async Task UploadToBucket_Uploads_FileStream() s3Client.Setup(m => m.GetPreSignedURL(It.IsAny())).Returns(url); transferUtility.Setup(m => m.S3Client).Returns(s3Client.Object); - using var awsApi = new AwsApi(transferUtility.Object); + using var awsApi = new AwsApi(transferUtility.Object, _mockOctoLogger.Object); var result = await awsApi.UploadToBucket(bucketName, stream, keyName); // Assert result.Should().Be(url); - transferUtility.Verify(m => m.UploadAsync(It.IsAny(), bucketName, keyName, It.IsAny())); + transferUtility.Verify(m => m.UploadAsync( + It.Is(req => + req.BucketName == bucketName && req.Key == keyName && (req.InputStream as MemoryStream).ToArray().GetString() == expectedContent), + It.IsAny())); } [Fact] @@ -69,7 +76,7 @@ public void It_Throws_If_Aws_Region_Is_Invalid() { // Arrange, Act const string awsRegion = "invalid-region"; - var awsApi = () => new AwsApi("awsAccessKeyId", "awsSecretAccessKey", awsRegion); + var awsApi = () => new AwsApi(_mockOctoLogger.Object, "awsAccessKeyId", "awsSecretAccessKey", awsRegion); // Assert awsApi.Should().Throw().WithMessage($"*{awsRegion}*"); @@ -88,9 +95,11 @@ public async Task UploadFileToBucket_Throws_If_TaskCanceledException_From_Timeou transferUtility.Setup(m => m.S3Client).Returns(s3Client.Object); - transferUtility.Setup(m => m.UploadAsync(fileName, bucketName, keyName, It.IsAny())).Throws(new TaskCanceledException()); + transferUtility.Setup(m => m.UploadAsync( + It.Is(req => req.BucketName == bucketName && req.Key == keyName && req.FilePath == fileName), + It.IsAny())).Throws(new TaskCanceledException()); - using var awsApi = new AwsApi(transferUtility.Object); + using var awsApi = new AwsApi(transferUtility.Object, _mockOctoLogger.Object); // Assert s3Client.Verify(m => m.GetPreSignedURL(It.IsAny()), Times.Never); @@ -112,9 +121,11 @@ public async Task UploadFileToBucket_Throws_If_TimeoutException_From_Timeout() transferUtility.Setup(m => m.S3Client).Returns(s3Client.Object); - transferUtility.Setup(m => m.UploadAsync(fileName, bucketName, keyName, It.IsAny())).Throws(new TimeoutException()); + transferUtility.Setup(m => m.UploadAsync( + It.Is(req => req.BucketName == bucketName && req.Key == keyName && req.FilePath == fileName), + It.IsAny())).Throws(new TimeoutException()); - using var awsApi = new AwsApi(transferUtility.Object); + using var awsApi = new AwsApi(transferUtility.Object, _mockOctoLogger.Object); // Assert s3Client.Verify(m => m.GetPreSignedURL(It.IsAny()), Times.Never); diff --git a/src/OctoshiftCLI.Tests/bbs2gh/Factories/AwsApiFactoryTests.cs b/src/OctoshiftCLI.Tests/bbs2gh/Factories/AwsApiFactoryTests.cs index 8a5c4a100..6fc119874 100644 --- a/src/OctoshiftCLI.Tests/bbs2gh/Factories/AwsApiFactoryTests.cs +++ b/src/OctoshiftCLI.Tests/bbs2gh/Factories/AwsApiFactoryTests.cs @@ -9,11 +9,12 @@ namespace OctoshiftCLI.Tests.bbs2gh.Factories; public class AwsApiFactoryTests { private readonly Mock _mockEnvironmentVariableProvider = TestHelpers.CreateMock(); + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); private readonly AwsApiFactory _awsApiFactory; public AwsApiFactoryTests() { - _awsApiFactory = new AwsApiFactory(_mockEnvironmentVariableProvider.Object); + _awsApiFactory = new AwsApiFactory(_mockEnvironmentVariableProvider.Object, _mockOctoLogger.Object); } [Fact] diff --git a/src/OctoshiftCLI.Tests/gei/Factories/AwsApiFactoryTests.cs b/src/OctoshiftCLI.Tests/gei/Factories/AwsApiFactoryTests.cs index 9256e147f..88c5a937a 100644 --- a/src/OctoshiftCLI.Tests/gei/Factories/AwsApiFactoryTests.cs +++ b/src/OctoshiftCLI.Tests/gei/Factories/AwsApiFactoryTests.cs @@ -9,11 +9,12 @@ namespace OctoshiftCLI.Tests.GithubEnterpriseImporter.Factories; public class AwsApiFactoryTests { private readonly Mock _mockEnvironmentVariableProvider = TestHelpers.CreateMock(); + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); private readonly AwsApiFactory _awsApiFactory; public AwsApiFactoryTests() { - _awsApiFactory = new AwsApiFactory(_mockEnvironmentVariableProvider.Object); + _awsApiFactory = new AwsApiFactory(_mockEnvironmentVariableProvider.Object, _mockOctoLogger.Object); } [Fact] diff --git a/src/bbs2gh/Factories/AwsApiFactory.cs b/src/bbs2gh/Factories/AwsApiFactory.cs index c657d83de..77de6f1cb 100644 --- a/src/bbs2gh/Factories/AwsApiFactory.cs +++ b/src/bbs2gh/Factories/AwsApiFactory.cs @@ -5,10 +5,12 @@ namespace OctoshiftCLI.BbsToGithub.Factories; public class AwsApiFactory { private readonly EnvironmentVariableProvider _environmentVariableProvider; + private readonly OctoLogger _octoLogger; - public AwsApiFactory(EnvironmentVariableProvider environmentVariableProvider) + public AwsApiFactory(EnvironmentVariableProvider environmentVariableProvider, OctoLogger octoLogger) { _environmentVariableProvider = environmentVariableProvider; + _octoLogger = octoLogger; } public virtual AwsApi Create(string awsRegion = null, string awsAccessKeyId = null, string awsSecretAccessKey = null, string awsSessionToken = null) @@ -18,6 +20,6 @@ public virtual AwsApi Create(string awsRegion = null, string awsAccessKeyId = nu awsSessionToken ??= _environmentVariableProvider.AwsSessionToken(false); awsRegion ??= _environmentVariableProvider.AwsRegion(); - return new AwsApi(awsAccessKeyId, awsSecretAccessKey, awsRegion, awsSessionToken); + return new AwsApi(_octoLogger, awsAccessKeyId, awsSecretAccessKey, awsRegion, awsSessionToken); } } diff --git a/src/gei/Factories/AwsApiFactory.cs b/src/gei/Factories/AwsApiFactory.cs index cb71759d2..9a2f28580 100644 --- a/src/gei/Factories/AwsApiFactory.cs +++ b/src/gei/Factories/AwsApiFactory.cs @@ -5,10 +5,12 @@ namespace OctoshiftCLI.GithubEnterpriseImporter.Factories; public class AwsApiFactory { private readonly EnvironmentVariableProvider _environmentVariableProvider; + private readonly OctoLogger _octoLogger; - public AwsApiFactory(EnvironmentVariableProvider environmentVariableProvider) + public AwsApiFactory(EnvironmentVariableProvider environmentVariableProvider, OctoLogger octoLogger) { _environmentVariableProvider = environmentVariableProvider; + _octoLogger = octoLogger; } public virtual AwsApi Create(string awsRegion = null, string awsAccessKeyId = null, string awsSecretAccessKey = null, string awsSessionToken = null) @@ -18,6 +20,6 @@ public virtual AwsApi Create(string awsRegion = null, string awsAccessKeyId = nu awsSessionToken ??= _environmentVariableProvider.AwsSessionToken(false); awsRegion ??= _environmentVariableProvider.AwsRegion(); - return new AwsApi(awsAccessKeyId, awsSecretAccessKey, awsRegion, awsSessionToken); + return new AwsApi(_octoLogger, awsAccessKeyId, awsSecretAccessKey, awsRegion, awsSessionToken); } } From 83c8aafb7067a25bbf0cdd09ad54b2805afdd6cd Mon Sep 17 00:00:00 2001 From: Arin Ghazarian Date: Fri, 6 Dec 2024 21:23:54 -0800 Subject: [PATCH 04/14] Linting --- src/Octoshift/Services/AzureApi.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Octoshift/Services/AzureApi.cs b/src/Octoshift/Services/AzureApi.cs index ab4a2bd1f..55f63cfd6 100644 --- a/src/Octoshift/Services/AzureApi.cs +++ b/src/Octoshift/Services/AzureApi.cs @@ -71,7 +71,7 @@ public virtual async Task UploadToBlob(string fileName, Stream content) }, ProgressHandler = progress }; - + await blobClient.UploadAsync(content, options); return GetServiceSasUriForBlob(blobClient); } From 56c25bae6e90710478ccdd904ab0315b05959f46 Mon Sep 17 00:00:00 2001 From: Arin Ghazarian Date: Fri, 6 Dec 2024 21:28:25 -0800 Subject: [PATCH 05/14] RELEASENOTES --- RELEASENOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 8b1378917..e7dc4f70d 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1 +1 @@ - +- Add progress report to `gh [gei|bbs2gh] migrate-repo` command when uploading migration archives to Azure Blob or AWS S3 From 5bbd47a7f829489f4db00aa4893d620e6d6ff6db Mon Sep 17 00:00:00 2001 From: Arin Ghazarian Date: Sat, 7 Dec 2024 11:45:20 -0800 Subject: [PATCH 06/14] test --- src/OctoshiftCLI.IntegrationTests/AdoBasicToGithub.cs | 2 +- src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs | 2 +- src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs | 6 +++--- src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs | 6 +++--- src/OctoshiftCLI.IntegrationTests/GithubToGithub.cs | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/OctoshiftCLI.IntegrationTests/AdoBasicToGithub.cs b/src/OctoshiftCLI.IntegrationTests/AdoBasicToGithub.cs index aef851c4d..79f06f064 100644 --- a/src/OctoshiftCLI.IntegrationTests/AdoBasicToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/AdoBasicToGithub.cs @@ -11,7 +11,7 @@ public AdoBasicToGithub(ITestOutputHelper output) : base(output) { } - [Fact] + // [Fact] public async Task Basic() { var adoOrg = $"gei-e2e-testing-basic-{TestHelper.GetOsName()}"; diff --git a/src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs b/src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs index d389fec2e..d167795df 100644 --- a/src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs @@ -11,7 +11,7 @@ public AdoCsvToGithub(ITestOutputHelper output) : base(output) { } - [Fact] + // [Fact] public async Task With_Inventory_Report_Csv() { var adoOrg = $"gei-e2e-testing-csv-{TestHelper.GetOsName()}"; diff --git a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs index 17ebc6965..7077fbb89 100644 --- a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs @@ -70,8 +70,8 @@ public BbsToGithub(ITestOutputHelper output) [Theory] [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.AzureStorage)] [InlineData("http://e2e-bbs-7-21-9-win-2019.eastus.cloudapp.azure.com:7990", false, ArchiveUploadOption.AzureStorage)] - [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.AwsS3)] - [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.GithubStorage)] + // [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.AwsS3)] + // [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.GithubStorage)] public async Task Basic(string bbsServer, bool useSshForArchiveDownload, ArchiveUploadOption uploadOption) { var bbsProjectKey = $"E2E-{TestHelper.GetOsName().ToUpper()}"; @@ -141,7 +141,7 @@ await _targetHelper.RunBbsCliMigration( // TODO: Assert migration logs are downloaded } - [Fact] + // [Fact] public async Task MigrateRepo_MultipartUpload() { var githubTargetOrg = $"octoshift-e2e-bbs-{TestHelper.GetOsName()}"; diff --git a/src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs b/src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs index 948fa83da..af5969e9f 100644 --- a/src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs @@ -63,9 +63,9 @@ public GhesToGithub(ITestOutputHelper output) _targetHelper = new TestHelper(_output, _targetGithubApi, _targetGithubClient, _blobServiceClient); } - [Theory] - [InlineData(false)] - [InlineData(true)] + // [Theory] + // [InlineData(false)] + // [InlineData(true)] public async Task Basic(bool useGithubStorage) { var githubSourceOrg = $"e2e-testing-{TestHelper.GetOsName()}"; diff --git a/src/OctoshiftCLI.IntegrationTests/GithubToGithub.cs b/src/OctoshiftCLI.IntegrationTests/GithubToGithub.cs index ef22f4cc6..1ecd0bf21 100644 --- a/src/OctoshiftCLI.IntegrationTests/GithubToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/GithubToGithub.cs @@ -39,7 +39,7 @@ public GithubToGithub(ITestOutputHelper output) _helper = new TestHelper(_output, _githubApi, _githubClient); } - [Fact] + // [Fact] public async Task Basic() { var githubSourceOrg = $"octoshift-e2e-source-{TestHelper.GetOsName()}"; From a9982ec3be6e4ed9a335540ab1a22b7c224a2754 Mon Sep 17 00:00:00 2001 From: Arin Ghazarian Date: Sat, 7 Dec 2024 13:38:09 -0800 Subject: [PATCH 07/14] Process tweaks --- .../TestHelper.cs | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/OctoshiftCLI.IntegrationTests/TestHelper.cs b/src/OctoshiftCLI.IntegrationTests/TestHelper.cs index 040b2ee72..daec5afc5 100644 --- a/src/OctoshiftCLI.IntegrationTests/TestHelper.cs +++ b/src/OctoshiftCLI.IntegrationTests/TestHelper.cs @@ -553,7 +553,10 @@ private async Task RunShellCommand(string command, string fileName, string worki { WorkingDirectory = workingDirectory ?? Directory.GetCurrentDirectory(), FileName = fileName, - Arguments = command + Arguments = command, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true }; if (environmentVariables != null) @@ -573,7 +576,23 @@ private async Task RunShellCommand(string command, string fileName, string worki _output.WriteLine($"Running command: {startInfo.FileName} {startInfo.Arguments}"); - var p = Process.Start(startInfo); + using var p = new Process(); + p.StartInfo = startInfo; + p.OutputDataReceived += (_, args) => + { + if (args.Data != null) + { + _output.WriteLine(args.Data); + } + }; + p.ErrorDataReceived += (_, args) => + { + if (args.Data != null) + { + _output.WriteLine(args.Data); + } + }; + p.Start(); await p.WaitForExitAsync(); p.ExitCode.Should().Be(0, $"{fileName} should return an exit code of 0."); From ed341aea02ef85a056881b353439d469ad723394 Mon Sep 17 00:00:00 2001 From: Arin Ghazarian Date: Sat, 7 Dec 2024 14:37:59 -0800 Subject: [PATCH 08/14] Add live console output --- .github/workflows/CI.yml | 2 +- .github/workflows/integration-tests.yml | 2 +- src/OctoshiftCLI.IntegrationTests/xunit.runner.json | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 src/OctoshiftCLI.IntegrationTests/xunit.runner.json diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 391ef1625..047fa075a 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -250,7 +250,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} LD_LIBRARY_PATH: '$LD_LIBRARY_PATH:${{ github.workspace }}/src/OctoshiftCLI.IntegrationTests/bin/Debug/net8.0/runtimes/ubuntu.18.04-x64/native' - run: dotnet test src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj --filter "${{ matrix.source-vcs }}ToGithub" --logger:"junit;LogFilePath=integration-tests.xml" /p:VersionPrefix=9.9 + run: dotnet test src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj --filter "${{ matrix.source-vcs }}ToGithub" --logger:"junit;LogFilePath=integration-tests.xml" --logger "console;verbosity=normal" /p:VersionPrefix=9.9 - name: Publish Integration Test Results uses: EnricoMi/publish-unit-test-result-action@v2 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 6c1c15dc8..7081b0fc7 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -175,7 +175,7 @@ jobs: AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} GEI_DEBUG_MODE: 'true' LD_LIBRARY_PATH: '$LD_LIBRARY_PATH:${{ github.workspace }}/src/OctoshiftCLI.IntegrationTests/bin/Debug/net8.0/runtimes/ubuntu.18.04-x64/native' - run: dotnet test src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj --filter "${{ matrix.source-vcs }}ToGithub" --logger:"junit;LogFilePath=integration-tests.xml" /p:VersionPrefix=9.9 + run: dotnet test src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj --filter "${{ matrix.source-vcs }}ToGithub" --logger:"junit;LogFilePath=integration-tests.xml" --logger "console;verbosity=normal" /p:VersionPrefix=9.9 - name: Publish Integration Test Results uses: EnricoMi/publish-unit-test-result-action@v2 diff --git a/src/OctoshiftCLI.IntegrationTests/xunit.runner.json b/src/OctoshiftCLI.IntegrationTests/xunit.runner.json new file mode 100644 index 000000000..09335e65e --- /dev/null +++ b/src/OctoshiftCLI.IntegrationTests/xunit.runner.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", + "showLiveOutput": true +} From 187f8cd81cd6541f9ece564d44a3e164e52629c5 Mon Sep 17 00:00:00 2001 From: Arin Ghazarian Date: Sat, 7 Dec 2024 15:14:35 -0800 Subject: [PATCH 09/14] CliWrap --- .../OctoshiftCLI.IntegrationTests.csproj | 1 + .../TestHelper.cs | 71 +++++-------------- 2 files changed, 19 insertions(+), 53 deletions(-) diff --git a/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj b/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj index 5d442b785..37ee26d42 100644 --- a/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj +++ b/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj @@ -7,6 +7,7 @@ + diff --git a/src/OctoshiftCLI.IntegrationTests/TestHelper.cs b/src/OctoshiftCLI.IntegrationTests/TestHelper.cs index daec5afc5..f6d4b3db8 100644 --- a/src/OctoshiftCLI.IntegrationTests/TestHelper.cs +++ b/src/OctoshiftCLI.IntegrationTests/TestHelper.cs @@ -6,6 +6,7 @@ using System.Runtime.InteropServices; using System.Threading.Tasks; using Azure.Storage.Blobs; +using CliWrap; using FluentAssertions; using Newtonsoft.Json.Linq; using OctoshiftCLI.Services; @@ -526,7 +527,7 @@ public static string GetOsName() : throw new InvalidOperationException("Could not determine OS"); } - public async Task RunCliMigration(string generateScriptCommand, string cliName, IDictionary tokens) + public async Task RunCliMigration(string generateScriptCommand, string cliName, IReadOnlyDictionary tokens) { await RunCliCommand(generateScriptCommand, cliName, tokens); LogMigrationScript("migrate.ps1"); @@ -541,70 +542,34 @@ private void LogMigrationScript(string filename) _output.WriteLine(scriptContents); } - public async Task RunPowershellScript(string script, IDictionary tokens) => + public async Task RunPowershellScript(string script, IReadOnlyDictionary tokens) => await RunShellCommand($"-File {Path.Join(GetOsDistPath(), script)}", "pwsh", GetOsDistPath(), tokens); - public async Task RunCliCommand(string command, string cliName, IDictionary tokens) => + public async Task RunCliCommand(string command, string cliName, IReadOnlyDictionary tokens) => await RunShellCommand(command, cliName, GetOsDistPath(), tokens); - private async Task RunShellCommand(string command, string fileName, string workingDirectory = null, IDictionary environmentVariables = null) + private async Task RunShellCommand(string command, string fileName, string workingDirectory = null, IReadOnlyDictionary environmentVariables = null) { - var startInfo = new ProcessStartInfo - { - WorkingDirectory = workingDirectory ?? Directory.GetCurrentDirectory(), - FileName = fileName, - Arguments = command, - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true - }; - - if (environmentVariables != null) - { - foreach (var token in environmentVariables) - { - if (startInfo.EnvironmentVariables.ContainsKey(token.Key)) - { - startInfo.EnvironmentVariables[token.Key] = token.Value; - } - else - { - startInfo.EnvironmentVariables.Add(token.Key, token.Value); - } - } - } - - _output.WriteLine($"Running command: {startInfo.FileName} {startInfo.Arguments}"); - - using var p = new Process(); - p.StartInfo = startInfo; - p.OutputDataReceived += (_, args) => - { - if (args.Data != null) - { - _output.WriteLine(args.Data); - } - }; - p.ErrorDataReceived += (_, args) => - { - if (args.Data != null) - { - _output.WriteLine(args.Data); - } - }; - p.Start(); - await p.WaitForExitAsync(); + _output.WriteLine($"Running command: {fileName} {command}"); - p.ExitCode.Should().Be(0, $"{fileName} should return an exit code of 0."); + var result = await Cli.Wrap(fileName) + .WithArguments(command) + .WithWorkingDirectory(workingDirectory ?? Directory.GetCurrentDirectory()) + .WithEnvironmentVariables(environmentVariables ?? new Dictionary()) + .WithStandardOutputPipe(PipeTarget.ToDelegate(line => _output.WriteLine(line))) + .WithStandardErrorPipe(PipeTarget.ToDelegate(line => _output.WriteLine(line))) + .ExecuteAsync(); + + result.ExitCode.Should().Be(0, $"{fileName} should return an exit code of 0."); } - public async Task RunAdoToGithubCliMigration(string generateScriptCommand, IDictionary tokens) => + public async Task RunAdoToGithubCliMigration(string generateScriptCommand, IReadOnlyDictionary tokens) => await RunCliMigration($"ado2gh {generateScriptCommand}", "gh", tokens); - public async Task RunGeiCliMigration(string generateScriptCommand, IDictionary tokens) => + public async Task RunGeiCliMigration(string generateScriptCommand, IReadOnlyDictionary tokens) => await RunCliMigration($"gei {generateScriptCommand}", "gh", tokens); - public async Task RunBbsCliMigration(string generateScriptCommand, IDictionary tokens) => + public async Task RunBbsCliMigration(string generateScriptCommand, IReadOnlyDictionary tokens) => await RunCliMigration($"bbs2gh {generateScriptCommand}", "gh", tokens); public static string GetOsDistPath() From da1e6e89d54917afe0ed3ea9f2a1b27469226231 Mon Sep 17 00:00:00 2001 From: Arin Ghazarian Date: Sat, 7 Dec 2024 15:33:58 -0800 Subject: [PATCH 10/14] mutex --- src/Octoshift/Services/OctoLogger.cs | 33 ++++++++++++++++--- .../BbsToGithub.cs | 13 +++++++- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/Octoshift/Services/OctoLogger.cs b/src/Octoshift/Services/OctoLogger.cs index d3ad02508..83d37e34e 100644 --- a/src/Octoshift/Services/OctoLogger.cs +++ b/src/Octoshift/Services/OctoLogger.cs @@ -23,6 +23,7 @@ public class OctoLogger private readonly string _logFilePath; private readonly string _verboseFilePath; private readonly bool _debugMode; + private readonly object _mutex = new(); private readonly Action _writeToLog; private readonly Action _writeToVerboseLog; @@ -49,10 +50,34 @@ public OctoLogger() _debugMode = true; } - _writeToLog = msg => File.AppendAllText(_logFilePath, msg); - _writeToVerboseLog = msg => File.AppendAllText(_verboseFilePath, msg); - _writeToConsoleOut = msg => Console.Write(msg); - _writeToConsoleError = msg => Console.Error.Write(msg); + _writeToLog = msg => + { + lock (_mutex) + { + File.AppendAllText(_logFilePath, msg); + } + }; + _writeToVerboseLog = msg => + { + lock (_mutex) + { + File.AppendAllText(_verboseFilePath, msg); + } + }; + _writeToConsoleOut = msg => + { + lock (_mutex) + { + Console.Write(msg); + } + }; + _writeToConsoleError = msg => + { + lock (_mutex) + { + Console.Error.Write(msg); + } + }; } public OctoLogger(Action writeToLog, Action writeToVerboseLog, Action writeToConsoleOut, Action writeToConsoleError) diff --git a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs index 7077fbb89..457cacc4e 100644 --- a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs @@ -31,6 +31,7 @@ public sealed class BbsToGithub : IDisposable private readonly Dictionary _tokens; private readonly DateTime _startTime; private readonly string _azureStorageConnectionString; + private readonly object _mutex = new(); public enum ArchiveUploadOption { AzureStorage, AwsS3, GithubStorage } @@ -39,7 +40,17 @@ public BbsToGithub(ITestOutputHelper output) _startTime = DateTime.Now; _output = output; - _logger = new OctoLogger(_ => { }, x => _output.WriteLine(x), _ => { }, _ => { }); + _logger = new OctoLogger( + _ => { }, + x => + { + lock (_mutex) + { + _output.WriteLine(x); + } + }, + _ => { }, + _ => { }); var sourceBbsUsername = Environment.GetEnvironmentVariable("BBS_USERNAME"); var sourceBbsPassword = Environment.GetEnvironmentVariable("BBS_PASSWORD"); From 6919e123969e53f84f8dc7e25f0678f42b1e8932 Mon Sep 17 00:00:00 2001 From: Arin Ghazarian Date: Sat, 7 Dec 2024 18:38:45 -0800 Subject: [PATCH 11/14] Revert "CliWrap" This reverts commit 187f8cd81cd6541f9ece564d44a3e164e52629c5. --- .../OctoshiftCLI.IntegrationTests.csproj | 1 - .../TestHelper.cs | 71 ++++++++++++++----- 2 files changed, 53 insertions(+), 19 deletions(-) diff --git a/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj b/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj index 37ee26d42..5d442b785 100644 --- a/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj +++ b/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj @@ -7,7 +7,6 @@ - diff --git a/src/OctoshiftCLI.IntegrationTests/TestHelper.cs b/src/OctoshiftCLI.IntegrationTests/TestHelper.cs index f6d4b3db8..daec5afc5 100644 --- a/src/OctoshiftCLI.IntegrationTests/TestHelper.cs +++ b/src/OctoshiftCLI.IntegrationTests/TestHelper.cs @@ -6,7 +6,6 @@ using System.Runtime.InteropServices; using System.Threading.Tasks; using Azure.Storage.Blobs; -using CliWrap; using FluentAssertions; using Newtonsoft.Json.Linq; using OctoshiftCLI.Services; @@ -527,7 +526,7 @@ public static string GetOsName() : throw new InvalidOperationException("Could not determine OS"); } - public async Task RunCliMigration(string generateScriptCommand, string cliName, IReadOnlyDictionary tokens) + public async Task RunCliMigration(string generateScriptCommand, string cliName, IDictionary tokens) { await RunCliCommand(generateScriptCommand, cliName, tokens); LogMigrationScript("migrate.ps1"); @@ -542,34 +541,70 @@ private void LogMigrationScript(string filename) _output.WriteLine(scriptContents); } - public async Task RunPowershellScript(string script, IReadOnlyDictionary tokens) => + public async Task RunPowershellScript(string script, IDictionary tokens) => await RunShellCommand($"-File {Path.Join(GetOsDistPath(), script)}", "pwsh", GetOsDistPath(), tokens); - public async Task RunCliCommand(string command, string cliName, IReadOnlyDictionary tokens) => + public async Task RunCliCommand(string command, string cliName, IDictionary tokens) => await RunShellCommand(command, cliName, GetOsDistPath(), tokens); - private async Task RunShellCommand(string command, string fileName, string workingDirectory = null, IReadOnlyDictionary environmentVariables = null) + private async Task RunShellCommand(string command, string fileName, string workingDirectory = null, IDictionary environmentVariables = null) { - _output.WriteLine($"Running command: {fileName} {command}"); + var startInfo = new ProcessStartInfo + { + WorkingDirectory = workingDirectory ?? Directory.GetCurrentDirectory(), + FileName = fileName, + Arguments = command, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + + if (environmentVariables != null) + { + foreach (var token in environmentVariables) + { + if (startInfo.EnvironmentVariables.ContainsKey(token.Key)) + { + startInfo.EnvironmentVariables[token.Key] = token.Value; + } + else + { + startInfo.EnvironmentVariables.Add(token.Key, token.Value); + } + } + } + + _output.WriteLine($"Running command: {startInfo.FileName} {startInfo.Arguments}"); + + using var p = new Process(); + p.StartInfo = startInfo; + p.OutputDataReceived += (_, args) => + { + if (args.Data != null) + { + _output.WriteLine(args.Data); + } + }; + p.ErrorDataReceived += (_, args) => + { + if (args.Data != null) + { + _output.WriteLine(args.Data); + } + }; + p.Start(); + await p.WaitForExitAsync(); - var result = await Cli.Wrap(fileName) - .WithArguments(command) - .WithWorkingDirectory(workingDirectory ?? Directory.GetCurrentDirectory()) - .WithEnvironmentVariables(environmentVariables ?? new Dictionary()) - .WithStandardOutputPipe(PipeTarget.ToDelegate(line => _output.WriteLine(line))) - .WithStandardErrorPipe(PipeTarget.ToDelegate(line => _output.WriteLine(line))) - .ExecuteAsync(); - - result.ExitCode.Should().Be(0, $"{fileName} should return an exit code of 0."); + p.ExitCode.Should().Be(0, $"{fileName} should return an exit code of 0."); } - public async Task RunAdoToGithubCliMigration(string generateScriptCommand, IReadOnlyDictionary tokens) => + public async Task RunAdoToGithubCliMigration(string generateScriptCommand, IDictionary tokens) => await RunCliMigration($"ado2gh {generateScriptCommand}", "gh", tokens); - public async Task RunGeiCliMigration(string generateScriptCommand, IReadOnlyDictionary tokens) => + public async Task RunGeiCliMigration(string generateScriptCommand, IDictionary tokens) => await RunCliMigration($"gei {generateScriptCommand}", "gh", tokens); - public async Task RunBbsCliMigration(string generateScriptCommand, IReadOnlyDictionary tokens) => + public async Task RunBbsCliMigration(string generateScriptCommand, IDictionary tokens) => await RunCliMigration($"bbs2gh {generateScriptCommand}", "gh", tokens); public static string GetOsDistPath() From 009ad0e2d61ab05d568c0c718e93d7683eae5619 Mon Sep 17 00:00:00 2001 From: Arin Ghazarian Date: Sat, 7 Dec 2024 19:22:33 -0800 Subject: [PATCH 12/14] Upgrade xunit --- .../OctoshiftCLI.IntegrationTests.csproj | 4 ++-- src/OctoshiftCLI.Tests/OctoshiftCLI.Tests.csproj | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj b/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj index 5d442b785..2060e9e72 100644 --- a/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj +++ b/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj @@ -12,8 +12,8 @@ - - + + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/src/OctoshiftCLI.Tests/OctoshiftCLI.Tests.csproj b/src/OctoshiftCLI.Tests/OctoshiftCLI.Tests.csproj index 2545e0709..833ba590f 100644 --- a/src/OctoshiftCLI.Tests/OctoshiftCLI.Tests.csproj +++ b/src/OctoshiftCLI.Tests/OctoshiftCLI.Tests.csproj @@ -21,8 +21,8 @@ - - + + runtime; build; native; contentfiles; analyzers; buildtransitive all From 113d2e1c14bf49e72e58f5b03ad45ce1995e0edc Mon Sep 17 00:00:00 2001 From: Arin Ghazarian Date: Sat, 7 Dec 2024 19:33:10 -0800 Subject: [PATCH 13/14] Copy xunit runner config to output --- .../OctoshiftCLI.IntegrationTests.csproj | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj b/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj index 2060e9e72..8ec25536f 100644 --- a/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj +++ b/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj @@ -27,4 +27,10 @@ + + + PreserveNewest + + + From 993fe58b1175a58ad990de7e61936bfdddc0504e Mon Sep 17 00:00:00 2001 From: Arin Ghazarian Date: Sat, 7 Dec 2024 19:45:03 -0800 Subject: [PATCH 14/14] Reapply "CliWrap" This reverts commit 6919e123969e53f84f8dc7e25f0678f42b1e8932. --- .../OctoshiftCLI.IntegrationTests.csproj | 1 + .../TestHelper.cs | 71 +++++-------------- 2 files changed, 19 insertions(+), 53 deletions(-) diff --git a/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj b/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj index 8ec25536f..b3b60c1ea 100644 --- a/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj +++ b/src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj @@ -7,6 +7,7 @@ + diff --git a/src/OctoshiftCLI.IntegrationTests/TestHelper.cs b/src/OctoshiftCLI.IntegrationTests/TestHelper.cs index daec5afc5..f6d4b3db8 100644 --- a/src/OctoshiftCLI.IntegrationTests/TestHelper.cs +++ b/src/OctoshiftCLI.IntegrationTests/TestHelper.cs @@ -6,6 +6,7 @@ using System.Runtime.InteropServices; using System.Threading.Tasks; using Azure.Storage.Blobs; +using CliWrap; using FluentAssertions; using Newtonsoft.Json.Linq; using OctoshiftCLI.Services; @@ -526,7 +527,7 @@ public static string GetOsName() : throw new InvalidOperationException("Could not determine OS"); } - public async Task RunCliMigration(string generateScriptCommand, string cliName, IDictionary tokens) + public async Task RunCliMigration(string generateScriptCommand, string cliName, IReadOnlyDictionary tokens) { await RunCliCommand(generateScriptCommand, cliName, tokens); LogMigrationScript("migrate.ps1"); @@ -541,70 +542,34 @@ private void LogMigrationScript(string filename) _output.WriteLine(scriptContents); } - public async Task RunPowershellScript(string script, IDictionary tokens) => + public async Task RunPowershellScript(string script, IReadOnlyDictionary tokens) => await RunShellCommand($"-File {Path.Join(GetOsDistPath(), script)}", "pwsh", GetOsDistPath(), tokens); - public async Task RunCliCommand(string command, string cliName, IDictionary tokens) => + public async Task RunCliCommand(string command, string cliName, IReadOnlyDictionary tokens) => await RunShellCommand(command, cliName, GetOsDistPath(), tokens); - private async Task RunShellCommand(string command, string fileName, string workingDirectory = null, IDictionary environmentVariables = null) + private async Task RunShellCommand(string command, string fileName, string workingDirectory = null, IReadOnlyDictionary environmentVariables = null) { - var startInfo = new ProcessStartInfo - { - WorkingDirectory = workingDirectory ?? Directory.GetCurrentDirectory(), - FileName = fileName, - Arguments = command, - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true - }; - - if (environmentVariables != null) - { - foreach (var token in environmentVariables) - { - if (startInfo.EnvironmentVariables.ContainsKey(token.Key)) - { - startInfo.EnvironmentVariables[token.Key] = token.Value; - } - else - { - startInfo.EnvironmentVariables.Add(token.Key, token.Value); - } - } - } - - _output.WriteLine($"Running command: {startInfo.FileName} {startInfo.Arguments}"); - - using var p = new Process(); - p.StartInfo = startInfo; - p.OutputDataReceived += (_, args) => - { - if (args.Data != null) - { - _output.WriteLine(args.Data); - } - }; - p.ErrorDataReceived += (_, args) => - { - if (args.Data != null) - { - _output.WriteLine(args.Data); - } - }; - p.Start(); - await p.WaitForExitAsync(); + _output.WriteLine($"Running command: {fileName} {command}"); - p.ExitCode.Should().Be(0, $"{fileName} should return an exit code of 0."); + var result = await Cli.Wrap(fileName) + .WithArguments(command) + .WithWorkingDirectory(workingDirectory ?? Directory.GetCurrentDirectory()) + .WithEnvironmentVariables(environmentVariables ?? new Dictionary()) + .WithStandardOutputPipe(PipeTarget.ToDelegate(line => _output.WriteLine(line))) + .WithStandardErrorPipe(PipeTarget.ToDelegate(line => _output.WriteLine(line))) + .ExecuteAsync(); + + result.ExitCode.Should().Be(0, $"{fileName} should return an exit code of 0."); } - public async Task RunAdoToGithubCliMigration(string generateScriptCommand, IDictionary tokens) => + public async Task RunAdoToGithubCliMigration(string generateScriptCommand, IReadOnlyDictionary tokens) => await RunCliMigration($"ado2gh {generateScriptCommand}", "gh", tokens); - public async Task RunGeiCliMigration(string generateScriptCommand, IDictionary tokens) => + public async Task RunGeiCliMigration(string generateScriptCommand, IReadOnlyDictionary tokens) => await RunCliMigration($"gei {generateScriptCommand}", "gh", tokens); - public async Task RunBbsCliMigration(string generateScriptCommand, IDictionary tokens) => + public async Task RunBbsCliMigration(string generateScriptCommand, IReadOnlyDictionary tokens) => await RunCliMigration($"bbs2gh {generateScriptCommand}", "gh", tokens); public static string GetOsDistPath()