[DO NOT MERGE] Debugging INT test

.github/workflows/CI.yml

@@ -250,7 +250,7 @@ jobs:
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }}
         LD_LIBRARY_PATH: '$LD_LIBRARY_PATH:${{ github.workspace }}/src/OctoshiftCLI.IntegrationTests/bin/Debug/net8.0/runtimes/ubuntu.18.04-x64/native'
-      run: dotnet test src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj --filter "${{ matrix.source-vcs }}ToGithub" --logger:"junit;LogFilePath=integration-tests.xml" /p:VersionPrefix=9.9
+      run: dotnet test src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj --filter "${{ matrix.source-vcs }}ToGithub" --logger:"junit;LogFilePath=integration-tests.xml" --logger "console;verbosity=normal" /p:VersionPrefix=9.9
 
     - name: Publish Integration Test Results
       uses: EnricoMi/publish-unit-test-result-action@v2

.github/workflows/integration-tests.yml

@@ -175,7 +175,7 @@ jobs:
         AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }}
         GEI_DEBUG_MODE: 'true'
         LD_LIBRARY_PATH: '$LD_LIBRARY_PATH:${{ github.workspace }}/src/OctoshiftCLI.IntegrationTests/bin/Debug/net8.0/runtimes/ubuntu.18.04-x64/native'
-      run: dotnet test src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj --filter "${{ matrix.source-vcs }}ToGithub" --logger:"junit;LogFilePath=integration-tests.xml" /p:VersionPrefix=9.9
+      run: dotnet test src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj --filter "${{ matrix.source-vcs }}ToGithub" --logger:"junit;LogFilePath=integration-tests.xml" --logger "console;verbosity=normal" /p:VersionPrefix=9.9
 
     - name: Publish Integration Test Results
       uses: EnricoMi/publish-unit-test-result-action@v2

RELEASENOTES.md

@@ -1 +1,2 @@
 - catches exception when Target is not a member of the organization and the `--skip-invitation` flag is enabled
+- Add progress report to `gh [gei|bbs2gh] migrate-repo` command when uploading migration archives to Azure Blob or AWS S3

src/Octoshift/Extensions/NumericExtensions.cs

@@ -0,0 +1,19 @@
+namespace OctoshiftCLI.Extensions;
+
+public static class NumericExtensions
+{
+    public static string ToLogFriendlySize(this long size)
+    {
+        const int kilobyte = 1024;
+        const int megabyte = 1024 * kilobyte;
+        const int gigabyte = 1024 * megabyte;
+
+        return size switch
+        {
+            < kilobyte => $"{size:n0} bytes",
+            < megabyte => $"{size / (double)kilobyte:n0} KB",
+            < gigabyte => $"{size / (double)megabyte:n0} MB",
+            _ => $"{size / (double)gigabyte:n2} GB"
+        };
+    }
+}

src/Octoshift/Services/AwsApi.cs

@@ -13,14 +13,22 @@ namespace OctoshiftCLI.Services;
 public class AwsApi : IDisposable
 {
     private const int AUTHORIZATION_TIMEOUT_IN_HOURS = 48;
+    private const int UPLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS = 10;
 
     private readonly ITransferUtility _transferUtility;
+    private readonly object _mutex = new();
+    private readonly OctoLogger _log;
+    private DateTime _nextProgressReport = DateTime.Now;
 
-    public AwsApi(ITransferUtility transferUtility) => _transferUtility = transferUtility;
+    public AwsApi(ITransferUtility transferUtility, OctoLogger log)
+    {
+        _transferUtility = transferUtility;
+        _log = log;
+    }
 
 #pragma warning disable CA2000
-    public AwsApi(string awsAccessKeyId, string awsSecretAccessKey, string awsRegion = null, string awsSessionToken = null)
-        : this(new TransferUtility(BuildAmazonS3Client(awsAccessKeyId, awsSecretAccessKey, awsRegion, awsSessionToken)))
+    public AwsApi(OctoLogger log, string awsAccessKeyId, string awsSecretAccessKey, string awsRegion = null, string awsSessionToken = null)
+        : this(new TransferUtility(BuildAmazonS3Client(awsAccessKeyId, awsSecretAccessKey, awsRegion, awsSessionToken)), log)
 #pragma warning restore CA2000
     {
     }
@@ -51,20 +59,37 @@ public virtual async Task<string> UploadToBucket(string bucketName, string fileN
     {
         try
         {
-            await _transferUtility.UploadAsync(fileName, bucketName, keyName);
+            var uploadRequest = new TransferUtilityUploadRequest
+            {
+                BucketName = bucketName,
+                Key = keyName,
+                FilePath = fileName
+            };
+            return await UploadToBucket(uploadRequest);
         }
         catch (Exception ex) when (ex is TaskCanceledException or TimeoutException)
         {
             throw new OctoshiftCliException($"Upload of archive \"{fileName}\" to AWS timed out", ex);
         }
-
-        return GetPreSignedUrlForFile(bucketName, keyName);
     }
 
     public virtual async Task<string> UploadToBucket(string bucketName, Stream content, string keyName)
     {
-        await _transferUtility.UploadAsync(content, bucketName, keyName);
-        return GetPreSignedUrlForFile(bucketName, keyName);
+        var uploadRequest = new TransferUtilityUploadRequest
+        {
+            BucketName = bucketName,
+            Key = keyName,
+            InputStream = content
+        };
+        return await UploadToBucket(uploadRequest);
+    }
+
+    private async Task<string> UploadToBucket(TransferUtilityUploadRequest uploadRequest)
+    {
+        uploadRequest.UploadProgressEvent += (_, args) => LogProgress(args.PercentDone, args.TransferredBytes, args.TotalBytes);
+        await _transferUtility.UploadAsync(uploadRequest);
+
+        return GetPreSignedUrlForFile(uploadRequest.BucketName, uploadRequest.Key);
     }
 
     private string GetPreSignedUrlForFile(string bucketName, string keyName)
@@ -81,6 +106,25 @@ private string GetPreSignedUrlForFile(string bucketName, string keyName)
         return _transferUtility.S3Client.GetPreSignedURL(urlRequest);
     }
 
+    private void LogProgress(int percentDone, long uploadedBytes, long totalBytes)
+    {
+        lock (_mutex)
+        {
+            if (DateTime.Now < _nextProgressReport)
+            {
+                return;
+            }
+
+            _nextProgressReport = _nextProgressReport.AddSeconds(UPLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS);
+        }
+
+        var progressMessage = uploadedBytes > 0
+            ? $", {uploadedBytes.ToLogFriendlySize()} out of {totalBytes.ToLogFriendlySize()} ({percentDone}%) completed"
+            : "";
+
+        _log.LogInformation($"Archive upload in progress{progressMessage}...");
+    }
+
     protected virtual void Dispose(bool disposing)
     {
         if (disposing)

src/Octoshift/Services/AzureApi.cs

@@ -6,6 +6,7 @@
 using Azure.Storage.Blobs.Models;
 using Azure.Storage.Blobs.Specialized;
 using Azure.Storage.Sas;
+using OctoshiftCLI.Extensions;
 
 namespace OctoshiftCLI.Services;
 
@@ -14,9 +15,12 @@ public class AzureApi
     private readonly HttpClient _client;
     private readonly BlobServiceClient _blobServiceClient;
     private readonly OctoLogger _log;
+    private readonly object _mutex = new();
     private const string CONTAINER_PREFIX = "migration-archives";
     private const int AUTHORIZATION_TIMEOUT_IN_HOURS = 48;
     private const int DEFAULT_BLOCK_SIZE = 4 * 1024 * 1024;
+    private const int UPLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS = 10;
+    private DateTime _nextProgressReport = DateTime.Now;
 
     public AzureApi(HttpClient client, BlobServiceClient blobServiceClient, OctoLogger log)
     {
@@ -48,16 +52,24 @@ public virtual async Task<Uri> UploadToBlob(string fileName, byte[] content)
 
     public virtual async Task<Uri> UploadToBlob(string fileName, Stream content)
     {
+        ArgumentNullException.ThrowIfNull(fileName, nameof(fileName));
+        ArgumentNullException.ThrowIfNull(content);
+
         var containerClient = await CreateBlobContainerAsync();
         var blobClient = containerClient.GetBlobClient(fileName);
 
+        var progress = new Progress<long>();
+        var archiveSize = content.Length;
+        progress.ProgressChanged += (_, uploadedBytes) => LogProgress(uploadedBytes, archiveSize);
+
         var options = new BlobUploadOptions
         {
             TransferOptions = new Azure.Storage.StorageTransferOptions()
             {
                 InitialTransferSize = DEFAULT_BLOCK_SIZE,
                 MaximumTransferSize = DEFAULT_BLOCK_SIZE
             },
+            ProgressHandler = progress
         };
 
         await blobClient.UploadAsync(content, options);
@@ -89,4 +101,24 @@ private Uri GetServiceSasUriForBlob(BlobClient blobClient)
 
         return blobClient.GenerateSasUri(sasBuilder);
     }
+
+    private void LogProgress(long uploadedBytes, long totalBytes)
+    {
+        lock (_mutex)
+        {
+            if (DateTime.Now < _nextProgressReport)
+            {
+                return;
+            }
+
+            _nextProgressReport = _nextProgressReport.AddSeconds(UPLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS);
+        }
+
+        var percentage = (int)(uploadedBytes * 100L / totalBytes);
+        var progressMessage = uploadedBytes > 0
+            ? $", {uploadedBytes.ToLogFriendlySize()} out of {totalBytes.ToLogFriendlySize()} ({percentage}%) completed"
+            : "";
+
+        _log.LogInformation($"Archive upload in progress{progressMessage}...");
+    }
 }

src/Octoshift/Services/OctoLogger.cs

@@ -23,6 +23,7 @@ public class OctoLogger
     private readonly string _logFilePath;
     private readonly string _verboseFilePath;
     private readonly bool _debugMode;
+    private readonly object _mutex = new();
 
     private readonly Action<string> _writeToLog;
     private readonly Action<string> _writeToVerboseLog;
@@ -49,10 +50,34 @@ public OctoLogger()
             _debugMode = true;
         }
 
-        _writeToLog = msg => File.AppendAllText(_logFilePath, msg);
-        _writeToVerboseLog = msg => File.AppendAllText(_verboseFilePath, msg);
-        _writeToConsoleOut = msg => Console.Write(msg);
-        _writeToConsoleError = msg => Console.Error.Write(msg);
+        _writeToLog = msg =>
+        {
+            lock (_mutex)
+            {
+                File.AppendAllText(_logFilePath, msg);
+            }
+        };
+        _writeToVerboseLog = msg =>
+        {
+            lock (_mutex)
+            {
+                File.AppendAllText(_verboseFilePath, msg);
+            }
+        };
+        _writeToConsoleOut = msg =>
+        {
+            lock (_mutex)
+            {
+                Console.Write(msg);
+            }
+        };
+        _writeToConsoleError = msg =>
+        {
+            lock (_mutex)
+            {
+                Console.Error.Write(msg);
+            }
+        };
     }
 
     public OctoLogger(Action<string> writeToLog, Action<string> writeToVerboseLog, Action<string> writeToConsoleOut, Action<string> writeToConsoleError)

src/OctoshiftCLI.IntegrationTests/AdoBasicToGithub.cs

@@ -11,7 +11,7 @@ public AdoBasicToGithub(ITestOutputHelper output) : base(output)
         {
         }
 
-        [Fact]
+        // [Fact]
         public async Task Basic()
         {
             var adoOrg = $"gei-e2e-testing-basic-{TestHelper.GetOsName()}";

src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs

@@ -11,7 +11,7 @@ public AdoCsvToGithub(ITestOutputHelper output) : base(output)
         {
         }
 
-        [Fact]
+        // [Fact]
         public async Task With_Inventory_Report_Csv()
         {
             var adoOrg = $"gei-e2e-testing-csv-{TestHelper.GetOsName()}";

src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs

@@ -31,6 +31,7 @@
     private readonly Dictionary<string, string> _tokens;
     private readonly DateTime _startTime;
     private readonly string _azureStorageConnectionString;
+    private readonly object _mutex = new();
 
     public enum ArchiveUploadOption { AzureStorage, AwsS3, GithubStorage }
 
@@ -39,7 +40,17 @@
         _startTime = DateTime.Now;
         _output = output;
 
-        _logger = new OctoLogger(_ => { }, x => _output.WriteLine(x), _ => { }, _ => { });
+        _logger = new OctoLogger(
+            _ => { },
+            x =>
+            {
+                lock (_mutex)
+                {
+                    _output.WriteLine(x);
+                }
+            }, 
+            _ => { }, 
+            _ => { });
 
         var sourceBbsUsername = Environment.GetEnvironmentVariable("BBS_USERNAME");
         var sourceBbsPassword = Environment.GetEnvironmentVariable("BBS_PASSWORD");
@@ -70,8 +81,8 @@
     [Theory]
     [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.AzureStorage)]
     [InlineData("http://e2e-bbs-7-21-9-win-2019.eastus.cloudapp.azure.com:7990", false, ArchiveUploadOption.AzureStorage)]
-    [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.AwsS3)]
-    [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.GithubStorage)]
+    // [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.AwsS3)]
+    // [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.GithubStorage)]
     public async Task Basic(string bbsServer, bool useSshForArchiveDownload, ArchiveUploadOption uploadOption)
     {
         var bbsProjectKey = $"E2E-{TestHelper.GetOsName().ToUpper()}";
@@ -141,8 +152,8 @@
         // TODO: Assert migration logs are downloaded
     }
 
-    [Fact]
+    // [Fact]
     public async Task MigrateRepo_MultipartUpload()
     {
         var githubTargetOrg = $"octoshift-e2e-bbs-{TestHelper.GetOsName()}";
         var bbsProjectKey = $"IN";

src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs

@@ -63,9 +63,9 @@ public GhesToGithub(ITestOutputHelper output)
         _targetHelper = new TestHelper(_output, _targetGithubApi, _targetGithubClient, _blobServiceClient);
     }
 
-    [Theory]
-    [InlineData(false)]
-    [InlineData(true)]
+    // [Theory]
+    // [InlineData(false)]
+    // [InlineData(true)]
     public async Task Basic(bool useGithubStorage)
     {
         var githubSourceOrg = $"e2e-testing-{TestHelper.GetOsName()}";

src/OctoshiftCLI.IntegrationTests/GithubToGithub.cs

@@ -39,7 +39,7 @@ public GithubToGithub(ITestOutputHelper output)
             _helper = new TestHelper(_output, _githubApi, _githubClient);
         }
 
-        [Fact]
+        // [Fact]
         public async Task Basic()
         {
             var githubSourceOrg = $"octoshift-e2e-source-{TestHelper.GetOsName()}";

src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj

@@ -7,13 +7,14 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="CliWrap" Version="3.6.7" />
     <PackageReference Include="FluentAssertions" Version="6.12.0" />
     <PackageReference Include="JunitXml.TestLogger" Version="3.0.134" />
     <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.7.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.2" />
     <PackageReference Include="Moq" Version="4.20.69" />
-    <PackageReference Include="xunit" Version="2.5.0" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.3">
+    <PackageReference Include="xunit" Version="2.9.2" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
@@ -27,4 +28,10 @@
     <ProjectReference Include="..\Octoshift\Octoshift.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <None Update="xunit.runner.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
 </Project>