Add test to compare msft and sb sdk contents

eng/source-build-tarball-build-official.yml

@@ -11,4 +11,4 @@ stages:
   jobs:
   - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml
     parameters:
-      tarballResourceId: installer-build-resource
+      installerBuildResourceId: installer-build-resource

eng/source-build-tarball-build.yml

@@ -11,4 +11,4 @@ stages:
   jobs:
   - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml
     parameters:
-      tarballResourceId: installer-build-resource
+      installerBuildResourceId: installer-build-resource

src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml

@@ -7,11 +7,12 @@ parameters:
   # Dependent jobs that must be completed before this job will run
   dependsOn:
 
-  # The resource id of the tarball to download and build
-  tarballResourceId: current
+  # Resource id of the installer build to retrieve source/product tarball's from
+  installerBuildResourceId: current
 
   # The following parameters aren't expected to be passed in rather they are used for encapsulation
   # -----------------------------------------------------------------------------------------------
+  buildJobTimeout: 300
   centOS7Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-source-build-20210714125450-5d87b80
   centOS8Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-8-source-build-20211118190102-9355e7b
   debian9Container: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-stretch-20211001171226-047508b
@@ -42,33 +43,38 @@ jobs:
         _BuildArch: x64
         _Container: ${{ parameters.centOS7Container }}
         _ExcludeOmniSharpTests: true
+        _Platform: linux
         _RunOnline: true
       CentOS7-Offline:
         _BootstrapPrep: true
         _BuildArch: x64
         _Container: ${{ parameters.centOS7Container }}
         _ExcludeOmniSharpTests: true
+        _Platform: linux
         _RunOnline: false
       ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
         CentOS8-Offline:
           _BootstrapPrep: true
           _BuildArch: x64
           _Container: ${{ parameters.centOS8Container }}
           _ExcludeOmniSharpTests: false
+          _Platform: linux
           _RunOnline: false
         Fedora33-Offline:
           _BootstrapPrep: false
           _BuildArch: x64
           _Container: ${{ parameters.fedora33Container }}
           _ExcludeOmniSharpTests: false
+          _Platform: linux
           _RunOnline: false
         Ubuntu1804-Offline:
           _BootstrapPrep: true
           _BuildArch: x64
           _Container: ${{ parameters.ubuntu1804Container }}
           _ExcludeOmniSharpTests: false
+          _Platform: linux
           _RunOnline: false
-  timeoutInMinutes: 300
+  timeoutInMinutes: ${{ parameters.buildJobTimeout }}
   variables:
   - ${{ if ne(variables['System.TeamProject'], 'public') }}:
     - group: AzureDevOps-Artifact-Feeds-Pats
@@ -79,20 +85,13 @@ jobs:
   - checkout: self
     clean: true
 
-  - ${{ if ne(variables['System.TeamProject'], 'public') }}:
-    - task: Bash@3
-      displayName: Setup Private Feeds Credentials
-      inputs:
-        filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
-        arguments: $(Build.SourcesDirectory)/NuGet.config $Token
-      env:
-        Token: $(dn-bot-dnceng-artifact-feeds-rw)
-
   - template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-build-tarball.yml
     parameters:
       buildArch: $(_BuildArch)
       container: $(_Container)
       excludeOmniSharpTests: $(_ExcludeOmniSharpTests)
+      installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
+      platform: $(_Platform)
       prepScript: |
         set -x
 
@@ -104,7 +103,6 @@ jobs:
         docker run --rm -v ${{ parameters.tarballDir }}:/tarball -w /tarball $(_Container) ./prep.sh  ${customPrepArgs}
       runOnline: $(_RunOnline)
       tarballDir: ${{ parameters.tarballDir }}
-      tarballResourceId: ${{ parameters.tarballResourceId }}
 
 - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
   - job: Rebuild_Tarball
@@ -121,8 +119,9 @@ jobs:
           _PreviousSourceBuildArtifact: Build Tarball Fedora33-Offline_Artifacts
           _BuildArch: x64
           _Container: ${{ parameters.fedora33Container }}
+          _Platform: linux
           _RunOnline: false
-    timeoutInMinutes: 180
+    timeoutInMinutes: ${{ parameters.buildJobTimeout }}
     variables:
     - ${{ if ne(variables['System.TeamProject'], 'public') }}:
       - group: AzureDevOps-Artifact-Feeds-Pats
@@ -133,15 +132,6 @@ jobs:
     - checkout: self
       clean: true
 
-    - ${{ if ne(variables['System.TeamProject'], 'public') }}:
-      - task: Bash@3
-        displayName: Setup Private Feeds Credentials
-        inputs:
-          filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
-          arguments: $(Build.SourcesDirectory)/NuGet.config $Token
-        env:
-          Token: $(dn-bot-dnceng-artifact-feeds-rw)
-
     - download: current
       artifact: $(_PreviousSourceBuildArtifact)
       patterns: '*.tar.gz'
@@ -159,6 +149,9 @@ jobs:
         additionalBuildArgs: --with-sdk /tarball/.dotnet
         buildArch: $(_BuildArch)
         container: $(_Container)
+        installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
+        isBootstrapped: true
+        platform: $(_Platform)
         prepScript: |
           set -x
 
@@ -168,4 +161,3 @@ jobs:
           eval rm -f "$tarballFilePath"
         runOnline: $(_RunOnline)
         tarballDir: ${{ parameters.tarballDir }}
-        tarballResourceId: ${{ parameters.tarballResourceId }}

src/SourceBuild/Arcade/eng/common/templates/steps/source-build-build-tarball.yml

@@ -13,6 +13,15 @@ parameters:
   # Skip running the OmniSharp smoke-tests
   excludeOmniSharpTests: false
 
+  # Resource id of the installer build to retrieve source/product tarball's from
+  installerBuildResourceId: null
+
+  # Whether or not this is a bootstrapped build
+  isBootstrapped: false
+
+  # Platform the build is running on (linux, osx, win)
+  platform: null
+
   # Script logic to prep the tarball for building
   prepScript: null
 
@@ -22,21 +31,32 @@ parameters:
   # Directory to build the tarball in
   tarballDir:
 
-  # Resource id of the tarball to download and build
-  tarballResourceId: null
-
 steps:
-  - download: ${{ parameters.tarballResourceId }}
+  - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+    - task: Bash@3
+      displayName: Setup Private Feeds Credentials
+      inputs:
+        filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
+        arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+      env:
+        Token: $(dn-bot-dnceng-artifact-feeds-rw)
+
+  - download: ${{ parameters.installerBuildResourceId }}
     artifact: BlobArtifacts
     patterns: '**/dotnet-sdk-source*.tar.gz'
-    displayName: Download Tarball
+    displayName: Download Source Tarball
+
+  - download: ${{ parameters.installerBuildResourceId }}
+    artifact: BlobArtifacts
+    patterns: '**/dotnet-sdk-!(*-*)-${{ parameters.platform }}-${{ parameters.buildArch }}.tar.gz'
+    displayName: Download MSFT sdk Tarball
 
   - script: |
       set -x
 
       resourceIdPathSegment=
-      if [ '${{ parameters.tarballResourceId }}' != 'current' ]; then
-        resourceIdPathSegment='${{ parameters.tarballResourceId }}/'
+      if [ '${{ parameters.installerBuildResourceId }}' != 'current' ]; then
+        resourceIdPathSegment='${{ parameters.installerBuildResourceId }}/'
       fi
 
       mkdir -p "${{ parameters.tarballDir }}"
@@ -70,7 +90,16 @@ steps:
       rm -f ${{ parameters.tarballDir }}/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/online.NuGet.Config
       cp NuGet.config ${{ parameters.tarballDir }}/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/online.NuGet.Config
 
-      docker run --rm -v ${{ parameters.tarballDir }}:/tarball -w /tarball -e EXCLUDE_OMNISHARP_TESTS=${{ parameters.excludeOmniSharpTests}} ${{ parameters.container }} ./build.sh --run-smoke-test ${{ parameters.additionalBuildArgs }}
+      dockerVolumeArgs="-v ${{ parameters.tarballDir }}:/tarball"
+      dockerEnvArgs="-e EXCLUDE_OMNISHARP_TESTS=${{ parameters.excludeOmniSharpTests}}"
+
+      if [ '${{ parameters.isBootstrapped}}' != 'true' && '${{ parameters.installerBuildResourceId }}' != 'current' ]; then
+        dockerVolumeArgs+=" -v $(PIPELINE.WORKSPACE)/${{ parameters.installerBuildResourceId }}/BlobArtifacts/:/BlobArtifacts"
+        msftSdkTarballName=$(find "$(PIPELINE.WORKSPACE)/${{ parameters.installerBuildResourceId }}/BlobArtifacts/" -name "dotnet-sdk-*-${{ parameters.Platform }}-${{ parameters.buildArch }}.tar.gz" -exec basename {} \;)
+        dockerEnvArgs+=" -e MSFT_SDK_TARBALL_PATH=/BlobArtifacts/$msftSdkTarballName"
+      fi
+
+      docker run --rm $dockerVolumeArgs -w /tarball $dockerEnvArgs ${{ parameters.container }} ./build.sh --run-smoke-test ${{ parameters.additionalBuildArgs }}
     displayName: Run Tests
 
   # Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph.

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/BaselineHelper.cs

@@ -4,9 +4,11 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using Xunit;
+using Xunit.Abstractions;
 
 namespace Microsoft.DotNet.SourceBuild.SmokeTests
 {
@@ -32,6 +34,35 @@ public static void Compare(string baselineFileName, IOrderedEnumerable<string> a
             Assert.Null(message);
         }
 
+        public static void Compare(string baselineFileName, string actual, ITestOutputHelper outputHelper)
+        {
+            string baselineFilePath = GetBaselineFilePath(baselineFileName);
+            string baseline = File.ReadAllText(baselineFilePath);
+
+            string? message = null;
+            if (baseline != actual)
+            {
+                string actualBaselineFilePath = Path.Combine(Environment.CurrentDirectory, $"{baselineFileName}");
+                File.WriteAllText(actualBaselineFilePath, actual);
+
+                // Retrieve a diff in order to provide a UX which calls out the diffs.
+                string diff = DiffFiles(baselineFilePath, actualBaselineFilePath, outputHelper);
+                message = $"{Environment.NewLine}Baseline '{baselineFilePath}' does not match actual '{actualBaselineFilePath}`.  {Environment.NewLine}"
+                    + $"{diff}{Environment.NewLine}";
+            }
+
+            Assert.Null(message);
+        }
+
+        public static string DiffFiles(string file1Path, string file2Path, ITestOutputHelper outputHelper)
+        {
+            (Process Process, string StdOut, string StdErr) diffResult =
+                ExecuteHelper.ExecuteProcess("git", $"diff --no-index {file1Path} {file2Path}", outputHelper);
+            Assert.Equal(1, diffResult.Process.ExitCode);
+
+            return diffResult.StdOut;
+        }
+
         private static string GetBaselineFilePath(string baselineFileName) => Path.Combine(Directory.GetCurrentDirectory(), "baselines", baselineFileName);
     }
 }

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs

@@ -3,17 +3,21 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
+using System.IO;
 
 namespace Microsoft.DotNet.SourceBuild.SmokeTests;
 
 internal static class Config
 {
-    public static string DotNetDirectory { get; } = Environment.GetEnvironmentVariable("DOTNET_DIR") ?? "./.dotnet";
+    public static string DotNetDirectory { get; } = 
+        Environment.GetEnvironmentVariable("DOTNET_DIR") ?? Path.Combine(Directory.GetCurrentDirectory(), ".dotnet");
     public static string DotNetTarballPath { get; } = Environment.GetEnvironmentVariable(DotNetTarballPathEnv) ?? string.Empty;
     public const string DotNetTarballPathEnv = "DOTNET_TARBALL_PATH";
     public static bool ExcludeOmniSharpTests { get; } =
         bool.TryParse(Environment.GetEnvironmentVariable("EXCLUDE_OMNISHARP_TESTS"), out bool excludeOmniSharpTests) ? excludeOmniSharpTests : false;
     public static bool ExcludeOnlineTests { get; } =
         bool.TryParse(Environment.GetEnvironmentVariable("EXCLUDE_ONLINE_TESTS"), out bool excludeOnlineTests) ? excludeOnlineTests : false;
+    public static string MsftSdkTarballPath { get; } = Environment.GetEnvironmentVariable(MsftSdkTarballPathEnv) ?? string.Empty;
+    public const string MsftSdkTarballPathEnv = "MSFT_SDK_TARBALL_PATH";
     public static string TargetRid { get; } = Environment.GetEnvironmentVariable("TARGET_RID") ?? string.Empty;
 }

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs

@@ -12,24 +12,27 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests;
 
 internal class DotNetHelper
 {
+    private static readonly object s_lockObj = new object();
+
     public string DotNetPath { get; }
-    public string DotNetInstallDirectory { get; }
 
     public DotNetHelper(ITestOutputHelper outputHelper)
     {
-        if (!Directory.Exists(Config.DotNetDirectory))
+        lock (s_lockObj)
         {
-            if (!File.Exists(Config.DotNetTarballPath))
+            if (!Directory.Exists(Config.DotNetDirectory))
             {
-                throw new InvalidOperationException($"Tarball path '{Config.DotNetTarballPath}' specified in {Config.DotNetTarballPathEnv} does not exist.");
-            }
+                if (!File.Exists(Config.DotNetTarballPath))
+                {
+                    throw new InvalidOperationException($"Tarball path '{Config.DotNetTarballPath}' specified in {Config.DotNetTarballPathEnv} does not exist.");
+                }
 
-            Directory.CreateDirectory(Config.DotNetDirectory);
-            ExecuteHelper.ExecuteProcess("tar", $"xzf {Config.DotNetTarballPath} -C {Config.DotNetDirectory}", outputHelper);
+                Directory.CreateDirectory(Config.DotNetDirectory);
+                ExecuteHelper.ExecuteProcessValidateExitCode("tar", $"xzf {Config.DotNetTarballPath} -C {Config.DotNetDirectory}", outputHelper);
+            }
         }
 
-        DotNetInstallDirectory = Path.Combine(Directory.GetCurrentDirectory(), Config.DotNetDirectory);
-        DotNetPath = Path.Combine(DotNetInstallDirectory, "dotnet");
+        DotNetPath = Path.Combine(Config.DotNetDirectory, "dotnet");
     }
 
     public void ExecuteDotNetCmd(string args, ITestOutputHelper outputHelper)

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ExecuteHelper.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using System;
 using System.Diagnostics;
 using System.Linq;
 using System.Text;
@@ -12,7 +13,7 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests;
 internal static class ExecuteHelper
 {
     public static (Process Process, string StdOut, string StdErr) ExecuteProcess(
-        string fileName, string args, ITestOutputHelper outputHelper)
+        string fileName, string args, ITestOutputHelper outputHelper, bool logOutput = false)
     {
         outputHelper.WriteLine($"Executing: {fileName} {args}");
 
@@ -47,17 +48,33 @@ public static (Process Process, string StdOut, string StdErr) ExecuteProcess(
         process.WaitForExit();
 
         string output = stdOutput.ToString().Trim();
-        if (outputHelper != null && !string.IsNullOrWhiteSpace(output))
+        if (logOutput && !string.IsNullOrWhiteSpace(output))
         {
             outputHelper.WriteLine(output);
         }
 
         string error = stdError.ToString().Trim();
-        if (outputHelper != null && !string.IsNullOrWhiteSpace(error))
+        if (logOutput && !string.IsNullOrWhiteSpace(error))
         {
             outputHelper.WriteLine(error);
         }
 
         return (process, output, error);
     }
+
+    public static string ExecuteProcessValidateExitCode(string fileName, string args, ITestOutputHelper outputHelper)
+    {
+        (Process Process, string StdOut, string StdErr) result = ExecuteHelper.ExecuteProcess(fileName, args, outputHelper);
+
+        if (result.Process.ExitCode != 0)
+        {
+            ProcessStartInfo startInfo = result.Process.StartInfo;
+            string msg = $"Failed to execute {startInfo.FileName} {startInfo.Arguments}" +
+                $"{Environment.NewLine}Exit code: {result.Process.ExitCode}" +
+                $"{Environment.NewLine}Standard Error: {result.StdErr}";
+            throw new InvalidOperationException(msg);
+        }
+
+        return result.StdOut;
+    }
 }