From 6ab9ab415ec77ef2da1846b6e008a00cb2a14dcd Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Wed, 27 Aug 2025 10:37:01 -0500 Subject: [PATCH 1/5] Run syft directly to collect SBOM --- eng/common/Dockerfile.syft | 10 ++++ eng/common/Pull-Image.ps1 | 18 +++++++ eng/common/pull-image.sh | 37 -------------- eng/common/templates/jobs/build-images.yml | 50 +++---------------- .../templates/steps/init-docker-linux.yml | 4 +- .../templates/variables/docker-images.yml | 1 + 6 files changed, 38 insertions(+), 82 deletions(-) create mode 100644 eng/common/Dockerfile.syft create mode 100644 eng/common/Pull-Image.ps1 delete mode 100755 eng/common/pull-image.sh diff --git a/eng/common/Dockerfile.syft b/eng/common/Dockerfile.syft new file mode 100644 index 000000000..0d4309cc7 --- /dev/null +++ b/eng/common/Dockerfile.syft @@ -0,0 +1,10 @@ +ARG SYFT_IMAGE_NAME +ARG TARGET_IMAGE_NAME + +FROM ${SYFT_IMAGE_NAME} AS syft +FROM ${TARGET_IMAGE_NAME} AS target + +RUN --mount=from=syft,source=/,target=/syft /syft/syft scan / --select-catalogers image -o spdx-json=/manifest.spdx.json + +FROM scratch +COPY --from=target /manifest.spdx.json /manifest.spdx.json diff --git a/eng/common/Pull-Image.ps1 b/eng/common/Pull-Image.ps1 new file mode 100644 index 000000000..2d0a82cf5 --- /dev/null +++ b/eng/common/Pull-Image.ps1 @@ -0,0 +1,18 @@ +#!/usr/bin/env pwsh + +[cmdletbinding()] +param( + [Parameter(Mandatory = $true, Position = 0)] + [string]$Image, + + [Parameter(Mandatory = $false)] + [int]$Retries = 2, + + [Parameter(Mandatory = $false)] + [int]$WaitFactor = 6 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +& "$PSScriptRoot/Invoke-WithRetry.ps1" "docker pull $Image" -Retries $Retries -WaitFactor $WaitFactor diff --git a/eng/common/pull-image.sh b/eng/common/pull-image.sh deleted file mode 100755 index d6c6ec85b..000000000 --- a/eng/common/pull-image.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -# Stop script on NZEC -set -e -# Stop script if unbound variable found (use ${var:-} if intentional) -set -u - -say_err() { - printf "%b\n" "Error: $1" >&2 -} - -# Executes a command and retries if it fails. -execute() { - local count=0 - until "$@"; do - local exit=$? - count=$(( $count + 1 )) - if [ $count -lt $retries ]; then - local wait=$(( waitFactor ** (( count - 1 )) )) - echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." - sleep $wait - else - say_err "Retry $count/$retries exited $exit, no more retries left." - return $exit - fi - done - - return 0 -} - -scriptName=$0 -retries=5 -waitFactor=6 -image=$1 - -echo "Pulling Docker image $image" -execute docker pull $image diff --git a/eng/common/templates/jobs/build-images.yml b/eng/common/templates/jobs/build-images.yml index e769c6a53..a4a2cb1a0 100644 --- a/eng/common/templates/jobs/build-images.yml +++ b/eng/common/templates/jobs/build-images.yml @@ -100,55 +100,19 @@ jobs: displayName: Publish Image Info File Artifact internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - # The following task depends on the SBOM Manifest Generator task installed on the agent. - # This task is auto-injected by 1ES Pipeline Templates so we don't need to install it ourselves. + - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.dockerClientOS, 'linux')) }}: - powershell: | $images = "$(BuildImages.builtImages)" if (-not $images) { return 0 } - - # There can be leftover versions of the task left on the agent if it's not fresh. So find the latest version. - $taskDir = $(Get-ChildItem -Recurse -Directory -Filter "ManifestGeneratorTask*" -Path '$(Agent.WorkFolder)')[-1].FullName - - # There may be multiple version directories within the task directory. Use the latest. - $taskVersionDir = $(Get-ChildItem -Directory $taskDir | Sort-Object)[-1].FullName - - $manifestToolDllPath = $(Get-ChildItem -Recurse -File -Filter "Microsoft.ManifestTool.dll" -Path $taskVersionDir).FullName - - # Check whether the manifest task installed its own version of .NET. - # To be more robust, we'll handle varying implementations that it's had. - # First check for a dotnet folder in the task location - $dotnetDir = $(Get-ChildItem -Recurse -Directory -Filter "dotnet-*" -Path $taskVersionDir).FullName - if (-not $dotnetDir) { - # If it's not there, check in the agent tools location - $dotnetDir = $(Get-ChildItem -Recurse -Directory -Filter "*dotnet-*" -Path "$(Agent.ToolsDirectory)").FullName - } - - # If the manifest task installed its own version of .NET use that; otherwise it's reusing an existing install of .NET - # which is executable by default. - if ($dotnetDir) { - $dotnetPath = "$dotnetDir/dotnet" - } - else { - $dotnetPath = "dotnet" - } - - # Call the manifest tool for each image to produce seperate SBOMs - # Manifest tool docs: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/secure-supply-chain/custom-sbom-generation-workflows + $syftImageName = "${{ parameters.publishConfig.publicMirrorAcr.server }}/$(imageNames.syft)" + docker pull $syftImageName $images -Split ',' | ForEach-Object { echo "Generating SBOM for $_"; - $formattedImageName = $_.Replace('${{ parameters.publishConfig.buildAcr.server }}/${{ parameters.publishConfig.buildAcr.repoPrefix }}', "").Replace('/', '_').Replace(':', '_'); + $targetImageName = "$_" + $formattedImageName = $targetImageName.Replace('${{ parameters.publishConfig.buildAcr.server }}/${{ parameters.publishConfig.buildAcr.repoPrefix }}', "").Replace('/', '_').Replace(':', '_'); $sbomChildDir = "$(sbomDirectory)/$formattedImageName"; New-Item -Type Directory -Path $sbomChildDir > $null; - & $dotnetPath "$manifestToolDllPath" ` - Generate ` - -BuildDropPath '$(Build.ArtifactStagingDirectory)' ` - -BuildComponentPath '$(Agent.BuildDirectory)' ` - -PackageName '.NET' ` - -PackageVersion '$(Build.BuildNumber)' ` - -ManifestDirPath $sbomChildDir ` - -DockerImagesToScan $_ ` - -Verbosity Information + docker build --output=$sbomChildDir -f $(engCommonPath)/Dockerfile.syft --build-arg SYFT_IMAGE_NAME=$syftImageName --build-arg TARGET_IMAGE_NAME=$targetImageName -t syft-sbom $(engCommonPath) } displayName: Generate SBOMs condition: and(succeeded(), ne(variables['BuildImages.builtImages'], '')) @@ -156,7 +120,7 @@ jobs: - template: /eng/common/templates/jobs/${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}@self parameters: condition: ne(variables.testScriptPath, '') - - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.dockerClientOS, 'linux')) }}: - template: /eng/common/templates/steps/publish-artifact.yml@self parameters: path: $(sbomDirectory) diff --git a/eng/common/templates/steps/init-docker-linux.yml b/eng/common/templates/steps/init-docker-linux.yml index 729546a38..8c554ad9b 100644 --- a/eng/common/templates/steps/init-docker-linux.yml +++ b/eng/common/templates/steps/init-docker-linux.yml @@ -25,7 +25,7 @@ steps: ################################################################################ - ${{ if eq(parameters.setupImageBuilder, 'true') }}: - - script: $(engCommonPath)/pull-image.sh $(imageNames.imageBuilder) + - powershell: $(engCommonPath)/Pull-Image.ps1 $(imageNames.imageBuilder) displayName: Pull Image Builder condition: and(succeeded(), ${{ parameters.condition }}) @@ -78,7 +78,7 @@ steps: # Setup Test Runner (Optional) ################################################################################ - ${{ if eq(parameters.setupTestRunner, 'true') }}: - - script: $(engCommonPath)/pull-image.sh $(imageNames.testrunner) + - powershell: $(engCommonPath)/Pull-Image.ps1 $(imageNames.testrunner) displayName: Pull Test Runner condition: and(succeeded(), ${{ parameters.condition }}) - script: > diff --git a/eng/common/templates/variables/docker-images.yml b/eng/common/templates/variables/docker-images.yml index 0d25f8821..10c7e5183 100644 --- a/eng/common/templates/variables/docker-images.yml +++ b/eng/common/templates/variables/docker-images.yml @@ -4,3 +4,4 @@ variables: imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner imageNames.testRunner.withrepo: testrunner-withrepo:$(Build.BuildId)-$(System.JobId) + imageNames.syft: anchore/syft:v1.26.1 From d61a6d18356611932bbe0ee06e4a095370ca1633 Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Thu, 28 Aug 2025 10:37:23 -0500 Subject: [PATCH 2/5] use script to pull image --- eng/common/templates/jobs/build-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/templates/jobs/build-images.yml b/eng/common/templates/jobs/build-images.yml index a4a2cb1a0..59fefdfe6 100644 --- a/eng/common/templates/jobs/build-images.yml +++ b/eng/common/templates/jobs/build-images.yml @@ -105,7 +105,7 @@ jobs: $images = "$(BuildImages.builtImages)" if (-not $images) { return 0 } $syftImageName = "${{ parameters.publishConfig.publicMirrorAcr.server }}/$(imageNames.syft)" - docker pull $syftImageName + & $(engCommonPath)/Pull-Image.ps1 $syftImageName $images -Split ',' | ForEach-Object { echo "Generating SBOM for $_"; $targetImageName = "$_" From 865cb637143a0873d042e6c94faebace726ded89 Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Thu, 28 Aug 2025 14:14:59 -0500 Subject: [PATCH 3/5] update image tag --- eng/common/templates/variables/docker-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/templates/variables/docker-images.yml b/eng/common/templates/variables/docker-images.yml index 10c7e5183..d630111a7 100644 --- a/eng/common/templates/variables/docker-images.yml +++ b/eng/common/templates/variables/docker-images.yml @@ -4,4 +4,4 @@ variables: imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner imageNames.testRunner.withrepo: testrunner-withrepo:$(Build.BuildId)-$(System.JobId) - imageNames.syft: anchore/syft:v1.26.1 + imageNames.syft: anchore/syft:v1.31.0-debug From 8fb672e2a7892ea0ef96780119038f2341aa0331 Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Thu, 28 Aug 2025 14:15:30 -0500 Subject: [PATCH 4/5] update dockerfile to handle distroless --- eng/common/Dockerfile.syft | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/eng/common/Dockerfile.syft b/eng/common/Dockerfile.syft index 0d4309cc7..2e564e2ae 100644 --- a/eng/common/Dockerfile.syft +++ b/eng/common/Dockerfile.syft @@ -2,9 +2,15 @@ ARG SYFT_IMAGE_NAME ARG TARGET_IMAGE_NAME FROM ${SYFT_IMAGE_NAME} AS syft -FROM ${TARGET_IMAGE_NAME} AS target +FROM ${TARGET_IMAGE_NAME} AS scan-image -RUN --mount=from=syft,source=/,target=/syft /syft/syft scan / --select-catalogers image -o spdx-json=/manifest.spdx.json +FROM syft AS run-scan +ARG TARGET_IMAGE_NAME +ENV SYFT_CHECK_FOR_APP_UPDATE=0 \ + SYFT_SOURCE_NAME=${TARGET_IMAGE_NAME} +USER root +RUN --mount=from=scan-image,source=/,target=/rootfs \ + ["/syft", "scan", "/rootfs/", "--select-catalogers", "image", "--output", "spdx-json=/manifest.spdx.json"] -FROM scratch -COPY --from=target /manifest.spdx.json /manifest.spdx.json +FROM scratch AS output +COPY --from=run-scan /manifest.spdx.json /manifest.spdx.json From 90ce77d1e7000bd8bb6dc5cd3405b03136ed79ce Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Thu, 28 Aug 2025 14:16:40 -0500 Subject: [PATCH 5/5] PowerShell syntax --- eng/common/templates/jobs/build-images.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/common/templates/jobs/build-images.yml b/eng/common/templates/jobs/build-images.yml index 59fefdfe6..65d9d75fe 100644 --- a/eng/common/templates/jobs/build-images.yml +++ b/eng/common/templates/jobs/build-images.yml @@ -108,11 +108,11 @@ jobs: & $(engCommonPath)/Pull-Image.ps1 $syftImageName $images -Split ',' | ForEach-Object { echo "Generating SBOM for $_"; - $targetImageName = "$_" + $targetImageName = "$_"; $formattedImageName = $targetImageName.Replace('${{ parameters.publishConfig.buildAcr.server }}/${{ parameters.publishConfig.buildAcr.repoPrefix }}', "").Replace('/', '_').Replace(':', '_'); $sbomChildDir = "$(sbomDirectory)/$formattedImageName"; New-Item -Type Directory -Path $sbomChildDir > $null; - docker build --output=$sbomChildDir -f $(engCommonPath)/Dockerfile.syft --build-arg SYFT_IMAGE_NAME=$syftImageName --build-arg TARGET_IMAGE_NAME=$targetImageName -t syft-sbom $(engCommonPath) + docker build --output=$sbomChildDir -f $(engCommonPath)/Dockerfile.syft --build-arg SYFT_IMAGE_NAME=$syftImageName --build-arg TARGET_IMAGE_NAME=$targetImageName -t syft-sbom $(engCommonPath); } displayName: Generate SBOMs condition: and(succeeded(), ne(variables['BuildImages.builtImages'], ''))