From 1356ec787366e047121362d1d6b7d95974cded99 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Thu, 22 Sep 2022 17:06:00 +0800 Subject: [PATCH 01/26] Support RPM --- azure-pipelines-arm64.yml | 311 ++++++++++++++++++++++++ azure-pipelines.yml | 14 +- scripts/release/debian/build.sh | 2 +- scripts/release/debian/prepare.sh | 2 +- scripts/release/rpm/azure-cli.spec | 1 - scripts/release/rpm/fedora.dockerfile | 3 +- scripts/release/rpm/mariner.dockerfile | 5 +- scripts/release/rpm/pipeline.sh | 2 +- scripts/release/rpm/pipeline_mariner.sh | 2 +- scripts/release/rpm/ubi.dockerfile | 3 +- 10 files changed, 329 insertions(+), 16 deletions(-) create mode 100644 azure-pipelines-arm64.yml diff --git a/azure-pipelines-arm64.yml b/azure-pipelines-arm64.yml new file mode 100644 index 00000000000..a0e81ad9d3c --- /dev/null +++ b/azure-pipelines-arm64.yml @@ -0,0 +1,311 @@ +resources: +- repo: self + +trigger: + batch: true + branches: + include: + - '*' + +jobs: +- job: ExtractMetadata + displayName: Extract Metadata + + condition: succeeded() + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: Bash@3 + displayName: 'Extract Version' + inputs: + targetType: 'filePath' + filePath: scripts/release/get_version.sh + + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: metadata' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: metadata + +- job: BuildRpmPackageMarinerArm64 + displayName: Build Rpm Package Mariner ARM64 + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: 'ubuntu-arm64-2004-pool' + vmImage: 'ubuntu-20.04' + variables: + # Disable Component Detection on ARM64 + skipComponentGovernanceDetection: true + strategy: + matrix: + 2.0: + image: mcr.microsoft.com/cbl-mariner/base/core:2.0 + artifact: rpm-mariner2.0 + steps: + - bash: | + # https://docs.docker.com/engine/security/rootless/ + /bin/bash -c "$(curl -fsSL https://get.docker.com)" + sudo apt-get install -y uidmap + dockerd-rootless-setuptool.sh install + export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run + PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & + sleep 5 + docker context use rootless + displayName: Install Docker + - task: Bash@3 + displayName: 'Build Rpm Package: Mariner' + inputs: + targetType: 'filePath' + filePath: scripts/release/rpm/pipeline_mariner.sh + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: rpm-mariner' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(artifact) + +# TODO: rpmbuild on Red Hat UBI 8 is slow for unknown reason. Still working with Red Hat to investigate. +- job: BuildRpmPackagesArm64 + displayName: Build Rpm Packages ARM64 + # Do not run this job for Pull Requests due to the slowness + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: 'ubuntu-arm64-2004-pool' + vmImage: 'ubuntu-20.04' + variables: + skipComponentGovernanceDetection: true + strategy: + matrix: + Red Hat Universal Base Image 8: + dockerfile: ubi + image: registry.access.redhat.com/ubi8/ubi:8.4 + artifact: rpm-ubi8 + python_package: python39 + Red Hat Universal Base Image 9: + dockerfile: ubi + image: registry.access.redhat.com/ubi9/ubi:9.0.0 + artifact: rpm-ubi9 + python_package: python3.9 + Fedora 35: + dockerfile: fedora + image: fedora:35 + artifact: rpm-fedora35 + python_package: python3 + Fedora 36: + dockerfile: fedora + image: fedora:36 + artifact: rpm-fedora36 + python_package: python3 + steps: + - bash: | + # https://docs.docker.com/engine/security/rootless/ + /bin/bash -c "$(curl -fsSL https://get.docker.com)" + sudo apt-get install -y uidmap + dockerd-rootless-setuptool.sh install + export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run + PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & + sleep 5 + docker context use rootless + displayName: Install Docker + - task: Bash@3 + displayName: 'Build Rpm Package' + inputs: + targetType: 'filePath' + filePath: scripts/release/rpm/pipeline.sh + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: rpm' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(artifact) + + +- job: TestRpmPackageArm64 + displayName: Test Rpm Package ARM64 + timeoutInMinutes: 120 + dependsOn: + - BuildRpmPackagesArm64 + - ExtractMetadata + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: 'ubuntu-arm64-2004-pool' + vmImage: 'ubuntu-20.04' + variables: + skipComponentGovernanceDetection: true + strategy: + matrix: + Red Hat Universal Base Image 8: + artifact: rpm-ubi8 + distro: el8 + image: registry.access.redhat.com/ubi8/ubi:8.4 + python_package: python39 + python_cmd: python3.9 + pip_cmd: pip3.9 + Red Hat Universal Base Image 9: + artifact: rpm-ubi9 + distro: el9 + image: registry.access.redhat.com/ubi9/ubi:9.0.0 + python_package: python3.9 + python_cmd: python3.9 + pip_cmd: pip3.9 + Fedora 35: + artifact: rpm-fedora35 + distro: fc35 + image: fedora:35 + python_package: python3 + python_cmd: python3 + pip_cmd: pip3 + Fedora 36: + artifact: rpm-fedora36 + distro: fc36 + image: fedora:36 + python_package: python3 + python_cmd: python3 + pip_cmd: pip3 + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' + artifactName: $(artifact) + + - bash: | + # https://docs.docker.com/engine/security/rootless/ + /bin/bash -c "$(curl -fsSL https://get.docker.com)" + sudo apt-get install -y uidmap + dockerd-rootless-setuptool.sh install + export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run + PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & + sleep 5 + docker context use rootless + displayName: Install Docker + + - bash: | + set -ex + + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + RPM_NAME=$(cd $SYSTEM_ARTIFACTSDIRECTORY/rpm/ && ls azure-cli-$CLI_VERSION-1.${DISTRO}.*.rpm) + + echo "== Test rpm package on ${IMAGE} ==" + docker pull $IMAGE + docker run --rm -e RPM_NAME=$RPM_NAME -e PYTHON_PACKAGE=${PYTHON_PACKAGE} -e PYTHON_CMD=${PYTHON_CMD} -e PIP_CMD=${PIP_CMD} -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh" + + displayName: 'Test Rpm Package' + +#- job: BuildWindowsMSIArm64 +# displayName: Build Windows MSI ARM64 +# +# dependsOn: ExtractMetadata +# condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) +# pool: +# name: 'windows-arm64-11-pool' +# vmImage: 'windows-2022' +# variables: +# skipComponentGovernanceDetection: true +# steps: +# - task: DownloadPipelineArtifact@1 +# displayName: 'Download Build Artifacts' +# inputs: +# TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' +# artifactName: metadata +# +# - script: | +# set /p CLI_VERSION=<$(System.ArtifactsDirectory)/metadata/version +# set +# +# build_scripts/windows/scripts/build.cmd +# displayName: 'Build Windows MSI' +# +# - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 +# displayName: 'SBOM' +# inputs: +# BuildDropPath: 'build_scripts/windows/out/' +# +# - task: PublishPipelineArtifact@0 +# displayName: 'Publish Artifact: MSI' +# inputs: +# TargetPath: 'build_scripts/windows/out/' +# ArtifactName: msi +# +#- job: TestWindowsMSIArm64 +# displayName: Test Windows MSI ARM64 +# +# dependsOn: BuildWindowsMSIArm64 +# condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) +# pool: +# name: 'windows-arm64-11-pool' +# vmImage: 'windows-2022' +# variables: +# skipComponentGovernanceDetection: true +# steps: +# - task: DownloadPipelineArtifact@1 +# displayName: 'Download Build Artifacts' +# inputs: +# TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' +# artifactName: metadata +# +# - task: DownloadPipelineArtifact@1 +# displayName: 'Download Build Artifacts' +# inputs: +# TargetPath: '$(Build.ArtifactStagingDirectory)/msi' +# artifactName: msi +# +# - task: PowerShell@2 +# displayName: Install and Load CLI +# inputs: +# targetType: inline +# script: | +# if (-NOT ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")) { +# # Start another Powershell process as Admin and execute this script again +# $arguments = "& '" +$myinvocation.mycommand.definition + "'" +# Start-Process powershell -Verb runAs -ArgumentList $arguments +# # Stop if the PowerShell is not run as Admin +# Break +# } +# # The following are executed by elevated PowerShell +# az --version +# +# $InstallArgs = @( +# "/i" +# "`"$env:SYSTEM_ARTIFACTSDIRECTORY\msi\Microsoft Azure CLI.msi`"" +# "/q" +# "/norestart" +# "/l*v" +# ".\install_logs.txt" +# ) +# $pre_installed_version=az version --query '\"azure-cli\"' -o tsv +# $to_be_installed_version=Get-Content $(System.ArtifactsDirectory)/metadata/version +# if ($pre_installed_version -eq $to_be_installed_version){ +# # See https://docs.microsoft.com/windows/win32/msi/reinstallmode about options of REINSTALLMODE +# $reinstall_option="REINSTALL=ALL REINSTALLMODE=emus" +# $InstallArgs += $reinstall_option +# } +# Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow +# $install_time=Measure-Command {Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow} | select -expand TotalSeconds +# $installed_version=az version --query '\"azure-cli\"' -o tsv +# if ($installed_version -ne $to_be_installed_version){ +# echo "The MSI failed to install." +# Exit 1 +# } +# echo 'Install time(seconds):' $install_time +# az --version +# # Test bundled pip with extension installation +# az extension add -n rdbms-connect +# az self-test +# +# Get-Content .\install_logs.txt diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e45e9682a66..5f5e5c9b93f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -643,12 +643,12 @@ jobs: image: registry.access.redhat.com/ubi9/ubi:9.0.0 artifact: rpm-ubi9 python_package: python3.9 - Fedora35: + Fedora 35: dockerfile: fedora image: fedora:35 artifact: rpm-fedora35 python_package: python3 - Fedora36: + Fedora 36: dockerfile: fedora image: fedora:36 artifact: rpm-fedora36 @@ -673,7 +673,9 @@ jobs: - job: TestRpmPackage displayName: Test Rpm Package timeoutInMinutes: 120 - dependsOn: BuildRpmPackages + dependsOn: + - BuildRpmPackages + - ExtractMetadata condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) pool: name: ${{ variables.ubuntu_pool }} @@ -693,14 +695,14 @@ jobs: python_package: python3.9 python_cmd: python3.9 pip_cmd: pip3.9 - Fedora35: + Fedora 35: artifact: rpm-fedora35 distro: fc35 image: fedora:35 python_package: python3 python_cmd: python3 pip_cmd: pip3 - Fedora36: + Fedora 36: artifact: rpm-fedora36 distro: fc36 image: fedora:36 @@ -714,14 +716,12 @@ jobs: TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' artifactName: metadata - - task: DownloadPipelineArtifact@1 displayName: 'Download Build Artifacts' inputs: TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' artifactName: $(artifact) - - bash: | set -ex diff --git a/scripts/release/debian/build.sh b/scripts/release/debian/build.sh index 0c344bb0c54..28185c3fb10 100755 --- a/scripts/release/debian/build.sh +++ b/scripts/release/debian/build.sh @@ -51,5 +51,5 @@ $SCRIPT_DIR/prepare.sh $WORKDIR/debian $WORKDIR/az.completion $WORKDIR cd $WORKDIR dpkg-buildpackage -us -uc -deb_file=$WORKDIR/../azure-cli_${CLI_VERSION}-${CLI_VERSION_REVISION:=1}_all.deb +deb_file=$WORKDIR/../azure-cli_${CLI_VERSION}-${CLI_VERSION_REVISION:=1}_*.deb cp $deb_file /mnt/output/ diff --git a/scripts/release/debian/prepare.sh b/scripts/release/debian/prepare.sh index 173aa02ba22..d0da3b02f29 100755 --- a/scripts/release/debian/prepare.sh +++ b/scripts/release/debian/prepare.sh @@ -57,7 +57,7 @@ Standards-Version: 3.9.5 Homepage: https://github.com/azure/azure-cli Package: azure-cli -Architecture: all +Architecture: any Depends: \${shlibs:Depends}, \${misc:Depends} Description: Azure CLI A great cloud needs great tools; we're excited to introduce Azure CLI, diff --git a/scripts/release/rpm/azure-cli.spec b/scripts/release/rpm/azure-cli.spec index 040618f1cf2..da46cdfd10e 100644 --- a/scripts/release/rpm/azure-cli.spec +++ b/scripts/release/rpm/azure-cli.spec @@ -27,7 +27,6 @@ Name: %{name} Version: %{version} Release: %{release} Url: https://docs.microsoft.com/cli/azure/install-azure-cli -BuildArch: x86_64 Requires: %{python_package} Prefix: /usr Prefix: /etc diff --git a/scripts/release/rpm/fedora.dockerfile b/scripts/release/rpm/fedora.dockerfile index ea8b6c45cae..9a6dd4ef80b 100644 --- a/scripts/release/rpm/fedora.dockerfile +++ b/scripts/release/rpm/fedora.dockerfile @@ -14,7 +14,8 @@ COPY . . RUN dos2unix ./scripts/release/rpm/azure-cli.spec && \ REPO_PATH=$(pwd) CLI_VERSION=$cli_version PYTHON_PACKAGE=$python_package PYTHON_CMD=python3 \ rpmbuild -v -bb --clean scripts/release/rpm/azure-cli.spec && \ - cp /root/rpmbuild/RPMS/x86_64/azure-cli-${cli_version}-1.*.x86_64.rpm /azure-cli-dev.rpm + cp /root/rpmbuild/RPMS/**/azure-cli-${cli_version}-1.*.rpm /azure-cli-dev.rpm && \ + mkdir /out && cp /root/rpmbuild/RPMS/**/azure-cli-${cli_version}-1.*.rpm /out/ FROM ${image} AS execution-env diff --git a/scripts/release/rpm/mariner.dockerfile b/scripts/release/rpm/mariner.dockerfile index d31c5c15eb8..02f6201d5a6 100644 --- a/scripts/release/rpm/mariner.dockerfile +++ b/scripts/release/rpm/mariner.dockerfile @@ -4,7 +4,7 @@ FROM ${image} AS build-env ARG cli_version=dev RUN tdnf update -y -RUN tdnf install -y binutils file rpm-build gcc libffi-devel python3-devel openssl-devel make diffutils patch dos2unix perl sed +RUN tdnf install -y binutils file rpm-build gcc libffi-devel python3-devel openssl-devel make diffutils patch dos2unix perl sed kernel-headers glibc-devel binutils WORKDIR /azure-cli @@ -16,7 +16,8 @@ COPY . . RUN dos2unix ./scripts/release/rpm/azure-cli.spec && \ REPO_PATH=$(pwd) CLI_VERSION=$cli_version PYTHON_PACKAGE=python3 PYTHON_CMD=python3 \ rpmbuild -v -bb --clean scripts/release/rpm/azure-cli.spec && \ - cp /usr/src/mariner/RPMS/x86_64/azure-cli-${cli_version}-1.x86_64.rpm /azure-cli-dev.rpm + cp /usr/src/mariner/RPMS/**/azure-cli-${cli_version}-1.*.rpm /azure-cli-dev.rpm && \ + mkdir /out && cp /usr/src/mariner/RPMS/**/azure-cli-${cli_version}-1.*.rpm /out/ FROM ${image} AS execution-env diff --git a/scripts/release/rpm/pipeline.sh b/scripts/release/rpm/pipeline.sh index 3f9874fbce2..52c99a6193c 100755 --- a/scripts/release/rpm/pipeline.sh +++ b/scripts/release/rpm/pipeline.sh @@ -39,7 +39,7 @@ docker build \ id=$(docker create azure/azure-cli:${DOCKERFILE}-builder) # https://docs.docker.com/engine/reference/commandline/cp/ # Append /. so that the x86_64 folder's content is copied, instead of x86_64 folder itself. -docker cp $id:/root/rpmbuild/RPMS/x86_64/. ${BUILD_STAGINGDIRECTORY} +docker cp $id:/out/. ${BUILD_STAGINGDIRECTORY} # Save these too a staging directory so that later build phases can choose to save them as Artifacts or publish them to # a registry. diff --git a/scripts/release/rpm/pipeline_mariner.sh b/scripts/release/rpm/pipeline_mariner.sh index be6f59c5369..c7274a3f5cb 100644 --- a/scripts/release/rpm/pipeline_mariner.sh +++ b/scripts/release/rpm/pipeline_mariner.sh @@ -33,7 +33,7 @@ docker build \ id=$(docker create azure/azure-cli:mariner-builder bash) # https://docs.docker.com/engine/reference/commandline/cp/ # Append /. so that the x86_64 folder's content is copied, instead of x86_64 folder itself. -docker cp $id:/usr/src/mariner/RPMS/x86_64/. ${BUILD_STAGINGDIRECTORY} +docker cp $id:/out/. ${BUILD_STAGINGDIRECTORY} # Save these too a staging directory so that later build phases can choose to save them as Artifacts or publish them to # a registry. diff --git a/scripts/release/rpm/ubi.dockerfile b/scripts/release/rpm/ubi.dockerfile index b1ecef72fad..e3c0a857123 100644 --- a/scripts/release/rpm/ubi.dockerfile +++ b/scripts/release/rpm/ubi.dockerfile @@ -19,7 +19,8 @@ COPY . . RUN dos2unix ./scripts/release/rpm/azure-cli.spec && \ REPO_PATH=$(pwd) CLI_VERSION=$cli_version PYTHON_PACKAGE=$python_package PYTHON_CMD=python3.9 \ rpmbuild -v -bb --clean scripts/release/rpm/azure-cli.spec && \ - cp /root/rpmbuild/RPMS/x86_64/azure-cli-${cli_version}-1.*.x86_64.rpm /azure-cli-dev.rpm + cp /root/rpmbuild/RPMS/**/azure-cli-${cli_version}-1.*.rpm /azure-cli-dev.rpm && \ + mkdir /out && cp /root/rpmbuild/RPMS/**/azure-cli-${cli_version}-1.*.rpm /out/ FROM ${image} AS execution-env From 0dccf4fc9708f02d3796e2c3f0eedd18522d0931 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Tue, 11 Oct 2022 11:28:59 +0800 Subject: [PATCH 02/26] Support DEB --- azure-pipelines-arm64.yml | 233 ++++++++++++++----------- scripts/release/debian/build.sh | 13 +- scripts/release/debian/prepare.sh | 3 +- scripts/release/rpm/mariner.dockerfile | 1 + 4 files changed, 142 insertions(+), 108 deletions(-) diff --git a/azure-pipelines-arm64.yml b/azure-pipelines-arm64.yml index a0e81ad9d3c..b6bf00c2bfc 100644 --- a/azure-pipelines-arm64.yml +++ b/azure-pipelines-arm64.yml @@ -197,115 +197,138 @@ jobs: - bash: | set -ex - + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` RPM_NAME=$(cd $SYSTEM_ARTIFACTSDIRECTORY/rpm/ && ls azure-cli-$CLI_VERSION-1.${DISTRO}.*.rpm) - + echo "== Test rpm package on ${IMAGE} ==" docker pull $IMAGE docker run --rm -e RPM_NAME=$RPM_NAME -e PYTHON_PACKAGE=${PYTHON_PACKAGE} -e PYTHON_CMD=${PYTHON_CMD} -e PIP_CMD=${PIP_CMD} -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh" - displayName: 'Test Rpm Package' -#- job: BuildWindowsMSIArm64 -# displayName: Build Windows MSI ARM64 -# -# dependsOn: ExtractMetadata -# condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) -# pool: -# name: 'windows-arm64-11-pool' -# vmImage: 'windows-2022' -# variables: -# skipComponentGovernanceDetection: true -# steps: -# - task: DownloadPipelineArtifact@1 -# displayName: 'Download Build Artifacts' -# inputs: -# TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' -# artifactName: metadata -# -# - script: | -# set /p CLI_VERSION=<$(System.ArtifactsDirectory)/metadata/version -# set -# -# build_scripts/windows/scripts/build.cmd -# displayName: 'Build Windows MSI' -# -# - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 -# displayName: 'SBOM' -# inputs: -# BuildDropPath: 'build_scripts/windows/out/' -# -# - task: PublishPipelineArtifact@0 -# displayName: 'Publish Artifact: MSI' -# inputs: -# TargetPath: 'build_scripts/windows/out/' -# ArtifactName: msi -# -#- job: TestWindowsMSIArm64 -# displayName: Test Windows MSI ARM64 -# -# dependsOn: BuildWindowsMSIArm64 -# condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) -# pool: -# name: 'windows-arm64-11-pool' -# vmImage: 'windows-2022' -# variables: -# skipComponentGovernanceDetection: true -# steps: -# - task: DownloadPipelineArtifact@1 -# displayName: 'Download Build Artifacts' -# inputs: -# TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' -# artifactName: metadata -# -# - task: DownloadPipelineArtifact@1 -# displayName: 'Download Build Artifacts' -# inputs: -# TargetPath: '$(Build.ArtifactStagingDirectory)/msi' -# artifactName: msi -# -# - task: PowerShell@2 -# displayName: Install and Load CLI -# inputs: -# targetType: inline -# script: | -# if (-NOT ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")) { -# # Start another Powershell process as Admin and execute this script again -# $arguments = "& '" +$myinvocation.mycommand.definition + "'" -# Start-Process powershell -Verb runAs -ArgumentList $arguments -# # Stop if the PowerShell is not run as Admin -# Break -# } -# # The following are executed by elevated PowerShell -# az --version -# -# $InstallArgs = @( -# "/i" -# "`"$env:SYSTEM_ARTIFACTSDIRECTORY\msi\Microsoft Azure CLI.msi`"" -# "/q" -# "/norestart" -# "/l*v" -# ".\install_logs.txt" -# ) -# $pre_installed_version=az version --query '\"azure-cli\"' -o tsv -# $to_be_installed_version=Get-Content $(System.ArtifactsDirectory)/metadata/version -# if ($pre_installed_version -eq $to_be_installed_version){ -# # See https://docs.microsoft.com/windows/win32/msi/reinstallmode about options of REINSTALLMODE -# $reinstall_option="REINSTALL=ALL REINSTALLMODE=emus" -# $InstallArgs += $reinstall_option -# } -# Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow -# $install_time=Measure-Command {Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow} | select -expand TotalSeconds -# $installed_version=az version --query '\"azure-cli\"' -o tsv -# if ($installed_version -ne $to_be_installed_version){ -# echo "The MSI failed to install." -# Exit 1 -# } -# echo 'Install time(seconds):' $install_time -# az --version -# # Test bundled pip with extension installation -# az extension add -n rdbms-connect -# az self-test -# -# Get-Content .\install_logs.txt +- job: BuildDebPackagesArm64 + displayName: Build Deb Packages ARM64 + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: 'ubuntu-arm64-2004-pool' + vmImage: 'ubuntu-20.04' + variables: + skipComponentGovernanceDetection: true + strategy: + matrix: + # https://wiki.ubuntu.com/Releases + Bionic: + # 18.04 + deb_system: ubuntu + distro: bionic + Focal: + # 20.04 + deb_system: ubuntu + distro: focal + Jammy: + # 22.04 + deb_system: ubuntu + distro: jammy + + # https://wiki.debian.org/DebianReleases + Buster: + # 10 + deb_system: debian + distro: buster + Bullseye: + # 11 + deb_system: debian + distro: bullseye + steps: + - bash: | + # https://docs.docker.com/engine/security/rootless/ + /bin/bash -c "$(curl -fsSL https://get.docker.com)" + sudo apt-get install -y uidmap + dockerd-rootless-setuptool.sh install + export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run + PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & + sleep 5 + docker context use rootless + displayName: Install Docker + - task: Bash@3 + displayName: 'Build $(deb_system) $(distro) Package' + inputs: + targetType: 'filePath' + filePath: scripts/release/debian/pipeline.sh + env: + DISTRO: $(distro) + DISTRO_BASE_IMAGE: $(deb_system):$(distro) + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: debian' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(deb_system)-$(distro) + +- job: TestDebPackagesArm64 + displayName: Test Deb Packages ARM64 + dependsOn: + - BuildDebPackagesArm64 + - ExtractMetadata + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + strategy: + matrix: + Focal: + deb_system: ubuntu + distro: focal + Jammy: + deb_system: ubuntu + distro: jammy + Buster: + deb_system: debian + distro: buster + Bullseye: + deb_system: debian + distro: bullseye + pool: + name: 'ubuntu-arm64-2004-pool' + vmImage: 'ubuntu-20.04' + variables: + skipComponentGovernanceDetection: true + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download $(deb_system):$(distro) Build' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/debian' + artifactName: $(deb_system)-$(distro) + + - bash: | + # https://docs.docker.com/engine/security/rootless/ + /bin/bash -c "$(curl -fsSL https://get.docker.com)" + sudo apt-get install -y uidmap + dockerd-rootless-setuptool.sh install + export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run + PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & + sleep 5 + docker context use rootless + displayName: Install Docker + + - task: Bash@3 + displayName: 'Test $(deb_system) $(distro) Package' + env: + DISTRO: $(distro) + DISTRO_BASE_IMAGE: $(deb_system):$(distro) + inputs: + targetType: 'inline' + script: | + set -exv + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + echo "== Test debian package on ${DISTRO} ==" + docker pull ${DISTRO_BASE_IMAGE} + docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" diff --git a/scripts/release/debian/build.sh b/scripts/release/debian/build.sh index 28185c3fb10..eafde994f01 100755 --- a/scripts/release/debian/build.sh +++ b/scripts/release/debian/build.sh @@ -21,8 +21,17 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # Update APT packages apt-get update # uuid-dev is used to build _uuid module: https://github.com/python/cpython/pull/3796 -apt-get install -y libssl-dev libffi-dev python3-dev debhelper zlib1g-dev uuid-dev -apt-get install -y wget +apt-get install -y libssl-dev libffi-dev python3-dev zlib1g-dev uuid-dev wget + +# In Ubuntu 18.04, debhelper 11.1.6 has bug which makes it fail to dpkg-buildpackage. Use backport version instead. +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=897569 +if cat /etc/lsb-release | grep 18.04 +then + apt-get install -y debhelper -t bionic-backports +else + apt-get install -y debhelper +fi + # Git is not strictly necessary, but it would allow building an experimental package # with dependency which is currently only available in its git repo feature branch. apt-get install -y git diff --git a/scripts/release/debian/prepare.sh b/scripts/release/debian/prepare.sh index d0da3b02f29..506b5896236 100755 --- a/scripts/release/debian/prepare.sh +++ b/scripts/release/debian/prepare.sh @@ -110,7 +110,8 @@ ${TAB}echo "\043!/usr/bin/env bash\nbin_dir=\140cd \"\044(dirname \"\044BASH_SOU ${TAB}chmod 0755 debian/azure-cli/usr/bin/az ${TAB}mkdir -p debian/azure-cli/etc/bash_completion.d/ ${TAB}cat ${completion_script} > debian/azure-cli/etc/bash_completion.d/azure-cli -${TAB}dpkg-shlibdeps -v --warnings=7 -Tdebian/azure-cli.substvars -dDepends -edebian/azure-cli/opt/az/bin/python3 debian/azure-cli/opt/az/lib/python3.10/lib-dynload/_ssl.cpython-310-x86_64-linux-gnu.so +${TAB}ssl_file=$(find debian/azure-cli/opt/az/lib/python3.10/ -type f -name '_ssl.cpython-310-*-linux-gnu.so') +${TAB}dpkg-shlibdeps -v --warnings=7 -Tdebian/azure-cli.substvars -dDepends -edebian/azure-cli/opt/az/bin/python3 $ssl_file override_dh_strip: diff --git a/scripts/release/rpm/mariner.dockerfile b/scripts/release/rpm/mariner.dockerfile index 02f6201d5a6..97dc2576366 100644 --- a/scripts/release/rpm/mariner.dockerfile +++ b/scripts/release/rpm/mariner.dockerfile @@ -4,6 +4,7 @@ FROM ${image} AS build-env ARG cli_version=dev RUN tdnf update -y +# kernel-headers, glibc-devel, binutils are needed to install psutil python package on ARM64 RUN tdnf install -y binutils file rpm-build gcc libffi-devel python3-devel openssl-devel make diffutils patch dos2unix perl sed kernel-headers glibc-devel binutils WORKDIR /azure-cli From cd91cdeb229a2e0b9d6b09624f380f59a72e5242 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Thu, 13 Oct 2022 15:03:02 +0800 Subject: [PATCH 03/26] Merge yml --- azure-pipelines-arm64.yml | 334 -------------------------------------- azure-pipelines.yml | 302 ++++++++++++++++++++++++++++++++++ 2 files changed, 302 insertions(+), 334 deletions(-) delete mode 100644 azure-pipelines-arm64.yml diff --git a/azure-pipelines-arm64.yml b/azure-pipelines-arm64.yml deleted file mode 100644 index b6bf00c2bfc..00000000000 --- a/azure-pipelines-arm64.yml +++ /dev/null @@ -1,334 +0,0 @@ -resources: -- repo: self - -trigger: - batch: true - branches: - include: - - '*' - -jobs: -- job: ExtractMetadata - displayName: Extract Metadata - - condition: succeeded() - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: Bash@3 - displayName: 'Extract Version' - inputs: - targetType: 'filePath' - filePath: scripts/release/get_version.sh - - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: metadata' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: metadata - -- job: BuildRpmPackageMarinerArm64 - displayName: Build Rpm Package Mariner ARM64 - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: 'ubuntu-arm64-2004-pool' - vmImage: 'ubuntu-20.04' - variables: - # Disable Component Detection on ARM64 - skipComponentGovernanceDetection: true - strategy: - matrix: - 2.0: - image: mcr.microsoft.com/cbl-mariner/base/core:2.0 - artifact: rpm-mariner2.0 - steps: - - bash: | - # https://docs.docker.com/engine/security/rootless/ - /bin/bash -c "$(curl -fsSL https://get.docker.com)" - sudo apt-get install -y uidmap - dockerd-rootless-setuptool.sh install - export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run - PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & - sleep 5 - docker context use rootless - displayName: Install Docker - - task: Bash@3 - displayName: 'Build Rpm Package: Mariner' - inputs: - targetType: 'filePath' - filePath: scripts/release/rpm/pipeline_mariner.sh - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: rpm-mariner' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(artifact) - -# TODO: rpmbuild on Red Hat UBI 8 is slow for unknown reason. Still working with Red Hat to investigate. -- job: BuildRpmPackagesArm64 - displayName: Build Rpm Packages ARM64 - # Do not run this job for Pull Requests due to the slowness - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: 'ubuntu-arm64-2004-pool' - vmImage: 'ubuntu-20.04' - variables: - skipComponentGovernanceDetection: true - strategy: - matrix: - Red Hat Universal Base Image 8: - dockerfile: ubi - image: registry.access.redhat.com/ubi8/ubi:8.4 - artifact: rpm-ubi8 - python_package: python39 - Red Hat Universal Base Image 9: - dockerfile: ubi - image: registry.access.redhat.com/ubi9/ubi:9.0.0 - artifact: rpm-ubi9 - python_package: python3.9 - Fedora 35: - dockerfile: fedora - image: fedora:35 - artifact: rpm-fedora35 - python_package: python3 - Fedora 36: - dockerfile: fedora - image: fedora:36 - artifact: rpm-fedora36 - python_package: python3 - steps: - - bash: | - # https://docs.docker.com/engine/security/rootless/ - /bin/bash -c "$(curl -fsSL https://get.docker.com)" - sudo apt-get install -y uidmap - dockerd-rootless-setuptool.sh install - export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run - PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & - sleep 5 - docker context use rootless - displayName: Install Docker - - task: Bash@3 - displayName: 'Build Rpm Package' - inputs: - targetType: 'filePath' - filePath: scripts/release/rpm/pipeline.sh - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: rpm' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(artifact) - - -- job: TestRpmPackageArm64 - displayName: Test Rpm Package ARM64 - timeoutInMinutes: 120 - dependsOn: - - BuildRpmPackagesArm64 - - ExtractMetadata - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: 'ubuntu-arm64-2004-pool' - vmImage: 'ubuntu-20.04' - variables: - skipComponentGovernanceDetection: true - strategy: - matrix: - Red Hat Universal Base Image 8: - artifact: rpm-ubi8 - distro: el8 - image: registry.access.redhat.com/ubi8/ubi:8.4 - python_package: python39 - python_cmd: python3.9 - pip_cmd: pip3.9 - Red Hat Universal Base Image 9: - artifact: rpm-ubi9 - distro: el9 - image: registry.access.redhat.com/ubi9/ubi:9.0.0 - python_package: python3.9 - python_cmd: python3.9 - pip_cmd: pip3.9 - Fedora 35: - artifact: rpm-fedora35 - distro: fc35 - image: fedora:35 - python_package: python3 - python_cmd: python3 - pip_cmd: pip3 - Fedora 36: - artifact: rpm-fedora36 - distro: fc36 - image: fedora:36 - python_package: python3 - python_cmd: python3 - pip_cmd: pip3 - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' - artifactName: $(artifact) - - - bash: | - # https://docs.docker.com/engine/security/rootless/ - /bin/bash -c "$(curl -fsSL https://get.docker.com)" - sudo apt-get install -y uidmap - dockerd-rootless-setuptool.sh install - export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run - PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & - sleep 5 - docker context use rootless - displayName: Install Docker - - - bash: | - set -ex - - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - RPM_NAME=$(cd $SYSTEM_ARTIFACTSDIRECTORY/rpm/ && ls azure-cli-$CLI_VERSION-1.${DISTRO}.*.rpm) - - echo "== Test rpm package on ${IMAGE} ==" - docker pull $IMAGE - docker run --rm -e RPM_NAME=$RPM_NAME -e PYTHON_PACKAGE=${PYTHON_PACKAGE} -e PYTHON_CMD=${PYTHON_CMD} -e PIP_CMD=${PIP_CMD} -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh" - displayName: 'Test Rpm Package' - -- job: BuildDebPackagesArm64 - displayName: Build Deb Packages ARM64 - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: 'ubuntu-arm64-2004-pool' - vmImage: 'ubuntu-20.04' - variables: - skipComponentGovernanceDetection: true - strategy: - matrix: - # https://wiki.ubuntu.com/Releases - Bionic: - # 18.04 - deb_system: ubuntu - distro: bionic - Focal: - # 20.04 - deb_system: ubuntu - distro: focal - Jammy: - # 22.04 - deb_system: ubuntu - distro: jammy - - # https://wiki.debian.org/DebianReleases - Buster: - # 10 - deb_system: debian - distro: buster - Bullseye: - # 11 - deb_system: debian - distro: bullseye - steps: - - bash: | - # https://docs.docker.com/engine/security/rootless/ - /bin/bash -c "$(curl -fsSL https://get.docker.com)" - sudo apt-get install -y uidmap - dockerd-rootless-setuptool.sh install - export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run - PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & - sleep 5 - docker context use rootless - displayName: Install Docker - - task: Bash@3 - displayName: 'Build $(deb_system) $(distro) Package' - inputs: - targetType: 'filePath' - filePath: scripts/release/debian/pipeline.sh - env: - DISTRO: $(distro) - DISTRO_BASE_IMAGE: $(deb_system):$(distro) - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: debian' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(deb_system)-$(distro) - -- job: TestDebPackagesArm64 - displayName: Test Deb Packages ARM64 - dependsOn: - - BuildDebPackagesArm64 - - ExtractMetadata - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - strategy: - matrix: - Focal: - deb_system: ubuntu - distro: focal - Jammy: - deb_system: ubuntu - distro: jammy - Buster: - deb_system: debian - distro: buster - Bullseye: - deb_system: debian - distro: bullseye - pool: - name: 'ubuntu-arm64-2004-pool' - vmImage: 'ubuntu-20.04' - variables: - skipComponentGovernanceDetection: true - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download $(deb_system):$(distro) Build' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/debian' - artifactName: $(deb_system)-$(distro) - - - bash: | - # https://docs.docker.com/engine/security/rootless/ - /bin/bash -c "$(curl -fsSL https://get.docker.com)" - sudo apt-get install -y uidmap - dockerd-rootless-setuptool.sh install - export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run - PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & - sleep 5 - docker context use rootless - displayName: Install Docker - - - task: Bash@3 - displayName: 'Test $(deb_system) $(distro) Package' - env: - DISTRO: $(distro) - DISTRO_BASE_IMAGE: $(deb_system):$(distro) - inputs: - targetType: 'inline' - script: | - set -exv - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - echo "== Test debian package on ${DISTRO} ==" - docker pull ${DISTRO_BASE_IMAGE} - docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5f5e5c9b93f..9f79b9cc486 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -834,6 +834,308 @@ jobs: docker pull ${DISTRO_BASE_IMAGE} docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" +- job: BuildRpmPackageMarinerArm64 + displayName: Build Rpm Package Mariner ARM64 + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: 'ubuntu-arm64-2004-pool' + vmImage: 'ubuntu-20.04' + variables: + # Disable Component Detection on ARM64 + skipComponentGovernanceDetection: true + strategy: + matrix: + 2.0: + image: mcr.microsoft.com/cbl-mariner/base/core:2.0 + artifact: rpm-mariner2.0 + steps: + - bash: | + # https://docs.docker.com/engine/security/rootless/ + /bin/bash -c "$(curl -fsSL https://get.docker.com)" + sudo apt-get install -y uidmap + dockerd-rootless-setuptool.sh install + export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run + PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & + sleep 5 + docker context use rootless + displayName: Install Docker + - task: Bash@3 + displayName: 'Build Rpm Package: Mariner' + inputs: + targetType: 'filePath' + filePath: scripts/release/rpm/pipeline_mariner.sh + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: rpm-mariner' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(artifact) + +# TODO: rpmbuild on Red Hat UBI 8 is slow for unknown reason. Still working with Red Hat to investigate. +- job: BuildRpmPackagesArm64 + displayName: Build Rpm Packages ARM64 + # Do not run this job for Pull Requests due to the slowness + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: 'ubuntu-arm64-2004-pool' + vmImage: 'ubuntu-20.04' + variables: + skipComponentGovernanceDetection: true + strategy: + matrix: + Red Hat Universal Base Image 8: + dockerfile: ubi + image: registry.access.redhat.com/ubi8/ubi:8.4 + artifact: rpm-ubi8 + python_package: python39 + Red Hat Universal Base Image 9: + dockerfile: ubi + image: registry.access.redhat.com/ubi9/ubi:9.0.0 + artifact: rpm-ubi9 + python_package: python3.9 + Fedora 35: + dockerfile: fedora + image: fedora:35 + artifact: rpm-fedora35 + python_package: python3 + Fedora 36: + dockerfile: fedora + image: fedora:36 + artifact: rpm-fedora36 + python_package: python3 + steps: + - bash: | + # https://docs.docker.com/engine/security/rootless/ + /bin/bash -c "$(curl -fsSL https://get.docker.com)" + sudo apt-get install -y uidmap + dockerd-rootless-setuptool.sh install + export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run + PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & + sleep 5 + docker context use rootless + displayName: Install Docker + - task: Bash@3 + displayName: 'Build Rpm Package' + inputs: + targetType: 'filePath' + filePath: scripts/release/rpm/pipeline.sh + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: rpm' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(artifact) + +- job: TestRpmPackageArm64 + displayName: Test Rpm Package ARM64 + timeoutInMinutes: 120 + dependsOn: + - BuildRpmPackagesArm64 + - ExtractMetadata + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: 'ubuntu-arm64-2004-pool' + vmImage: 'ubuntu-20.04' + variables: + skipComponentGovernanceDetection: true + strategy: + matrix: + Red Hat Universal Base Image 8: + artifact: rpm-ubi8 + distro: el8 + image: registry.access.redhat.com/ubi8/ubi:8.4 + python_package: python39 + python_cmd: python3.9 + pip_cmd: pip3.9 + Red Hat Universal Base Image 9: + artifact: rpm-ubi9 + distro: el9 + image: registry.access.redhat.com/ubi9/ubi:9.0.0 + python_package: python3.9 + python_cmd: python3.9 + pip_cmd: pip3.9 + Fedora 35: + artifact: rpm-fedora35 + distro: fc35 + image: fedora:35 + python_package: python3 + python_cmd: python3 + pip_cmd: pip3 + Fedora 36: + artifact: rpm-fedora36 + distro: fc36 + image: fedora:36 + python_package: python3 + python_cmd: python3 + pip_cmd: pip3 + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' + artifactName: $(artifact) + + - bash: | + # https://docs.docker.com/engine/security/rootless/ + /bin/bash -c "$(curl -fsSL https://get.docker.com)" + sudo apt-get install -y uidmap + dockerd-rootless-setuptool.sh install + export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run + PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & + sleep 5 + docker context use rootless + displayName: Install Docker + - bash: | + set -ex + + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + RPM_NAME=$(cd $SYSTEM_ARTIFACTSDIRECTORY/rpm/ && ls azure-cli-$CLI_VERSION-1.${DISTRO}.*.rpm) + + echo "== Test rpm package on ${IMAGE} ==" + docker pull $IMAGE + docker run --rm -e RPM_NAME=$RPM_NAME -e PYTHON_PACKAGE=${PYTHON_PACKAGE} -e PYTHON_CMD=${PYTHON_CMD} -e PIP_CMD=${PIP_CMD} -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh" + displayName: 'Test Rpm Package' + +- job: BuildDebPackagesArm64 + displayName: Build Deb Packages ARM64 + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: 'ubuntu-arm64-2004-pool' + vmImage: 'ubuntu-20.04' + variables: + skipComponentGovernanceDetection: true + strategy: + matrix: + # https://wiki.ubuntu.com/Releases + Bionic: + # 18.04 + deb_system: ubuntu + distro: bionic + Focal: + # 20.04 + deb_system: ubuntu + distro: focal + Jammy: + # 22.04 + deb_system: ubuntu + distro: jammy + + # https://wiki.debian.org/DebianReleases + Buster: + # 10 + deb_system: debian + distro: buster + Bullseye: + # 11 + deb_system: debian + distro: bullseye + steps: + - bash: | + # https://docs.docker.com/engine/security/rootless/ + /bin/bash -c "$(curl -fsSL https://get.docker.com)" + sudo apt-get install -y uidmap + dockerd-rootless-setuptool.sh install + export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run + PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & + sleep 5 + docker context use rootless + displayName: Install Docker + - task: Bash@3 + displayName: 'Build $(deb_system) $(distro) Package' + inputs: + targetType: 'filePath' + filePath: scripts/release/debian/pipeline.sh + env: + DISTRO: $(distro) + DISTRO_BASE_IMAGE: $(deb_system):$(distro) + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: debian' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(deb_system)-$(distro) + +- job: TestDebPackagesArm64 + displayName: Test Deb Packages ARM64 + dependsOn: + - BuildDebPackagesArm64 + - ExtractMetadata + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + strategy: + matrix: + Focal: + deb_system: ubuntu + distro: focal + Jammy: + deb_system: ubuntu + distro: jammy + Buster: + deb_system: debian + distro: buster + Bullseye: + deb_system: debian + distro: bullseye + pool: + name: 'ubuntu-arm64-2004-pool' + vmImage: 'ubuntu-20.04' + variables: + skipComponentGovernanceDetection: true + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download $(deb_system):$(distro) Build' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/debian' + artifactName: $(deb_system)-$(distro) + - bash: | + # https://docs.docker.com/engine/security/rootless/ + /bin/bash -c "$(curl -fsSL https://get.docker.com)" + sudo apt-get install -y uidmap + dockerd-rootless-setuptool.sh install + export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run + PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & + sleep 5 + docker context use rootless + displayName: Install Docker + + - task: Bash@3 + displayName: 'Test $(deb_system) $(distro) Package' + env: + DISTRO: $(distro) + DISTRO_BASE_IMAGE: $(deb_system):$(distro) + inputs: + targetType: 'inline' + script: | + set -exv + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + echo "== Test debian package on ${DISTRO} ==" + docker pull ${DISTRO_BASE_IMAGE} + docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" + - job: CheckStyle displayName: "Check CLI Style" From 80eedec9cbbe8014097fd424891fc377a39b6f1e Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Tue, 18 Oct 2022 10:09:10 +0800 Subject: [PATCH 04/26] Use variables --- .azure-pipelines/templates/variables.yml | 3 ++- azure-pipelines.yml | 15 +++++---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.azure-pipelines/templates/variables.yml b/.azure-pipelines/templates/variables.yml index dbc4960483d..f53f0c89e30 100644 --- a/.azure-pipelines/templates/variables.yml +++ b/.azure-pipelines/templates/variables.yml @@ -1,3 +1,4 @@ variables: ubuntu_pool: 'pool-ubuntu-2004' - windows_pool: 'pool-windows-2019' \ No newline at end of file + windows_pool: 'pool-windows-2019' + ubuntu_arm64_pool: 'ubuntu-arm64-2004-pool' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9f79b9cc486..398f04df294 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -838,8 +838,7 @@ jobs: displayName: Build Rpm Package Mariner ARM64 condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) pool: - name: 'ubuntu-arm64-2004-pool' - vmImage: 'ubuntu-20.04' + name: ${{ variables.ubuntu_arm64_pool }} variables: # Disable Component Detection on ARM64 skipComponentGovernanceDetection: true @@ -882,8 +881,7 @@ jobs: # Do not run this job for Pull Requests due to the slowness condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) pool: - name: 'ubuntu-arm64-2004-pool' - vmImage: 'ubuntu-20.04' + name: ${{ variables.ubuntu_arm64_pool }} variables: skipComponentGovernanceDetection: true strategy: @@ -942,8 +940,7 @@ jobs: - ExtractMetadata condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) pool: - name: 'ubuntu-arm64-2004-pool' - vmImage: 'ubuntu-20.04' + name: ${{ variables.ubuntu_arm64_pool }} variables: skipComponentGovernanceDetection: true strategy: @@ -1014,8 +1011,7 @@ jobs: displayName: Build Deb Packages ARM64 condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) pool: - name: 'ubuntu-arm64-2004-pool' - vmImage: 'ubuntu-20.04' + name: ${{ variables.ubuntu_arm64_pool }} variables: skipComponentGovernanceDetection: true strategy: @@ -1095,8 +1091,7 @@ jobs: deb_system: debian distro: bullseye pool: - name: 'ubuntu-arm64-2004-pool' - vmImage: 'ubuntu-20.04' + name: ${{ variables.ubuntu_arm64_pool }} variables: skipComponentGovernanceDetection: true steps: From dc4a1861b7971468c966749e1351695383605deb Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Tue, 18 Oct 2022 12:13:58 +0800 Subject: [PATCH 05/26] Fix artifact name --- azure-pipelines.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 398f04df294..f589fe985d2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -846,7 +846,7 @@ jobs: matrix: 2.0: image: mcr.microsoft.com/cbl-mariner/base/core:2.0 - artifact: rpm-mariner2.0 + artifact: rpm-mariner2.0-arm64 steps: - bash: | # https://docs.docker.com/engine/security/rootless/ @@ -889,22 +889,22 @@ jobs: Red Hat Universal Base Image 8: dockerfile: ubi image: registry.access.redhat.com/ubi8/ubi:8.4 - artifact: rpm-ubi8 + artifact: rpm-ubi8-arm64 python_package: python39 Red Hat Universal Base Image 9: dockerfile: ubi image: registry.access.redhat.com/ubi9/ubi:9.0.0 - artifact: rpm-ubi9 + artifact: rpm-ubi9-arm64 python_package: python3.9 Fedora 35: dockerfile: fedora image: fedora:35 - artifact: rpm-fedora35 + artifact: rpm-fedora35-arm64 python_package: python3 Fedora 36: dockerfile: fedora image: fedora:36 - artifact: rpm-fedora36 + artifact: rpm-fedora36-arm64 python_package: python3 steps: - bash: | @@ -946,28 +946,28 @@ jobs: strategy: matrix: Red Hat Universal Base Image 8: - artifact: rpm-ubi8 + artifact: rpm-ubi8-arm64 distro: el8 image: registry.access.redhat.com/ubi8/ubi:8.4 python_package: python39 python_cmd: python3.9 pip_cmd: pip3.9 Red Hat Universal Base Image 9: - artifact: rpm-ubi9 + artifact: rpm-ubi9-arm64 distro: el9 image: registry.access.redhat.com/ubi9/ubi:9.0.0 python_package: python3.9 python_cmd: python3.9 pip_cmd: pip3.9 Fedora 35: - artifact: rpm-fedora35 + artifact: rpm-fedora35-arm64 distro: fc35 image: fedora:35 python_package: python3 python_cmd: python3 pip_cmd: pip3 Fedora 36: - artifact: rpm-fedora36 + artifact: rpm-fedora36-arm64 distro: fc36 image: fedora:36 python_package: python3 @@ -1068,7 +1068,7 @@ jobs: displayName: 'Publish Artifact: debian' inputs: TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(deb_system)-$(distro) + ArtifactName: $(deb_system)-$(distro)-arm64 - job: TestDebPackagesArm64 displayName: Test Deb Packages ARM64 @@ -1105,7 +1105,7 @@ jobs: displayName: 'Download $(deb_system):$(distro) Build' inputs: TargetPath: '$(Build.ArtifactStagingDirectory)/debian' - artifactName: $(deb_system)-$(distro) + artifactName: $(deb_system)-$(distro)-arm64 - bash: | # https://docs.docker.com/engine/security/rootless/ /bin/bash -c "$(curl -fsSL https://get.docker.com)" From d9cf405f214ba03c8afb1fb3cc98fe82e19d0500 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Tue, 18 Oct 2022 13:23:46 +0800 Subject: [PATCH 06/26] Add DEB_NAME --- azure-pipelines.yml | 13 ++++++++----- scripts/release/debian/test_deb_in_docker.sh | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f589fe985d2..2c7ff4c6cb7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -726,8 +726,7 @@ jobs: set -ex CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - RPM_NAME=azure-cli-$CLI_VERSION-1.${DISTRO}.x86_64.rpm - RPM_FILE=$SYSTEM_ARTIFACTSDIRECTORY/rpm/$RPM_NAME + RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name 'azure-cli-$CLI_VERSION-1.${DISTRO}.*.rpm' -printf '%f\n') echo "== Test rpm package on ${IMAGE} ==" docker pull $IMAGE @@ -830,9 +829,11 @@ jobs: script: | set -exv CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + DEB_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/debian/ -type f -name 'azure-cli_$CLI_VERSION-1~${DISTRO}_*.deb' -printf '%f\n') + echo "== Test debian package on ${DISTRO} ==" docker pull ${DISTRO_BASE_IMAGE} - docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" + docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -e DEB_NAME=$DEB_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" - job: BuildRpmPackageMarinerArm64 displayName: Build Rpm Package Mariner ARM64 @@ -1000,7 +1001,7 @@ jobs: set -ex CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - RPM_NAME=$(cd $SYSTEM_ARTIFACTSDIRECTORY/rpm/ && ls azure-cli-$CLI_VERSION-1.${DISTRO}.*.rpm) + RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name 'azure-cli-$CLI_VERSION-1.${DISTRO}.*.rpm' -printf '%f\n') echo "== Test rpm package on ${IMAGE} ==" docker pull $IMAGE @@ -1127,9 +1128,11 @@ jobs: script: | set -exv CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + DEB_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/debian/ -type f -name 'azure-cli_$CLI_VERSION-1~${DISTRO}_*.deb' -printf '%f\n') + echo "== Test debian package on ${DISTRO} ==" docker pull ${DISTRO_BASE_IMAGE} - docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" + docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -e DEB_NAME=$DEB_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" - job: CheckStyle displayName: "Check CLI Style" diff --git a/scripts/release/debian/test_deb_in_docker.sh b/scripts/release/debian/test_deb_in_docker.sh index 2809fa907f1..1f902c432a6 100644 --- a/scripts/release/debian/test_deb_in_docker.sh +++ b/scripts/release/debian/test_deb_in_docker.sh @@ -14,7 +14,7 @@ case ${DISTRO} in *) :;; esac -dpkg -i /mnt/artifacts/azure-cli_$CLI_VERSION-1~${DISTRO}_all.deb +dpkg -i /mnt/artifacts/$DEB_NAME time az self-test time az --version From 5eed879572cefe6d64d841d177469e34339828b2 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Tue, 18 Oct 2022 14:26:08 +0800 Subject: [PATCH 07/26] Minor fix --- azure-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2c7ff4c6cb7..471a64766be 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -726,7 +726,7 @@ jobs: set -ex CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name 'azure-cli-$CLI_VERSION-1.${DISTRO}.*.rpm' -printf '%f\n') + RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$CLI_VERSION-1.${DISTRO}.*.rpm" -printf '%f\n') echo "== Test rpm package on ${IMAGE} ==" docker pull $IMAGE @@ -829,7 +829,7 @@ jobs: script: | set -exv CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - DEB_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/debian/ -type f -name 'azure-cli_$CLI_VERSION-1~${DISTRO}_*.deb' -printf '%f\n') + DEB_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/debian/ -type f -name "azure-cli_$CLI_VERSION-1~${DISTRO}_*.deb" -printf '%f\n') echo "== Test debian package on ${DISTRO} ==" docker pull ${DISTRO_BASE_IMAGE} @@ -1001,7 +1001,7 @@ jobs: set -ex CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name 'azure-cli-$CLI_VERSION-1.${DISTRO}.*.rpm' -printf '%f\n') + RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$CLI_VERSION-1.${DISTRO}.*.rpm" -printf '%f\n') echo "== Test rpm package on ${IMAGE} ==" docker pull $IMAGE @@ -1128,7 +1128,7 @@ jobs: script: | set -exv CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - DEB_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/debian/ -type f -name 'azure-cli_$CLI_VERSION-1~${DISTRO}_*.deb' -printf '%f\n') + DEB_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/debian/ -type f -name "azure-cli_$CLI_VERSION-1~${DISTRO}_*.deb" -printf '%f\n') echo "== Test debian package on ${DISTRO} ==" docker pull ${DISTRO_BASE_IMAGE} From 4294fd53cc8c257293bbe75c1d695a3f80fe9c72 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Tue, 18 Oct 2022 14:45:10 +0800 Subject: [PATCH 08/26] Fix architecture --- scripts/release/debian/deb_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release/debian/deb_install.sh b/scripts/release/debian/deb_install.sh index 873e5bfa97b..49660bcc422 100755 --- a/scripts/release/debian/deb_install.sh +++ b/scripts/release/debian/deb_install.sh @@ -81,7 +81,7 @@ setup() { exit 1 fi fi - echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ ${CLI_REPO} main" \ + echo "deb [arch=$(dpkg --print-architecture)] https://packages.microsoft.com/repos/azure-cli/ ${CLI_REPO} main" \ > /etc/apt/sources.list.d/azure-cli.list apt-get update set +v From 2489f7d1d90f03533c91e253aa645b3285d91766 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Wed, 2 Nov 2022 11:43:21 +0800 Subject: [PATCH 09/26] Add arm64 docker image --- azure-pipelines.yml | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 471a64766be..aca071558b5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -256,8 +256,16 @@ jobs: dependsOn: ExtractMetadata condition: succeeded() + strategy: + matrix: + AMD64: + pool: ${{ variables.ubuntu_pool }} + artifactName: docker + ARM64: + pool: ${{ variables.ubuntu_arm64_pool }} + artifactName: docker-arm64 pool: - name: ${{ variables.ubuntu_pool }} + name: $(pool) steps: - task: Bash@3 displayName: 'Bash Script' @@ -275,15 +283,23 @@ jobs: displayName: 'Publish Artifact: docker image' inputs: TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: docker + ArtifactName: $(artifactName) - job: TestDockerImage displayName: Test Docker Image dependsOn: BuildDockerImage condition: succeeded() + strategy: + matrix: + AMD64: + pool: ${{ variables.ubuntu_pool }} + artifactName: docker + ARM64: + pool: ${{ variables.ubuntu_arm64_pool }} + artifactName: docker-arm64 pool: - name: ${{ variables.ubuntu_pool }} + name: $(pool) steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Metadata' @@ -296,7 +312,7 @@ jobs: displayName: 'Download Docker Image' inputs: TargetPath: '$(Build.ArtifactStagingDirectory)/docker' - artifactName: docker + artifactName: $(artifactName) - bash: | From d7f9cfa7f95e6143e67509e012806952dd1f5a16 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Wed, 2 Nov 2022 14:14:53 +0800 Subject: [PATCH 10/26] Minor fix --- azure-pipelines.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fb25e6f2ebe..e42ac4e7165 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -267,6 +267,20 @@ jobs: pool: name: $(pool) steps: + - bash: | + if [[ $(dpkg --print-architecture) == "amd64" ]]; then + echo "Docker is already installed on AMD64" + exit 0 + fi + # https://docs.docker.com/engine/security/rootless/ + /bin/bash -c "$(curl -fsSL https://get.docker.com)" + sudo apt-get install -y uidmap + dockerd-rootless-setuptool.sh install + export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run + PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & + sleep 5 + docker context use rootless + displayName: Install Docker - task: Bash@3 displayName: 'Bash Script' inputs: @@ -314,6 +328,20 @@ jobs: TargetPath: '$(Build.ArtifactStagingDirectory)/docker' artifactName: $(artifactName) + - bash: | + if [[ $(dpkg --print-architecture) == "amd64" ]]; then + echo "Docker is already installed on AMD64" + exit 0 + fi + # https://docs.docker.com/engine/security/rootless/ + /bin/bash -c "$(curl -fsSL https://get.docker.com)" + sudo apt-get install -y uidmap + dockerd-rootless-setuptool.sh install + export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run + PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & + sleep 5 + docker context use rootless + displayName: Install Docker - bash: | set -exv From 8513c2867ec1d005ff7db2fb4a13e715245f5ffb Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Wed, 2 Nov 2022 15:13:45 +0800 Subject: [PATCH 11/26] Use matrix in pool --- azure-pipelines.yml | 77 ++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 46 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e42ac4e7165..f7ff1f95074 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -15,6 +15,15 @@ pr: variables: - template: ${{ variables.Pipeline.Workspace }}/.azure-pipelines/templates/variables.yml +parameters: +- name: architectures + type: object + default: + - value: amd64 + pool: ${{ variables.ubuntu_pool }} + - value: arm64 + pool: ${{ variables.ubuntu_arm64_pool }} + jobs: - job: CheckPullRequest displayName: "Check the Format of Pull Request Title and Content" @@ -643,14 +652,31 @@ jobs: - job: BuildRpmPackageMariner displayName: Build Rpm Package Mariner condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: ${{ variables.ubuntu_pool }} strategy: matrix: - 2.0: - image: mcr.microsoft.com/cbl-mariner/base/core:2.0 - artifact: rpm-mariner2.0 + ${{ each arch in parameters.architectures }}: + 2.0 ${{ arch.value }}: + image: mcr.microsoft.com/cbl-mariner/base/core:2.0 + artifact: rpm-mariner2.0-${{ arch.value }} + pool: ${{ arch.pool }} + pool: + name: $(pool) steps: + - bash: | + if [[ $(dpkg --print-architecture) == "amd64" ]]; then + echo "Docker is already installed on AMD64" + exit 0 + fi + # https://docs.docker.com/engine/security/rootless/ + /bin/bash -c "$(curl -fsSL https://get.docker.com)" + sudo apt-get install -y uidmap + dockerd-rootless-setuptool.sh install + export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run + PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & + sleep 5 + docker context use rootless + displayName: Install Docker + - task: Bash@3 displayName: 'Build Rpm Package: Mariner' inputs: @@ -879,47 +905,6 @@ jobs: docker pull ${DISTRO_BASE_IMAGE} docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -e DEB_NAME=$DEB_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" -- job: BuildRpmPackageMarinerArm64 - displayName: Build Rpm Package Mariner ARM64 - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: ${{ variables.ubuntu_arm64_pool }} - variables: - # Disable Component Detection on ARM64 - skipComponentGovernanceDetection: true - strategy: - matrix: - 2.0: - image: mcr.microsoft.com/cbl-mariner/base/core:2.0 - artifact: rpm-mariner2.0-arm64 - steps: - - bash: | - # https://docs.docker.com/engine/security/rootless/ - /bin/bash -c "$(curl -fsSL https://get.docker.com)" - sudo apt-get install -y uidmap - dockerd-rootless-setuptool.sh install - export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run - PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & - sleep 5 - docker context use rootless - displayName: Install Docker - - task: Bash@3 - displayName: 'Build Rpm Package: Mariner' - inputs: - targetType: 'filePath' - filePath: scripts/release/rpm/pipeline_mariner.sh - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: rpm-mariner' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(artifact) - # TODO: rpmbuild on Red Hat UBI 8 is slow for unknown reason. Still working with Red Hat to investigate. - job: BuildRpmPackagesArm64 displayName: Build Rpm Packages ARM64 From c8b0e95e7002549e2a047c7551daecdbec93fe8b Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Wed, 2 Nov 2022 16:42:50 +0800 Subject: [PATCH 12/26] Enhance rpm build --- azure-pipelines.yml | 294 ++++++++++++++++---------------------------- 1 file changed, 103 insertions(+), 191 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f7ff1f95074..3718d0d1454 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,10 +19,12 @@ parameters: - name: architectures type: object default: - - value: amd64 - pool: ${{ variables.ubuntu_pool }} - - value: arm64 - pool: ${{ variables.ubuntu_arm64_pool }} + - name: AMD64 + value: amd64 + pool: pool-ubuntu-2004 + - name: ARM64 + value: arm64 + pool: ubuntu-arm64-2004-pool jobs: - job: CheckPullRequest @@ -654,11 +656,14 @@ jobs: condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) strategy: matrix: - ${{ each arch in parameters.architectures }}: - 2.0 ${{ arch.value }}: - image: mcr.microsoft.com/cbl-mariner/base/core:2.0 - artifact: rpm-mariner2.0-${{ arch.value }} - pool: ${{ arch.pool }} + 2.0 AMD64: + image: mcr.microsoft.com/cbl-mariner/base/core:2.0 + artifact: rpm-mariner2.0-amd64 + pool: ${{ variables.ubuntu_pool }} + 2.0 ARM64: + image: mcr.microsoft.com/cbl-mariner/base/core:2.0 + artifact: rpm-mariner2.0-arm64 + pool: ${{ variables.ubuntu_arm64_pool }} pool: name: $(pool) steps: @@ -700,30 +705,49 @@ jobs: # Do not run this job for Pull Requests due to the slowness condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) pool: - name: ${{ variables.ubuntu_pool }} + name: $(pool) strategy: matrix: - Red Hat Universal Base Image 8: - dockerfile: ubi - image: registry.access.redhat.com/ubi8/ubi:8.4 - artifact: rpm-ubi8 - python_package: python39 - Red Hat Universal Base Image 9: - dockerfile: ubi - image: registry.access.redhat.com/ubi9/ubi:9.0.0 - artifact: rpm-ubi9 - python_package: python3.9 - Fedora 35: - dockerfile: fedora - image: fedora:35 - artifact: rpm-fedora35 - python_package: python3 - Fedora 36: - dockerfile: fedora - image: fedora:36 - artifact: rpm-fedora36 - python_package: python3 + ${{ each arch in parameters.architectures }}: + Red Hat Universal Base Image 8 ${{ arch.name }}: + dockerfile: ubi + image: registry.access.redhat.com/ubi8/ubi:8.4 + artifact: rpm-ubi8-${{ arch.value }} + python_package: python39 + pool: ${{ arch.pool }} + Red Hat Universal Base Image 9 ${{ arch.name }}: + dockerfile: ubi + image: registry.access.redhat.com/ubi9/ubi:9.0.0 + artifact: rpm-ubi9-${{ arch.value }} + python_package: python3.9 + pool: ${{ arch.pool }} + Fedora 35 ${{ arch.name }}: + dockerfile: fedora + image: fedora:35 + artifact: rpm-fedora35-${{ arch.value }} + python_package: python3 + pool: ${{ arch.pool }} + Fedora 36 ${{ arch.name }}: + dockerfile: fedora + image: fedora:36 + artifact: rpm-fedora36-${{ arch.value }} + python_package: python3 + pool: ${{ arch.pool }} steps: + - bash: | + if [[ $(dpkg --print-architecture) == "amd64" ]]; then + echo "Docker is already installed on AMD64" + exit 0 + fi + # https://docs.docker.com/engine/security/rootless/ + /bin/bash -c "$(curl -fsSL https://get.docker.com)" + sudo apt-get install -y uidmap + dockerd-rootless-setuptool.sh install + export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run + PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & + sleep 5 + docker context use rootless + displayName: Install Docker - task: Bash@3 displayName: 'Build Rpm Package' inputs: @@ -748,37 +772,42 @@ jobs: - ExtractMetadata condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) pool: - name: ${{ variables.ubuntu_pool }} + name: $(pool) strategy: matrix: - Red Hat Universal Base Image 8: - artifact: rpm-ubi8 - distro: el8 - image: registry.access.redhat.com/ubi8/ubi:8.4 - python_package: python39 - python_cmd: python3.9 - pip_cmd: pip3.9 - Red Hat Universal Base Image 9: - artifact: rpm-ubi9 - distro: el9 - image: registry.access.redhat.com/ubi9/ubi:9.0.0 - python_package: python3.9 - python_cmd: python3.9 - pip_cmd: pip3.9 - Fedora 35: - artifact: rpm-fedora35 - distro: fc35 - image: fedora:35 - python_package: python3 - python_cmd: python3 - pip_cmd: pip3 - Fedora 36: - artifact: rpm-fedora36 - distro: fc36 - image: fedora:36 - python_package: python3 - python_cmd: python3 - pip_cmd: pip3 + ${{ each arch in parameters.architectures }}: + Red Hat Universal Base Image 8 ${{ arch.name }}: + artifact: rpm-ubi8-${{ arch.value }} + distro: el8 + image: registry.access.redhat.com/ubi8/ubi:8.4 + python_package: python39 + python_cmd: python3.9 + pip_cmd: pip3.9 + pool: ${{ arch.pool }} + Red Hat Universal Base Image 9 ${{ arch.name }}: + artifact: rpm-ubi9-${{ arch.value }} + distro: el9 + image: registry.access.redhat.com/ubi9/ubi:9.0.0 + python_package: python3.9 + python_cmd: python3.9 + pip_cmd: pip3.9 + pool: ${{ arch.pool }} + Fedora 35 ${{ arch.name }}: + artifact: rpm-fedora35-${{ arch.value }} + distro: fc35 + image: fedora:35 + python_package: python3 + python_cmd: python3 + pip_cmd: pip3 + pool: ${{ arch.pool }} + Fedora 36 ${{ arch.name }}: + artifact: rpm-fedora36-${{ arch.value }} + distro: fc36 + image: fedora:36 + python_package: python3 + python_cmd: python3 + pip_cmd: pip3 + pool: ${{ arch.pool }} steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Metadata' @@ -792,6 +821,21 @@ jobs: TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' artifactName: $(artifact) + - bash: | + if [[ $(dpkg --print-architecture) == "amd64" ]]; then + echo "Docker is already installed on AMD64" + exit 0 + fi + # https://docs.docker.com/engine/security/rootless/ + /bin/bash -c "$(curl -fsSL https://get.docker.com)" + sudo apt-get install -y uidmap + dockerd-rootless-setuptool.sh install + export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run + PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & + sleep 5 + docker context use rootless + displayName: Install Docker + - bash: | set -ex @@ -905,138 +949,6 @@ jobs: docker pull ${DISTRO_BASE_IMAGE} docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -e DEB_NAME=$DEB_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" -# TODO: rpmbuild on Red Hat UBI 8 is slow for unknown reason. Still working with Red Hat to investigate. -- job: BuildRpmPackagesArm64 - displayName: Build Rpm Packages ARM64 - # Do not run this job for Pull Requests due to the slowness - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: ${{ variables.ubuntu_arm64_pool }} - variables: - skipComponentGovernanceDetection: true - strategy: - matrix: - Red Hat Universal Base Image 8: - dockerfile: ubi - image: registry.access.redhat.com/ubi8/ubi:8.4 - artifact: rpm-ubi8-arm64 - python_package: python39 - Red Hat Universal Base Image 9: - dockerfile: ubi - image: registry.access.redhat.com/ubi9/ubi:9.0.0 - artifact: rpm-ubi9-arm64 - python_package: python3.9 - Fedora 35: - dockerfile: fedora - image: fedora:35 - artifact: rpm-fedora35-arm64 - python_package: python3 - Fedora 36: - dockerfile: fedora - image: fedora:36 - artifact: rpm-fedora36-arm64 - python_package: python3 - steps: - - bash: | - # https://docs.docker.com/engine/security/rootless/ - /bin/bash -c "$(curl -fsSL https://get.docker.com)" - sudo apt-get install -y uidmap - dockerd-rootless-setuptool.sh install - export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run - PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & - sleep 5 - docker context use rootless - displayName: Install Docker - - task: Bash@3 - displayName: 'Build Rpm Package' - inputs: - targetType: 'filePath' - filePath: scripts/release/rpm/pipeline.sh - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: rpm' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(artifact) - -- job: TestRpmPackageArm64 - displayName: Test Rpm Package ARM64 - timeoutInMinutes: 120 - dependsOn: - - BuildRpmPackagesArm64 - - ExtractMetadata - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: ${{ variables.ubuntu_arm64_pool }} - variables: - skipComponentGovernanceDetection: true - strategy: - matrix: - Red Hat Universal Base Image 8: - artifact: rpm-ubi8-arm64 - distro: el8 - image: registry.access.redhat.com/ubi8/ubi:8.4 - python_package: python39 - python_cmd: python3.9 - pip_cmd: pip3.9 - Red Hat Universal Base Image 9: - artifact: rpm-ubi9-arm64 - distro: el9 - image: registry.access.redhat.com/ubi9/ubi:9.0.0 - python_package: python3.9 - python_cmd: python3.9 - pip_cmd: pip3.9 - Fedora 35: - artifact: rpm-fedora35-arm64 - distro: fc35 - image: fedora:35 - python_package: python3 - python_cmd: python3 - pip_cmd: pip3 - Fedora 36: - artifact: rpm-fedora36-arm64 - distro: fc36 - image: fedora:36 - python_package: python3 - python_cmd: python3 - pip_cmd: pip3 - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' - artifactName: $(artifact) - - - bash: | - # https://docs.docker.com/engine/security/rootless/ - /bin/bash -c "$(curl -fsSL https://get.docker.com)" - sudo apt-get install -y uidmap - dockerd-rootless-setuptool.sh install - export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run - PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & - sleep 5 - docker context use rootless - displayName: Install Docker - - bash: | - set -ex - - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$CLI_VERSION-1.${DISTRO}.*.rpm" -printf '%f\n') - - echo "== Test rpm package on ${IMAGE} ==" - docker pull $IMAGE - docker run --rm -e RPM_NAME=$RPM_NAME -e PYTHON_PACKAGE=${PYTHON_PACKAGE} -e PYTHON_CMD=${PYTHON_CMD} -e PIP_CMD=${PIP_CMD} -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh" - displayName: 'Test Rpm Package' - - job: BuildDebPackagesArm64 displayName: Build Deb Packages ARM64 condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) From 791c42276e9a74d7499fe38eb9ae6887e07b9518 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Thu, 3 Nov 2022 10:41:23 +0800 Subject: [PATCH 13/26] Enhance deb --- azure-pipelines.yml | 242 +++++++++++++++----------------------------- 1 file changed, 81 insertions(+), 161 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3718d0d1454..1e0eae4c36e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -852,40 +852,64 @@ jobs: displayName: Build Deb Packages condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) pool: - name: ${{ variables.ubuntu_pool }} + name: $(pool) strategy: matrix: - # https://wiki.ubuntu.com/Releases - Bionic: - # 18.04 - deb_system: ubuntu - distro: bionic - Focal: - # 20.04 - deb_system: ubuntu - distro: focal - Jammy: - # 22.04 - deb_system: ubuntu - distro: jammy - - # https://wiki.debian.org/DebianReleases - Buster: - # 10 - deb_system: debian - distro: buster - Bullseye: - # 11 - deb_system: debian - distro: bullseye + ${{ each arch in parameters.architectures }}: + # https://wiki.ubuntu.com/Releases + Bionic ${{ arch.name }}: + # 18.04 + deb_system: ubuntu + distro: bionic + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + Focal ${{ arch.name }}: + # 20.04 + deb_system: ubuntu + distro: focal + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + Jammy ${{ arch.name }}: + # 22.04 + deb_system: ubuntu + distro: jammy + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + + # https://wiki.debian.org/DebianReleases + Buster ${{ arch.name }}: + # 10 + deb_system: debian + distro: buster + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + Bullseye ${{ arch.name }}: + # 11 + deb_system: debian + distro: bullseye + arch: ${{ arch.value }} + pool: ${{ arch.pool }} steps: + - bash: | + if [[ $(dpkg --print-architecture) == "amd64" ]]; then + echo "Docker is already installed on AMD64" + exit 0 + fi + # https://docs.docker.com/engine/security/rootless/ + /bin/bash -c "$(curl -fsSL https://get.docker.com)" + sudo apt-get install -y uidmap + dockerd-rootless-setuptool.sh install + export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run + PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & + sleep 5 + docker context use rootless + displayName: Install Docker - task: Bash@3 - displayName: 'Build $(deb_system) $(distro) Package' + displayName: 'Build $(deb_system) $(distro) $(arch) Package' inputs: targetType: 'filePath' filePath: scripts/release/debian/pipeline.sh env: - DISTRO: $(distro) DISTRO_BASE_IMAGE: $(deb_system):$(distro) - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 @@ -894,10 +918,10 @@ jobs: BuildDropPath: $(Build.ArtifactStagingDirectory) - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: debian' + displayName: 'Publish Artifact: $(deb_system) $(distro) $(arch)' inputs: TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(deb_system)-$(distro) + ArtifactName: $(deb_system)-$(distro)-$(arch) - job: TestDebPackages displayName: Test Deb Packages @@ -906,83 +930,35 @@ jobs: condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) strategy: matrix: - Focal: - deb_system: ubuntu - distro: focal - Jammy: - deb_system: ubuntu - distro: jammy - Buster: - deb_system: debian - distro: buster - Bullseye: - deb_system: debian - distro: bullseye - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download $(deb_system):$(distro) Build' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/debian' - artifactName: $(deb_system)-$(distro) - - - task: Bash@3 - displayName: 'Test $(deb_system) $(distro) Package' - env: - DISTRO: $(distro) - DISTRO_BASE_IMAGE: $(deb_system):$(distro) - inputs: - targetType: 'inline' - script: | - set -exv - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - DEB_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/debian/ -type f -name "azure-cli_$CLI_VERSION-1~${DISTRO}_*.deb" -printf '%f\n') - - echo "== Test debian package on ${DISTRO} ==" - docker pull ${DISTRO_BASE_IMAGE} - docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -e DEB_NAME=$DEB_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" - -- job: BuildDebPackagesArm64 - displayName: Build Deb Packages ARM64 - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + ${{ each arch in parameters.architectures }}: + Focal ${{ arch.name }}: + deb_system: ubuntu + distro: focal + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + Jammy ${{ arch.name }}: + deb_system: ubuntu + distro: jammy + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + Buster ${{ arch.name }}: + deb_system: debian + distro: buster + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + Bullseye ${{ arch.name }}: + deb_system: debian + distro: bullseye + arch: ${{ arch.value }} + pool: ${{ arch.pool }} pool: - name: ${{ variables.ubuntu_arm64_pool }} - variables: - skipComponentGovernanceDetection: true - strategy: - matrix: - # https://wiki.ubuntu.com/Releases - Bionic: - # 18.04 - deb_system: ubuntu - distro: bionic - Focal: - # 20.04 - deb_system: ubuntu - distro: focal - Jammy: - # 22.04 - deb_system: ubuntu - distro: jammy - - # https://wiki.debian.org/DebianReleases - Buster: - # 10 - deb_system: debian - distro: buster - Bullseye: - # 11 - deb_system: debian - distro: bullseye + name: $(pool) steps: - bash: | + if [[ $(dpkg --print-architecture) == "amd64" ]]; then + echo "Docker is already installed on AMD64" + exit 0 + fi # https://docs.docker.com/engine/security/rootless/ /bin/bash -c "$(curl -fsSL https://get.docker.com)" sudo apt-get install -y uidmap @@ -992,51 +968,6 @@ jobs: sleep 5 docker context use rootless displayName: Install Docker - - task: Bash@3 - displayName: 'Build $(deb_system) $(distro) Package' - inputs: - targetType: 'filePath' - filePath: scripts/release/debian/pipeline.sh - env: - DISTRO: $(distro) - DISTRO_BASE_IMAGE: $(deb_system):$(distro) - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: debian' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(deb_system)-$(distro)-arm64 - -- job: TestDebPackagesArm64 - displayName: Test Deb Packages ARM64 - dependsOn: - - BuildDebPackagesArm64 - - ExtractMetadata - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - strategy: - matrix: - Focal: - deb_system: ubuntu - distro: focal - Jammy: - deb_system: ubuntu - distro: jammy - Buster: - deb_system: debian - distro: buster - Bullseye: - deb_system: debian - distro: bullseye - pool: - name: ${{ variables.ubuntu_arm64_pool }} - variables: - skipComponentGovernanceDetection: true - steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Metadata' inputs: @@ -1044,25 +975,14 @@ jobs: artifactName: metadata - task: DownloadPipelineArtifact@1 - displayName: 'Download $(deb_system):$(distro) Build' + displayName: 'Download $(deb_system):$(distro) $(arch) Build' inputs: TargetPath: '$(Build.ArtifactStagingDirectory)/debian' - artifactName: $(deb_system)-$(distro)-arm64 - - bash: | - # https://docs.docker.com/engine/security/rootless/ - /bin/bash -c "$(curl -fsSL https://get.docker.com)" - sudo apt-get install -y uidmap - dockerd-rootless-setuptool.sh install - export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run - PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & - sleep 5 - docker context use rootless - displayName: Install Docker + artifactName: $(deb_system)-$(distro)-$(arch) - task: Bash@3 - displayName: 'Test $(deb_system) $(distro) Package' + displayName: 'Test $(deb_system) $(distro) $(arch) Package' env: - DISTRO: $(distro) DISTRO_BASE_IMAGE: $(deb_system):$(distro) inputs: targetType: 'inline' From 00b57a0947cb1aa0ee513cf48a39eb9e82011a0c Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Thu, 3 Nov 2022 11:53:46 +0800 Subject: [PATCH 14/26] Add suffix in docker amd64 artifact --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1e0eae4c36e..61170c81816 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -271,7 +271,7 @@ jobs: matrix: AMD64: pool: ${{ variables.ubuntu_pool }} - artifactName: docker + artifactName: docker-amd64 ARM64: pool: ${{ variables.ubuntu_arm64_pool }} artifactName: docker-arm64 @@ -319,7 +319,7 @@ jobs: matrix: AMD64: pool: ${{ variables.ubuntu_pool }} - artifactName: docker + artifactName: docker-amd64 ARM64: pool: ${{ variables.ubuntu_arm64_pool }} artifactName: docker-arm64 From 68688ba91fb8be99a9ad68164514ce0f7e8effe9 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Mon, 7 Nov 2022 14:31:09 +0800 Subject: [PATCH 15/26] Remove DEB_NAME env --- azure-pipelines.yml | 3 +-- scripts/release/debian/test_deb_in_docker.sh | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 61170c81816..afa3c010799 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -989,11 +989,10 @@ jobs: script: | set -exv CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - DEB_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/debian/ -type f -name "azure-cli_$CLI_VERSION-1~${DISTRO}_*.deb" -printf '%f\n') echo "== Test debian package on ${DISTRO} ==" docker pull ${DISTRO_BASE_IMAGE} - docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -e DEB_NAME=$DEB_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" + docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" - job: CheckStyle displayName: "Check CLI Style" diff --git a/scripts/release/debian/test_deb_in_docker.sh b/scripts/release/debian/test_deb_in_docker.sh index 7d88471c339..e3d7c21ea9b 100644 --- a/scripts/release/debian/test_deb_in_docker.sh +++ b/scripts/release/debian/test_deb_in_docker.sh @@ -14,7 +14,8 @@ case ${DISTRO} in *) :;; esac -dpkg -i /mnt/artifacts/$DEB_NAME +DEB_NAME=$(find /mnt/artifacts -type f -name "azure-cli_$CLI_VERSION-1~${DISTRO}_*.deb") +dpkg -i $DEB_NAME time az self-test time az --version From 2e9ae4dac631f0ab041c3983cbfe5f8dd5b8736e Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Fri, 16 Dec 2022 14:47:51 +0800 Subject: [PATCH 16/26] Fix rpm path --- scripts/release/rpm/mariner.dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/release/rpm/mariner.dockerfile b/scripts/release/rpm/mariner.dockerfile index 6795f9518aa..8ac3031efac 100644 --- a/scripts/release/rpm/mariner.dockerfile +++ b/scripts/release/rpm/mariner.dockerfile @@ -18,7 +18,6 @@ RUN dos2unix ./scripts/release/rpm/azure-cli.spec && \ rpmbuild -v -bb --clean scripts/release/rpm/azure-cli.spec && \ cp /usr/src/mariner/RPMS/**/azure-cli-${cli_version}-1.*.rpm /azure-cli-dev.rpm && \ mkdir /out && cp /usr/src/mariner/RPMS/**/azure-cli-${cli_version}-1.*.rpm /out/ - cp /usr/src/mariner/RPMS/x86_64/azure-cli-${cli_version}-1.cm2.x86_64.rpm /azure-cli-dev.rpm FROM ${image} AS execution-env From 2d478e3f28bd19712f53c53dbd38bd7e58d273f5 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Mon, 19 Dec 2022 16:00:48 +0800 Subject: [PATCH 17/26] Add install_docker.sh --- azure-pipelines.yml | 98 +++--------------------------------- scripts/ci/install_docker.sh | 15 ++++++ 2 files changed, 22 insertions(+), 91 deletions(-) create mode 100644 scripts/ci/install_docker.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 173705d4494..ba737fd07cc 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -298,19 +298,7 @@ jobs: pool: name: $(pool) steps: - - bash: | - if [[ $(dpkg --print-architecture) == "amd64" ]]; then - echo "Docker is already installed on AMD64" - exit 0 - fi - # https://docs.docker.com/engine/security/rootless/ - /bin/bash -c "$(curl -fsSL https://get.docker.com)" - sudo apt-get install -y uidmap - dockerd-rootless-setuptool.sh install - export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run - PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & - sleep 5 - docker context use rootless + - bash: ./scripts/ci/install_docker.sh displayName: Install Docker - task: Bash@3 displayName: 'Bash Script' @@ -359,19 +347,7 @@ jobs: TargetPath: '$(Build.ArtifactStagingDirectory)/docker' artifactName: $(artifactName) - - bash: | - if [[ $(dpkg --print-architecture) == "amd64" ]]; then - echo "Docker is already installed on AMD64" - exit 0 - fi - # https://docs.docker.com/engine/security/rootless/ - /bin/bash -c "$(curl -fsSL https://get.docker.com)" - sudo apt-get install -y uidmap - dockerd-rootless-setuptool.sh install - export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run - PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & - sleep 5 - docker context use rootless + - bash: ./scripts/ci/install_docker.sh displayName: Install Docker - bash: | @@ -687,19 +663,7 @@ jobs: pool: name: $(pool) steps: - - bash: | - if [[ $(dpkg --print-architecture) == "amd64" ]]; then - echo "Docker is already installed on AMD64" - exit 0 - fi - # https://docs.docker.com/engine/security/rootless/ - /bin/bash -c "$(curl -fsSL https://get.docker.com)" - sudo apt-get install -y uidmap - dockerd-rootless-setuptool.sh install - export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run - PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & - sleep 5 - docker context use rootless + - bash: ./scripts/ci/install_docker.sh displayName: Install Docker - task: Bash@3 @@ -754,19 +718,7 @@ jobs: python_package: python3 pool: ${{ arch.pool }} steps: - - bash: | - if [[ $(dpkg --print-architecture) == "amd64" ]]; then - echo "Docker is already installed on AMD64" - exit 0 - fi - # https://docs.docker.com/engine/security/rootless/ - /bin/bash -c "$(curl -fsSL https://get.docker.com)" - sudo apt-get install -y uidmap - dockerd-rootless-setuptool.sh install - export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run - PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & - sleep 5 - docker context use rootless + - bash: ./scripts/ci/install_docker.sh displayName: Install Docker - task: Bash@3 displayName: 'Build Rpm Package' @@ -841,19 +793,7 @@ jobs: TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' artifactName: $(artifact) - - bash: | - if [[ $(dpkg --print-architecture) == "amd64" ]]; then - echo "Docker is already installed on AMD64" - exit 0 - fi - # https://docs.docker.com/engine/security/rootless/ - /bin/bash -c "$(curl -fsSL https://get.docker.com)" - sudo apt-get install -y uidmap - dockerd-rootless-setuptool.sh install - export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run - PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & - sleep 5 - docker context use rootless + - bash: ./scripts/ci/install_docker.sh displayName: Install Docker - bash: | @@ -910,19 +850,7 @@ jobs: arch: ${{ arch.value }} pool: ${{ arch.pool }} steps: - - bash: | - if [[ $(dpkg --print-architecture) == "amd64" ]]; then - echo "Docker is already installed on AMD64" - exit 0 - fi - # https://docs.docker.com/engine/security/rootless/ - /bin/bash -c "$(curl -fsSL https://get.docker.com)" - sudo apt-get install -y uidmap - dockerd-rootless-setuptool.sh install - export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run - PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & - sleep 5 - docker context use rootless + - bash: ./scripts/ci/install_docker.sh displayName: Install Docker - task: Bash@3 displayName: 'Build $(deb_system) $(distro) $(arch) Package' @@ -975,19 +903,7 @@ jobs: pool: name: $(pool) steps: - - bash: | - if [[ $(dpkg --print-architecture) == "amd64" ]]; then - echo "Docker is already installed on AMD64" - exit 0 - fi - # https://docs.docker.com/engine/security/rootless/ - /bin/bash -c "$(curl -fsSL https://get.docker.com)" - sudo apt-get install -y uidmap - dockerd-rootless-setuptool.sh install - export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run - PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & - sleep 5 - docker context use rootless + - bash: ./scripts/ci/install_docker.sh displayName: Install Docker - task: DownloadPipelineArtifact@1 displayName: 'Download Metadata' diff --git a/scripts/ci/install_docker.sh b/scripts/ci/install_docker.sh new file mode 100644 index 00000000000..06afc6b2b10 --- /dev/null +++ b/scripts/ci/install_docker.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# ARM64 image does not have docker, install manually + +if [[ $(dpkg --print-architecture) == "amd64" ]]; then + echo "Docker is already installed on AMD64" + exit 0 +fi +# https://docs.docker.com/engine/security/rootless/ +/bin/bash -c "$(curl -fsSL https://get.docker.com)" +sudo apt-get install -y uidmap +dockerd-rootless-setuptool.sh install +export XDG_RUNTIME_DIR=/home/cloudtest/.docker/run +PATH=/usr/bin:/sbin:/usr/sbin:$PATH dockerd-rootless.sh & +sleep 5 +docker context use rootless From 3f0b95852e5c06fd2cd21082e2f413eb555d9af7 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Mon, 19 Dec 2022 16:01:04 +0800 Subject: [PATCH 18/26] Minor fix --- scripts/release/debian/test_deb_in_docker.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/release/debian/test_deb_in_docker.sh b/scripts/release/debian/test_deb_in_docker.sh index 3821ad008ec..003bdafd7ce 100644 --- a/scripts/release/debian/test_deb_in_docker.sh +++ b/scripts/release/debian/test_deb_in_docker.sh @@ -8,8 +8,7 @@ export USERNAME=azureuser apt update apt install -y apt-transport-https git gcc python3-dev -DEB_NAME=$(find /mnt/artifacts -type f -name "azure-cli_$CLI_VERSION-1~${DISTRO}_*.deb") -dpkg -i $DEB_NAME +dpkg -i /mnt/artifacts/azure-cli_$CLI_VERSION-1~${DISTRO}_*.deb time az self-test time az --version From 3dfdc69cd35a8593ac0afbbb30c73bdf0c4844ab Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Mon, 19 Dec 2022 17:08:03 +0800 Subject: [PATCH 19/26] Fix permission --- scripts/ci/install_docker.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/ci/install_docker.sh diff --git a/scripts/ci/install_docker.sh b/scripts/ci/install_docker.sh old mode 100644 new mode 100755 From 11e2a2df0d953d13ee011435f69b630868f2d7e2 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Tue, 20 Dec 2022 10:52:28 +0800 Subject: [PATCH 20/26] Remove %dist define --- scripts/release/rpm/azure-cli.spec | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scripts/release/rpm/azure-cli.spec b/scripts/release/rpm/azure-cli.spec index da46cdfd10e..b8b87417071 100644 --- a/scripts/release/rpm/azure-cli.spec +++ b/scripts/release/rpm/azure-cli.spec @@ -5,11 +5,6 @@ # Turn off python byte compilation %global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g') -# .el7.centos -> .el7 -%if 0%{?rhel} - %define dist .el%{?rhel} -%endif - # The Python package name for dnf/yum/tdnf, such as python39, python3 %define python_package %{getenv:PYTHON_PACKAGE} # The Python executable name, such as python3.9, python3 From 4225271ba60c21523510688fc285f344f2d38c59 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Tue, 20 Dec 2022 10:55:50 +0800 Subject: [PATCH 21/26] Use yum/dnf install package --- scripts/release/rpm/fedora.dockerfile | 2 +- scripts/release/rpm/mariner.dockerfile | 3 +-- scripts/release/rpm/ubi.dockerfile | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/release/rpm/fedora.dockerfile b/scripts/release/rpm/fedora.dockerfile index 9a6dd4ef80b..bc950fecd2f 100644 --- a/scripts/release/rpm/fedora.dockerfile +++ b/scripts/release/rpm/fedora.dockerfile @@ -20,5 +20,5 @@ RUN dos2unix ./scripts/release/rpm/azure-cli.spec && \ FROM ${image} AS execution-env COPY --from=build-env /azure-cli-dev.rpm ./ -RUN rpm -i ./azure-cli-dev.rpm && \ +RUN dnf install -y ./azure-cli-dev.rpm && \ az --version diff --git a/scripts/release/rpm/mariner.dockerfile b/scripts/release/rpm/mariner.dockerfile index 8ac3031efac..c24e9bae152 100644 --- a/scripts/release/rpm/mariner.dockerfile +++ b/scripts/release/rpm/mariner.dockerfile @@ -22,8 +22,7 @@ RUN dos2unix ./scripts/release/rpm/azure-cli.spec && \ FROM ${image} AS execution-env RUN tdnf update -y -RUN tdnf install -y python3 rpm COPY --from=build-env /azure-cli-dev.rpm ./ -RUN rpm -i ./azure-cli-dev.rpm && \ +RUN tdnf install -y ./azure-cli-dev.rpm && \ az --version diff --git a/scripts/release/rpm/ubi.dockerfile b/scripts/release/rpm/ubi.dockerfile index e3c0a857123..e969fe6b2c8 100644 --- a/scripts/release/rpm/ubi.dockerfile +++ b/scripts/release/rpm/ubi.dockerfile @@ -25,8 +25,7 @@ RUN dos2unix ./scripts/release/rpm/azure-cli.spec && \ FROM ${image} AS execution-env RUN yum update -y -RUN yum install -y python39 COPY --from=build-env /azure-cli-dev.rpm ./ -RUN rpm -i ./azure-cli-dev.rpm && \ +RUN yum install -y ./azure-cli-dev.rpm && \ az --version From ee61fa70aa45ec22d7cb1fdd57d70c87890eff9f Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Tue, 3 Jan 2023 18:27:58 +0800 Subject: [PATCH 22/26] Minor fix --- scripts/release/rpm/fedora.dockerfile | 4 ++-- scripts/release/rpm/ubi.dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/release/rpm/fedora.dockerfile b/scripts/release/rpm/fedora.dockerfile index bc950fecd2f..66f0295af42 100644 --- a/scripts/release/rpm/fedora.dockerfile +++ b/scripts/release/rpm/fedora.dockerfile @@ -14,8 +14,8 @@ COPY . . RUN dos2unix ./scripts/release/rpm/azure-cli.spec && \ REPO_PATH=$(pwd) CLI_VERSION=$cli_version PYTHON_PACKAGE=$python_package PYTHON_CMD=python3 \ rpmbuild -v -bb --clean scripts/release/rpm/azure-cli.spec && \ - cp /root/rpmbuild/RPMS/**/azure-cli-${cli_version}-1.*.rpm /azure-cli-dev.rpm && \ - mkdir /out && cp /root/rpmbuild/RPMS/**/azure-cli-${cli_version}-1.*.rpm /out/ + cp /root/rpmbuild/RPMS/*/azure-cli-${cli_version}-1.*.rpm /azure-cli-dev.rpm && \ + mkdir /out && cp /root/rpmbuild/RPMS/*/azure-cli-${cli_version}-1.*.rpm /out/ FROM ${image} AS execution-env diff --git a/scripts/release/rpm/ubi.dockerfile b/scripts/release/rpm/ubi.dockerfile index e969fe6b2c8..558d0e85e2e 100644 --- a/scripts/release/rpm/ubi.dockerfile +++ b/scripts/release/rpm/ubi.dockerfile @@ -19,8 +19,8 @@ COPY . . RUN dos2unix ./scripts/release/rpm/azure-cli.spec && \ REPO_PATH=$(pwd) CLI_VERSION=$cli_version PYTHON_PACKAGE=$python_package PYTHON_CMD=python3.9 \ rpmbuild -v -bb --clean scripts/release/rpm/azure-cli.spec && \ - cp /root/rpmbuild/RPMS/**/azure-cli-${cli_version}-1.*.rpm /azure-cli-dev.rpm && \ - mkdir /out && cp /root/rpmbuild/RPMS/**/azure-cli-${cli_version}-1.*.rpm /out/ + cp /root/rpmbuild/RPMS/*/azure-cli-${cli_version}-1.*.rpm /azure-cli-dev.rpm && \ + mkdir /out && cp /root/rpmbuild/RPMS/*/azure-cli-${cli_version}-1.*.rpm /out/ FROM ${image} AS execution-env From 26a4ba62fb0087a5a819311b22e1e1b2ebfb80a7 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Tue, 3 Jan 2023 18:30:57 +0800 Subject: [PATCH 23/26] Minor fix --- scripts/release/rpm/mariner.dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/release/rpm/mariner.dockerfile b/scripts/release/rpm/mariner.dockerfile index c24e9bae152..eb29b07711e 100644 --- a/scripts/release/rpm/mariner.dockerfile +++ b/scripts/release/rpm/mariner.dockerfile @@ -16,8 +16,8 @@ COPY . . RUN dos2unix ./scripts/release/rpm/azure-cli.spec && \ REPO_PATH=$(pwd) CLI_VERSION=$cli_version PYTHON_PACKAGE=python3 PYTHON_CMD=python3 \ rpmbuild -v -bb --clean scripts/release/rpm/azure-cli.spec && \ - cp /usr/src/mariner/RPMS/**/azure-cli-${cli_version}-1.*.rpm /azure-cli-dev.rpm && \ - mkdir /out && cp /usr/src/mariner/RPMS/**/azure-cli-${cli_version}-1.*.rpm /out/ + cp /usr/src/mariner/RPMS/*/azure-cli-${cli_version}-1.*.rpm /azure-cli-dev.rpm && \ + mkdir /out && cp /usr/src/mariner/RPMS/*/azure-cli-${cli_version}-1.*.rpm /out/ FROM ${image} AS execution-env From 7d44f648ac422de2217af47491f271e616b2ea50 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Wed, 4 Jan 2023 16:16:53 +0800 Subject: [PATCH 24/26] Minor fix --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ba737fd07cc..511f076fbfe 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -21,10 +21,10 @@ parameters: default: - name: AMD64 value: amd64 - pool: pool-ubuntu-2004 + pool: ${{ variables.ubuntu_pool }} - name: ARM64 value: arm64 - pool: ubuntu-arm64-2004-pool + pool: ${{ variables.ubuntu_arm64_pool }} jobs: - job: CheckPullRequest From 6d8a840ae88ed9cd1fb7847d4cfe15ca385beaa6 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Mon, 16 Jan 2023 15:00:56 +0800 Subject: [PATCH 25/26] Revert "Minor fix" This reverts commit 7d44f648ac422de2217af47491f271e616b2ea50. --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 511f076fbfe..ba737fd07cc 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -21,10 +21,10 @@ parameters: default: - name: AMD64 value: amd64 - pool: ${{ variables.ubuntu_pool }} + pool: pool-ubuntu-2004 - name: ARM64 value: arm64 - pool: ${{ variables.ubuntu_arm64_pool }} + pool: ubuntu-arm64-2004-pool jobs: - job: CheckPullRequest From 92b4d74da138c81fbc9ec489c73e2cddf8e84d30 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Mon, 13 Feb 2023 15:37:47 +0800 Subject: [PATCH 26/26] Minor fix --- scripts/release/debian/prepare.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/release/debian/prepare.sh b/scripts/release/debian/prepare.sh index 506b5896236..9a3c314cf49 100755 --- a/scripts/release/debian/prepare.sh +++ b/scripts/release/debian/prepare.sh @@ -110,8 +110,7 @@ ${TAB}echo "\043!/usr/bin/env bash\nbin_dir=\140cd \"\044(dirname \"\044BASH_SOU ${TAB}chmod 0755 debian/azure-cli/usr/bin/az ${TAB}mkdir -p debian/azure-cli/etc/bash_completion.d/ ${TAB}cat ${completion_script} > debian/azure-cli/etc/bash_completion.d/azure-cli -${TAB}ssl_file=$(find debian/azure-cli/opt/az/lib/python3.10/ -type f -name '_ssl.cpython-310-*-linux-gnu.so') -${TAB}dpkg-shlibdeps -v --warnings=7 -Tdebian/azure-cli.substvars -dDepends -edebian/azure-cli/opt/az/bin/python3 $ssl_file +${TAB}dpkg-shlibdeps -v --warnings=7 -Tdebian/azure-cli.substvars -dDepends -edebian/azure-cli/opt/az/bin/python3 debian/azure-cli/opt/az/lib/python3.10/lib-dynload/_ssl.cpython-310-*-linux-gnu.so override_dh_strip: