From e2219f7433818880e089e5c8cd7386e5a8960625 Mon Sep 17 00:00:00 2001 From: Mike McLaughlin Date: Tue, 2 May 2023 18:55:25 -0700 Subject: [PATCH 1/2] Fix native build Update dependencies from https://github.com/dotnet/arcade build 20230424.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.CodeAnalysis From Version 8.0.0-beta.23168.1 -> To Version 8.0.0-beta.23224.1 Dependency coherency updates Microsoft.SourceLink.GitHub From Version 1.2.0-beta-23165-02 -> To Version 8.0.0-beta.23218.3 (parent: Microsoft.DotNet.Arcade.Sdk Update dependencies from https://github.com/dotnet/arcade build 20230505.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.CodeAnalysis From Version 8.0.0-beta.23168.1 -> To Version 8.0.0-beta.23255.2 Dependency coherency updates Microsoft.SourceLink.GitHub From Version 1.2.0-beta-23165-02 -> To Version 8.0.0-beta.23218.3 (parent: Microsoft.DotNet.Arcade.Sdk Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20230512.1 Microsoft.SourceBuild.Intermediate.source-build-reference-packages From Version 8.0.0-alpha.1.23258.2 -> To Version 8.0.0-alpha.1.23262.1 Update dependencies from https://github.com/dotnet/arcade build 20230512.5 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.CodeAnalysis From Version 8.0.0-beta.23168.1 -> To Version 8.0.0-beta.23262.5 Dependency coherency updates Microsoft.SourceLink.GitHub From Version 1.2.0-beta-23165-02 -> To Version 8.0.0-beta.23218.3 (parent: Microsoft.DotNet.Arcade.Sdk Update dependencies from https://github.com/dotnet/installer build 20230514.2 Microsoft.Dotnet.Sdk.Internal From Version 8.0.100-preview.3.23156.1 -> To Version 8.0.100-preview.5.23264.2 Update dependencies from https://github.com/dotnet/runtime build 20230515.1 Microsoft.NETCore.App.Runtime.win-x64 , VS.Redist.Common.NetCore.SharedFramework.x64.8.0 From Version 8.0.0-preview.3.23155.1 -> To Version 8.0.0-preview.5.23265.1 Update dependencies from https://github.com/dotnet/aspnetcore build 20230515.1 Microsoft.AspNetCore.App.Ref , Microsoft.AspNetCore.App.Ref.Internal From Version 8.0.0-preview.4.23213.2 -> To Version 8.0.0-preview.5.23265.1 --- Build.cmd | 2 +- build.sh | 2 +- diagnostics-codeql.yml | 231 +++--- diagnostics.yml | 680 ++++++++++-------- eng/CIBuild.cmd | 3 - eng/Version.Details.xml | 32 +- eng/Versions.props | 22 +- eng/build.ps1 | 2 +- eng/build.sh | 50 +- eng/ci-prepare-artifacts.cmd | 2 +- eng/cibuild.sh | 43 -- eng/common/cross/build-rootfs.sh | 166 +++-- eng/common/cross/toolchain.cmake | 52 +- eng/common/native/init-compiler.sh | 2 +- eng/common/templates/job/job.yml | 17 +- .../templates/steps/component-governance.yml | 13 + eng/common/tools.ps1 | 28 +- eng/common/tools.sh | 35 +- eng/docker-build.sh | 79 -- eng/{ => pipelines}/build.yml | 111 +-- eng/pipelines/pipeline-resources.yml | 61 ++ eng/{ => pipelines}/prepare-release.yml | 0 global.json | 4 +- .../ClrMDHelper.cs | 36 +- .../DumpGenCommand.cs | 28 +- .../DumpStackObjectsCommand.cs | 2 +- .../GCGeneration.cs | 3 +- .../Unix/Debugger.Tests.Config.txt | 12 +- .../Windows/Debugger.Tests.Config.txt | 14 +- src/SOS/SOS.UnitTests/SOS.cs | 10 +- src/SOS/SOS.UnitTests/SOSRunner.cs | 4 + src/SOS/SOS.UnitTests/Scripts/DumpGen.script | 2 +- src/SOS/lldbplugin/CMakeLists.txt | 10 +- 33 files changed, 978 insertions(+), 780 deletions(-) delete mode 100644 eng/CIBuild.cmd delete mode 100755 eng/cibuild.sh create mode 100644 eng/common/templates/steps/component-governance.yml delete mode 100755 eng/docker-build.sh rename eng/{ => pipelines}/build.yml (73%) create mode 100644 eng/pipelines/pipeline-resources.yml rename eng/{ => pipelines}/prepare-release.yml (100%) diff --git a/Build.cmd b/Build.cmd index 3de9ae405f..0a7b2aa973 100644 --- a/Build.cmd +++ b/Build.cmd @@ -1,3 +1,3 @@ @echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\build.ps1""" -restore %*" +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\build.ps1""" -restore -build %*" exit /b %ErrorLevel% diff --git a/build.sh b/build.sh index 58a2abb50f..75055e9c92 100755 --- a/build.sh +++ b/build.sh @@ -13,4 +13,4 @@ while [[ -h $source ]]; do done scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" -"$scriptroot/eng/build.sh" --restore $@ +"$scriptroot/eng/build.sh" --restore --build $@ diff --git a/diagnostics-codeql.yml b/diagnostics-codeql.yml index ee506ce6de..5cacd47b62 100644 --- a/diagnostics-codeql.yml +++ b/diagnostics-codeql.yml @@ -28,125 +28,128 @@ variables: - name: skipComponentGovernanceDetection value: true -stages: -- stage: build - displayName: Build and Test Diagnostics - jobs: - - template: /eng/build.yml - parameters: - name: Windows - osGroup: Windows_NT - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - Build_Release_x86: - _BuildConfig: Release - _BuildArch: x86 - Build_Release_arm: - _BuildConfig: Release - _BuildArch: arm - Build_Release_arm64: - _BuildConfig: Release - _BuildArch: arm64 +extends: + template: /eng/pipelines/pipeline-resources.yml + parameters: + stages: + - stage: build + displayName: Build and Test Diagnostics + jobs: + - template: /eng/pipelines/build.yml + parameters: + name: Windows + osGroup: Windows_NT + isCodeQLRun: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 + Build_Release_x86: + _BuildConfig: Release + _BuildArch: x86 + Build_Release_arm: + _BuildConfig: Release + _BuildArch: arm + Build_Release_arm64: + _BuildConfig: Release + _BuildArch: arm64 - - template: /eng/build.yml - parameters: - name: CentOS_7 - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343 - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 + - template: /eng/pipelines/build.yml + parameters: + name: Linux_x64 + osGroup: Linux + nativeBuildContainer: linux_x64 + isCodeQLRun: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 - - template: /eng/build.yml - parameters: - name: Alpine3_13 - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.13-WithNode-20210910135845-c401c85 - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 + - template: /eng/pipelines/build.yml + parameters: + name: Linux_musl + osGroup: Linux + nativeBuildContainer: linux_musl_x64 + isCodeQLRun: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 - - template: /eng/build.yml - parameters: - name: MacOS - osGroup: MacOS - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 + - template: /eng/pipelines/build.yml + parameters: + name: MacOS + osGroup: MacOS + isCodeQLRun: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 - - template: /eng/build.yml - parameters: - name: MacOS_arm64 - osGroup: MacOS_cross - crossbuild: true - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 + - template: /eng/pipelines/build.yml + parameters: + name: MacOS_arm64 + osGroup: MacOS_cross + crossBuild: true + isCodeQLRun: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm64 - - template: /eng/build.yml - parameters: - name: Linux_arm - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-20210719121212-8a8d3be - crossrootfsDir: '/crossrootfs/arm' - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm + - template: /eng/pipelines/build.yml + parameters: + name: Linux_arm + osGroup: Linux + nativeBuildContainer: linux_arm + crossBuild: true + isCodeQLRun: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm - - template: /eng/build.yml - parameters: - name: Linux_arm64 - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-20210719121212-8a8d3be - crossrootfsDir: '/crossrootfs/arm64' - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 + - template: /eng/pipelines/build.yml + parameters: + name: Linux_arm64 + osGroup: Linux + nativeBuildContainer: linux_arm64 + crossBuild: true + isCodeQLRun: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm64 - - template: /eng/build.yml - parameters: - name: Linux_musl_arm - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm-alpine-20210923140502-78f7860 - crossrootfsDir: '/crossrootfs/arm' - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm + - template: /eng/pipelines/build.yml + parameters: + name: Linux_musl_arm + osGroup: Linux + nativeBuildContainer: linux_musl_arm + crossBuild: true + isCodeQLRun: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm - - template: /eng/build.yml - parameters: - name: Linux_musl_arm64 - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-alpine-20210923140502-78f7860 - crossrootfsDir: '/crossrootfs/arm64' - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 \ No newline at end of file + - template: /eng/pipelines/build.yml + parameters: + name: Linux_musl_arm64 + osGroup: Linux + nativeBuildContainer: linux_musl_arm64 + crossBuild: true + isCodeQLRun: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm64 diff --git a/diagnostics.yml b/diagnostics.yml index df452f23b1..8fa4c3c2d6 100644 --- a/diagnostics.yml +++ b/diagnostics.yml @@ -70,344 +70,394 @@ variables: - name: RuntimeFeedBase64SasToken value: $(dotnetclimsrc-read-sas-token-base64) -stages: - - stage: build - displayName: Build and Test Diagnostics - jobs: +extends: + template: /eng/pipelines/pipeline-resources.yml + parameters: + stages: + - stage: build + displayName: Build and Test Diagnostics + jobs: - ############################ - # # - # Source Build legs # - # # - ############################ + ############################ + # # + # Source Build legs # + # # + ############################ - - template: /eng/common/templates/job/source-build.yml - parameters: - platform: - name: Complete - buildScript: ./eng/common/build.sh + - template: /eng/common/templates/job/source-build.yml + parameters: + platform: + name: Complete + buildScript: ./eng/common/build.sh - ############################ - # # - # Build legs # - # # - ############################ + ############################ + # # + # Build legs # + # # + ############################ - - template: /eng/build.yml - parameters: - name: Windows - osGroup: Windows_NT - strategy: - matrix: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - _PublishArtifacts: bin - Build_Release_x86: - _BuildConfig: Release - _BuildArch: x86 - _PublishArtifacts: bin/Windows_NT.x86.Release - ${{ if ne(variables['System.TeamProject'], 'public') }}: - Build_Release_arm: - _BuildConfig: Release - _BuildArch: arm - _PublishArtifacts: bin/Windows_NT.arm.Release - Build_Release_arm64: - _BuildConfig: Release - _BuildArch: arm64 - _PublishArtifacts: bin/Windows_NT.arm64.Release + - template: /eng/pipelines/build.yml + parameters: + name: Windows + osGroup: Windows_NT + strategy: + matrix: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 + Build_Release: + _BuildConfig: Release + _BuildArch: x64 + _PublishArtifacts: bin + Build_Release_x86: + _BuildConfig: Release + _BuildArch: x86 + _PublishArtifacts: bin/Windows_NT.x86.Release + ${{ if ne(variables['System.TeamProject'], 'public') }}: + Build_Release_arm: + _BuildConfig: Release + _BuildArch: arm + _PublishArtifacts: bin/Windows_NT.arm.Release + Build_Release_arm64: + _BuildConfig: Release + _BuildArch: arm64 + _PublishArtifacts: bin/Windows_NT.arm64.Release - - template: /eng/build.yml - parameters: - name: CentOS_7 - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7 - requiresCapPtraceContainer: true - buildAndSkipTest: true - strategy: - matrix: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 - _PublishArtifacts: bin/Linux.x64.Debug - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - _PublishArtifacts: bin/Linux.x64.Release + - template: /eng/pipelines/build.yml + parameters: + name: Linux_x64 + osGroup: Linux + nativeBuildContainer: linux_x64 + buildOnly: true + strategy: + matrix: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 + _PublishArtifacts: bin/Linux.x64.Debug + Build_Release: + _BuildConfig: Release + _BuildArch: x64 + _PublishArtifacts: bin/Linux.x64.Release - - template: /eng/build.yml - parameters: - name: Alpine3_13 - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.13-WithNode - artifactsTargetPath: bin/Linux-musl.x64.Release - requiresCapPtraceContainer: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - _PublishArtifacts: bin/Linux.x64.Release - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 + - template: /eng/pipelines/build.yml + parameters: + name: Linux_musl + osGroup: Linux + osSuffix: -musl + nativeBuildContainer: linux_musl_x64 + buildOnly: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 + _PublishArtifacts: bin/Linux.x64.Release + _ArtifactsTargetPath: bin/Linux-musl.x64.Release + ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 + _PublishArtifacts: bin/Linux.x64.Debug + _ArtifactsTargetPath: bin/Linux-musl.x64.Debug - - template: /eng/build.yml - parameters: - name: MacOS - osGroup: MacOS - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - _PublishArtifacts: bin/OSX.x64.Release - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 + - template: /eng/pipelines/build.yml + parameters: + name: MacOS + osGroup: MacOS + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 + _PublishArtifacts: bin/OSX.x64.Release + ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 - - template: /eng/build.yml - parameters: - name: MacOS_arm64 - osGroup: MacOS_cross - crossbuild: true - buildAndSkipTest: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 - _PublishArtifacts: bin/OSX.arm64.Release - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - Build_Debug: - _BuildConfig: Debug - _BuildArch: arm64 + - template: /eng/pipelines/build.yml + parameters: + name: MacOS_arm64 + osGroup: MacOS_cross + crossBuild: true + buildOnly: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm64 + _PublishArtifacts: bin/OSX.arm64.Release + ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + Build_Debug: + _BuildConfig: Debug + _BuildArch: arm64 + + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/pipelines/build.yml + parameters: + name: Linux_arm + osGroup: Linux + nativeBuildContainer: linux_arm + crossBuild: true + buildOnly: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm + _PublishArtifacts: bin/Linux.arm.Release - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - template: /eng/build.yml - parameters: - name: Linux_arm - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross - crossrootfsDir: '/crossrootfs/arm' - buildAndSkipTest: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm - _PublishArtifacts: bin/Linux.arm.Release + - template: /eng/pipelines/build.yml + parameters: + name: Linux_arm64 + osGroup: Linux + nativeBuildContainer: linux_arm64 + crossBuild: true + buildOnly: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm64 + _PublishArtifacts: bin/Linux.arm64.Release - - template: /eng/build.yml - parameters: - name: Linux_arm64 - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64 - crossrootfsDir: '/crossrootfs/arm64' - buildAndSkipTest: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 - _PublishArtifacts: bin/Linux.arm64.Release + - template: /eng/pipelines/build.yml + parameters: + name: Linux_musl_arm + osGroup: Linux + osSuffix: -musl + nativeBuildContainer: linux_musl_arm + crossBuild: true + buildOnly: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm + _PublishArtifacts: bin/Linux.arm.Release + _ArtifactsTargetPath: bin/Linux-musl.arm.Release - - template: /eng/build.yml - parameters: - name: Linux_musl_arm - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm-alpine - crossrootfsDir: '/crossrootfs/arm' - artifactsTargetPath: bin/Linux-musl.arm.Release - buildAndSkipTest: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm - _PublishArtifacts: bin/Linux.arm.Release + - template: /eng/pipelines/build.yml + parameters: + name: Linux_musl_arm64 + osGroup: Linux + osSuffix: -musl + nativeBuildContainer: linux_musl_arm64 + crossBuild: true + buildOnly: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm64 + _PublishArtifacts: bin/Linux.arm64.Release + _ArtifactsTargetPath: bin/Linux-musl.arm64.Release - - template: /eng/build.yml - parameters: - name: Linux_musl_arm64 - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-alpine - crossrootfsDir: '/crossrootfs/arm64' - artifactsTargetPath: bin/Linux-musl.arm64.Release - buildAndSkipTest: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 - _PublishArtifacts: bin/Linux.arm64.Release + ############################ + # # + # Test only legs # + # # + ############################ - ############################ - # # - # Test only legs # - # # - ############################ + - template: /eng/pipelines/build.yml + parameters: + name: Alpine3_13 + osGroup: Linux + osSuffix: -musl + container: test_linux_musl_x64 + dependsOn: Linux_musl + testOnly: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 + ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 - - template: /eng/build.yml - parameters: - name: Debian_Bullseye - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 - dependsOn: CentOS_7 - testOnly: true - strategy: - matrix: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 + - template: /eng/pipelines/build.yml + parameters: + name: Debian_Bullseye + osGroup: Linux + container: test_debian_11_amd64 + dependsOn: Linux_x64 + testOnly: true + strategy: + matrix: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 -# - template: /eng/build.yml -# parameters: -# name: Fedora_34 -# osGroup: Linux -# dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-34-helix -# dependsOn: CentOS_7 -# testOnly: true -# requiresCapPtraceContainer: true -# strategy: -# matrix: -# Build_Debug: -# _BuildConfig: Debug -# _BuildArch: x64 + - template: /eng/pipelines/build.yml + parameters: + name: Fedora_36 + osGroup: Linux + container: test_fedora_36 + dependsOn: Linux_x64 + testOnly: true + strategy: + matrix: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 -# - template: /eng/build.yml -# parameters: -# name: OpenSuse_15_2 -# osGroup: Linux -# dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:opensuse-15.2-helix-amd64 -# dependsOn: CentOS_7 -# testOnly: true -# strategy: -# matrix: -# Build_Debug: -# _BuildConfig: Debug -# _BuildArch: x64 + #- template: /eng/pipelines/build.yml + # parameters: + # name: OpenSuse_15_2 + # osGroup: Linux + # container: test_opensuse_15_2 + # dependsOn: Linux_x64 + # testOnly: true + # strategy: + # matrix: + # Build_Debug: + # _BuildConfig: Debug + # _BuildArch: x64 - - template: /eng/build.yml - parameters: - name: Ubuntu_18_04 - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04 - dependsOn: CentOS_7 - testOnly: true - strategy: - matrix: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 + #- template: /eng/pipelines/build.yml + # parameters: + # name: Ubuntu_18_04 + # osGroup: Linux + # container: test_ubuntu_18_04 + # dependsOn: Linux_x64 + # testOnly: true + # strategy: + # matrix: + # Build_Debug: + # _BuildConfig: Debug + # _BuildArch: x64 - # Download, sign, package and publish - - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - - template: /eng/common/templates/job/job.yml + - template: /eng/pipelines/build.yml parameters: - name: Sign_Package_Publish - displayName: Sign, Package, and Generate BAR Manifests - dependsOn: - - Windows - - CentOS_7 - - Alpine3_13 - - MacOS - - MacOS_arm64 - - Linux_arm - - Linux_arm64 - - Linux_musl_arm - - Linux_musl_arm64 - condition: succeeded() - pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals windows.vs2022.amd64 - enablePublishUsingPipelines: true - enableMicrobuild: true - artifacts: - publish: - logs: - name: Logs_Packaging_Signing - steps: - - task: DownloadBuildArtifacts@0 - displayName: 'Download release builds' - inputs: - downloadPath: '$(Build.ArtifactStagingDirectory)/__download__' - artifactName: Build_Release - checkDownloadedFiles: true - - task: CopyFiles@2 - displayName: 'Binplace Product' - inputs: - sourceFolder: $(Build.ArtifactStagingDirectory)/__download__/Build_Release - targetFolder: '$(Build.SourcesDirectory)/artifacts/' + name: Ubuntu_20_04 + osGroup: Linux + container: test_ubuntu_20_04 + dependsOn: Linux_x64 + testOnly: true + strategy: + matrix: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 + + #- template: /eng/pipelines/build.yml + # parameters: + # name: Ubuntu_22_04 + # osGroup: Linux + # container: test_ubuntu_22_04 + # dependsOn: Linux_x64 + # testOnly: true + # strategy: + # matrix: + # Build_Debug: + # _BuildConfig: Debug + # _BuildArch: x64 - # Windows x64 download. Everything under "bin" is published for the Windows x64 build. - # Create nuget packages, sign binaries and publish to blob feed - - script: $(Build.SourcesDirectory)\eng\ci-prepare-artifacts.cmd $(_InternalBuildArgs) - displayName: Package, Sign, and Publish - continueOnError: false + # Download, sign, package and publish + - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/common/templates/job/job.yml + parameters: + name: Sign_Package_Publish + displayName: Sign, Package, and Generate BAR Manifests + dependsOn: + - Windows + - MacOS + - MacOS_arm64 + - Linux_x64 + - Linux_musl + - Linux_arm + - Linux_arm64 + - Linux_musl_arm + - Linux_musl_arm64 condition: succeeded() + pool: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals windows.vs2022.amd64 + enablePublishUsingPipelines: true + enableMicrobuild: true + artifacts: + publish: + logs: + name: Logs_Packaging_Signing + steps: + - task: DownloadBuildArtifacts@0 + displayName: 'Download release builds' + inputs: + downloadPath: '$(Build.ArtifactStagingDirectory)/__download__' + artifactName: Build_Release + checkDownloadedFiles: true + - task: CopyFiles@2 + displayName: 'Binplace Product' + inputs: + sourceFolder: $(Build.ArtifactStagingDirectory)/__download__/Build_Release + targetFolder: '$(Build.SourcesDirectory)/artifacts/' - # Publish package and log build artifacts - - task: PublishBuildArtifacts@1 - displayName: Publish Package Artifacts - inputs: - publishLocation: Container - pathtoPublish: '$(Build.SourcesDirectory)/artifacts/packages' - artifactName: Packages - continueOnError: true - condition: always() + # Windows x64 download. Everything under "bin" is published for the Windows x64 build. + # Create nuget packages, sign binaries and publish to blob feed + - script: $(Build.SourcesDirectory)\eng\ci-prepare-artifacts.cmd $(_InternalBuildArgs) + displayName: Package, Sign, and Publish + continueOnError: false + condition: succeeded() - - task: PublishBuildArtifacts@1 - displayName: Publish Bundled Tools - inputs: - publishLocation: Container - pathtoPublish: '$(Build.SourcesDirectory)/artifacts/bundledtools' - artifactName: BundledTools - continueOnError: true - condition: always() + # Publish package and log build artifacts + - task: PublishBuildArtifacts@1 + displayName: Publish Package Artifacts + inputs: + publishLocation: Container + pathtoPublish: '$(Build.SourcesDirectory)/artifacts/packages' + artifactName: Packages + continueOnError: true + condition: always() - - template: /eng/common/templates/job/publish-build-assets.yml - parameters: - configuration: Release - dependsOn: Sign_Package_Publish - publishUsingPipelines: true - pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals windows.vs2022.amd64 + - task: PublishBuildArtifacts@1 + displayName: Publish Bundled Tools + inputs: + publishLocation: Container + pathtoPublish: '$(Build.SourcesDirectory)/artifacts/bundledtools' + artifactName: BundledTools + continueOnError: true + condition: always() - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/common/templates/post-build/post-build.yml - parameters: - # This is to enable SDL runs part of Post-Build Validation Stage. - # as well as NuGet, SourceLink, and signing validation. - # The variables get imported from group dotnet-diagnostics-sdl-params - publishingInfraVersion: 3 - enableSourceLinkValidation: true - enableSigningValidation: false - enableSymbolValidation: false - enableNugetValidation: true - symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=false' - publishInstallersAndChecksums: true - SDLValidationParameters: - enable: true - continueOnError: true - params: ' -SourceToolsList @("policheck","credscan") - -TsaInstanceURL $(_TsaInstanceURL) - -TsaProjectName $(_TsaProjectName) - -TsaNotificationEmail $(_TsaNotificationEmail) - -TsaCodebaseAdmin $(_TsaCodebaseAdmin) - -TsaBugAreaPath $(_TsaBugAreaPath) - -TsaIterationPath $(_TsaIterationPath) - -TsaRepositoryName "diagnostics" - -TsaCodebaseName "diagnostics" - -TsaPublish $True' - artifactNames: - - 'Packages' + - template: /eng/common/templates/job/publish-build-assets.yml + parameters: + configuration: Release + dependsOn: Sign_Package_Publish + publishUsingPipelines: true + pool: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals windows.vs2022.amd64 + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/common/templates/post-build/post-build.yml + parameters: + # This is to enable SDL runs part of Post-Build Validation Stage. + # as well as NuGet, SourceLink, and signing validation. + # The variables get imported from group dotnet-diagnostics-sdl-params + publishingInfraVersion: 3 + enableSourceLinkValidation: true + enableSigningValidation: false + enableSymbolValidation: false + enableNugetValidation: true + symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=false' + publishInstallersAndChecksums: true + SDLValidationParameters: + enable: true + continueOnError: true + params: ' -SourceToolsList @("policheck","credscan") + -TsaInstanceURL $(_TsaInstanceURL) + -TsaProjectName $(_TsaProjectName) + -TsaNotificationEmail $(_TsaNotificationEmail) + -TsaCodebaseAdmin $(_TsaCodebaseAdmin) + -TsaBugAreaPath $(_TsaBugAreaPath) + -TsaIterationPath $(_TsaIterationPath) + -TsaRepositoryName "diagnostics" + -TsaCodebaseName "diagnostics" + -TsaPublish $True' + artifactNames: + - 'Packages' - # This sets up the bits to do a Release. - - template: /eng/prepare-release.yml + # This sets up the bits to do a Release. + - template: /eng/pipelines/prepare-release.yml diff --git a/eng/CIBuild.cmd b/eng/CIBuild.cmd deleted file mode 100644 index df9ae6479c..0000000000 --- a/eng/CIBuild.cmd +++ /dev/null @@ -1,3 +0,0 @@ -@echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0build.ps1""" -restore -ci -prepareMachine %*" -exit /b %ErrorLevel% diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index dd1dfa94df..0718fc1b4d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -14,47 +14,47 @@ - + https://github.com/dotnet/arcade - b12f035e893c34ec2c965d75f6e21b7a2667e98d + 1aff4eb33aa7cbf26ccd9fc43c17cb609a14dad4 - + https://github.com/dotnet/arcade - b12f035e893c34ec2c965d75f6e21b7a2667e98d + 1aff4eb33aa7cbf26ccd9fc43c17cb609a14dad4 https://github.com/dotnet/arcade ccfe6da198c5f05534863bbb1bff66e830e0c6ab - + https://github.com/dotnet/installer - 51e06f6931e859f56564556fa6ba519761fa7141 + 8488614afec7497b9e74b47c976af2a97506bb96 - + https://github.com/dotnet/aspnetcore - 68ae6b0d8aa2f4a0ff189d5cedc741e32cc643d2 + dba677f5fbf85666aac3009b16e5c05d93ec63ca - + https://github.com/dotnet/aspnetcore - 68ae6b0d8aa2f4a0ff189d5cedc741e32cc643d2 + dba677f5fbf85666aac3009b16e5c05d93ec63ca - + https://github.com/dotnet/runtime - a64420c79cb63c485138f4f1352b8730f27d7b19 + 4c0f2e7bd4cb86917e362052e0642df600984b6c - + https://github.com/dotnet/runtime - a64420c79cb63c485138f4f1352b8730f27d7b19 + 4c0f2e7bd4cb86917e362052e0642df600984b6c https://github.com/dotnet/source-build-reference-packages e2e64d25662c00a35726d3c52f969a50edaa4f48 - + https://github.com/dotnet/sourcelink - 3f43bf1b2dead2cb51f20dc47f6dfd7981248820 + 47c52dd2ebf9edfd40abdcff999c13eb461f6ce2 diff --git a/eng/Versions.props b/eng/Versions.props index b03d8a0e1b..7aac7efb20 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -18,24 +18,24 @@ 1.0.426501 - 8.0.0-preview.3.23155.1 - 8.0.0-preview.3.23155.1 + 8.0.0-preview.5.23265.1 + 8.0.0-preview.5.23265.1 - 8.0.0-preview.4.23213.2 - 8.0.0-preview.4.23213.2 + 8.0.0-preview.5.23265.1 + 8.0.0-preview.5.23265.1 - 8.0.100-preview.3.23156.1 + 8.0.100-preview.5.23264.2 - 6.0.14 + 6.0.16 $(MicrosoftNETCoreApp60Version) - 7.0.3 + 7.0.5 $(MicrosoftNETCoreApp70Version) $(MicrosoftNETCoreApp60Version) - 7.0.2 - 8.0.0-preview.2.23127.4 + $(MicrosoftNETCoreApp70Version) + 8.0.0-preview.5.23260.3 @@ -61,13 +61,13 @@ 4.7.2 4.7.1 2.0.3 - 8.0.0-beta.23168.1 + 8.0.0-beta.23262.5 1.2.0-beta.406 7.0.0-beta.22316.2 10.0.18362 13.0.1 8.0.0-alpha.1.23265.3 - 1.2.0-beta-23165-02 + 8.0.0-beta.23218.3 3.11.0 diff --git a/eng/build.ps1 b/eng/build.ps1 index b107d42088..6bf2d4f694 100644 --- a/eng/build.ps1 +++ b/eng/build.ps1 @@ -59,7 +59,7 @@ if ($cleanupprivatebuild) { # Install sdk for building, restore and build managed components. if (-not $skipmanaged) { - Invoke-Expression "& `"$engroot\common\build.ps1`" -build -configuration $configuration -verbosity $verbosity /p:BuildArch=$architecture /p:TestArchitectures=$architecture $remainingargs" + Invoke-Expression "& `"$engroot\common\build.ps1`" -configuration $configuration -verbosity $verbosity /p:BuildArch=$architecture /p:TestArchitectures=$architecture $remainingargs" if ($lastExitCode -ne 0) { exit $lastExitCode } diff --git a/eng/build.sh b/eng/build.sh index 92bef5bea0..ae11013d07 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -41,14 +41,14 @@ usage_list+=("-test: run xunit tests") handle_arguments() { - lowerI="$(echo "$1" | tr "[:upper:]" "[:lower:]")" + lowerI="$(echo "${1/--/-}" | tr "[:upper:]" "[:lower:]")" case "$lowerI" in architecture|-architecture|-a) __BuildArch="$(echo "$2" | tr "[:upper:]" "[:lower:]")" __ShiftArgs=1 ;; - -binarylog|-bl|-clean|-integrationtest|-pack|-performancetest|-pipelineslog|-pl|-preparemachine|-publish|-r|-rebuild|-restore|-sign|-sb) + -binarylog|-bl|-clean|-integrationtest|-pack|-performancetest|-pipelineslog|-pl|-preparemachine|-publish|-r|-rebuild|-build|-restore|-sign|-sb) __ManagedBuildArgs="$__ManagedBuildArgs $1" ;; @@ -63,10 +63,6 @@ handle_arguments() { __ShiftArgs=1 ;; - -clean|-binarylog|-bl|-pipelineslog|-pl|-restore|-r|-rebuild|-pack|-integrationtest|-performancetest|-sign|-publish|-preparemachine|-sb) - __ManagedBuildArgs="$__ManagedBuildArgs $1" - ;; - -dotnetruntimeversion) __DotnetRuntimeVersion="$2" __ShiftArgs=1 @@ -148,11 +144,32 @@ fi # if [[ "$__ManagedBuild" == 1 ]]; then + echo "Commencing managed build for $__BuildType in $__RootBinDir/bin" - "$__RepoRootDir/eng/common/build.sh" --build --configuration "$__BuildType" $__CommonMSBuildArgs $__ManagedBuildArgs $__UnprocessedBuildArgs + "$__RepoRootDir/eng/common/build.sh" --configuration "$__BuildType" $__CommonMSBuildArgs $__ManagedBuildArgs $__UnprocessedBuildArgs + if [ "$?" != 0 ]; then exit 1 fi + + echo "Generating Version Source File" + __GenerateVersionLog="$__LogsDir/GenerateVersion.binlog" + + "$__RepoRootDir/eng/common/msbuild.sh" \ + $__RepoRootDir/eng/CreateVersionFile.proj \ + /bl:$__GenerateVersionLog \ + /t:GenerateVersionFiles \ + /restore \ + /p:GenerateVersionSourceFile=true \ + /p:NativeVersionSourceFile="$__ArtifactsIntermediatesDir/_version.c" \ + /p:Configuration="$__BuildType" \ + /p:Platform="$__BuildArch" \ + $__UnprocessedBuildArgs + + if [ $? != 0 ]; then + echo "Generating Version Source File FAILED" + exit 1 + fi fi # @@ -198,25 +215,6 @@ fi # Build native components # if [[ "$__NativeBuild" == 1 ]]; then - echo "Generating Version Source File" - __GenerateVersionLog="$__LogsDir/GenerateVersion.binlog" - - "$__RepoRootDir/eng/common/msbuild.sh" \ - $__RepoRootDir/eng/CreateVersionFile.proj \ - /bl:$__GenerateVersionLog \ - /t:GenerateVersionFiles \ - /restore \ - /p:GenerateVersionSourceFile=true \ - /p:NativeVersionSourceFile="$__ArtifactsIntermediatesDir/_version.c" \ - /p:Configuration="$__BuildType" \ - /p:Platform="$__BuildArch" \ - $__UnprocessedBuildArgs - - if [ $? != 0 ]; then - echo "Generating Version Source File FAILED" - exit 1 - fi - build_native "$__TargetOS" "$__BuildArch" "$__RepoRootDir" "$__IntermediatesDir" "install" "$__ExtraCmakeArgs" "diagnostic component" | tee "$__LogsDir"/make.log if [ "$?" != 0 ]; then diff --git a/eng/ci-prepare-artifacts.cmd b/eng/ci-prepare-artifacts.cmd index 5632c47db3..af95f7c3aa 100644 --- a/eng/ci-prepare-artifacts.cmd +++ b/eng/ci-prepare-artifacts.cmd @@ -9,7 +9,7 @@ powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0common\Build.p if NOT '%ERRORLEVEL%' == '0' goto ExitWithCode echo Creating bundles -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" %_commonArgs% -bundletools %*" +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" %_commonArgs% -build -bundletools %*" if NOT '%ERRORLEVEL%' == '0' goto ExitWithCode echo Creating dbgshim packages diff --git a/eng/cibuild.sh b/eng/cibuild.sh deleted file mode 100755 index cb0000ebc1..0000000000 --- a/eng/cibuild.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. - -source="${BASH_SOURCE[0]}" - -# resolve $SOURCE until the file is no longer a symlink -while [[ -h $source ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - - # if $source was a relative symlink, we need to resolve it relative to the path where - # the symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done - -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -# Fix any CI lab docker image problems - -__osname=$(uname -s) -if [ "$__osname" == "Linux" ]; then - if [ -e /etc/os-release ]; then - source /etc/os-release - if [[ $ID == "ubuntu" ]]; then - if [[ $VERSION_ID == "18.04" ]]; then - # Fix the CI lab's ubuntu 18.04 docker image: install curl. - sudo apt-get update - sudo apt-get install -y curl - fi - fi - elif [ -e /etc/redhat-release ]; then - __redhatRelease=$( "$__ApkKeysDir/alpine-devel@lists.alpinelinux.org-$id.rsa.pub" + done + + if [[ "$__SkipSigCheck" == "1" ]]; then + __ApkSignatureArg="--allow-untrusted" + else + __ApkSignatureArg="--keys-dir $__ApkKeysDir" + fi + # initialize DB "$__ApkToolsDir/apk.static" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ - -U --allow-untrusted --root "$__RootfsDir" --arch "$__AlpineArch" --initdb add + -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" --initdb add if [[ "$__AlpineLlvmLibsLookup" == 1 ]]; then __AlpinePackages+=" $("$__ApkToolsDir/apk.static" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ - -U --allow-untrusted --root "$__RootfsDir" --arch "$__AlpineArch" \ + -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \ search 'llvm*-libs' | sort | tail -1 | sed 's/-[^-]*//2g')" fi @@ -438,7 +494,7 @@ if [[ "$__CodeName" == "alpine" ]]; then "$__ApkToolsDir/apk.static" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ - -U --allow-untrusted --root "$__RootfsDir" --arch "$__AlpineArch" \ + -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \ add $__AlpinePackages rm -r "$__ApkToolsDir" @@ -512,69 +568,61 @@ elif [[ "$__CodeName" == "illumos" ]]; then elif [[ "$__CodeName" == "haiku" ]]; then JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} - echo "Building Haiku sysroot for x86_64" + echo "Building Haiku sysroot for $__HaikuArch" mkdir -p "$__RootfsDir/tmp" - cd "$__RootfsDir/tmp" - git clone -b hrev56235 https://review.haiku-os.org/haiku - git clone -b btrev43195 https://review.haiku-os.org/buildtools - cd "$__RootfsDir/tmp/buildtools" && git checkout 7487388f5110021d400b9f3b88e1a7f310dc066d - - # Fetch some unmerged patches - cd "$__RootfsDir/tmp/haiku" - ## Add development build profile (slimmer than nightly) - git fetch origin refs/changes/64/4164/1 && git -c commit.gpgsign=false cherry-pick FETCH_HEAD - - # Build jam - cd "$__RootfsDir/tmp/buildtools/jam" - make - - # Configure cross tools - echo "Building cross-compiler" - mkdir -p "$__RootfsDir/generated" - cd "$__RootfsDir/generated" - "$__RootfsDir/tmp/haiku/configure" -j"$JOBS" --sysroot "$__RootfsDir" --cross-tools-source "$__RootfsDir/tmp/buildtools" --build-cross-tools x86_64 - - # Build Haiku packages - echo "Building Haiku" - echo 'HAIKU_BUILD_PROFILE = "development-raw" ;' > UserProfileConfig - "$__RootfsDir/tmp/buildtools/jam/jam0" -j"$JOBS" -q 'package' 'Haiku' - - BaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg" - - # Download additional packages - echo "Downloading additional required packages" + pushd "$__RootfsDir/tmp" + + mkdir "$__RootfsDir/tmp/download" + + echo "Downloading Haiku package tool" + git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 $__RootfsDir/tmp/script + wget -O "$__RootfsDir/tmp/download/hosttools.zip" $($__RootfsDir/tmp/script/fetch.sh --hosttools) + unzip -o "$__RootfsDir/tmp/download/hosttools.zip" -d "$__RootfsDir/tmp/bin" + + DepotBaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg" + HpkgBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current" + + # Download Haiku packages + echo "Downloading Haiku packages" read -ra array <<<"$__HaikuPackages" for package in "${array[@]}"; do echo "Downloading $package..." # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60 # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598 - hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_x86_64","versionType":"LATEST","naturalLanguageCode":"en"}' \ - --header='Content-Type:application/json' "$BaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" - wget -P "$__RootfsDir/generated/download" "$hpkgDownloadUrl" + hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \ + --header='Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" + wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl" + done + for package in haiku haiku_devel; do + echo "Downloading $package..." + hpkgVersion="$(wget -qO- $HpkgBaseUrl | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" + wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" done - # Setup the sysroot - echo "Setting up sysroot and extracting needed packages" + # Set up the sysroot + echo "Setting up sysroot and extracting required packages" mkdir -p "$__RootfsDir/boot/system" - for file in "$__RootfsDir/generated/objects/haiku/x86_64/packaging/packages/"*.hpkg; do - "$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file" - done - for file in "$__RootfsDir/generated/download/"*.hpkg; do - "$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file" + for file in "$__RootfsDir/tmp/download/"*.hpkg; do + echo "Extracting $file..." + LD_LIBRARY_PATH="$__RootfsDir/tmp/bin" "$__RootfsDir/tmp/bin/package" extract -C "$__RootfsDir/boot/system" "$file" done + # Download buildtools + echo "Downloading Haiku buildtools" + wget -O "$__RootfsDir/tmp/download/buildtools.zip" $($__RootfsDir/tmp/script/fetch.sh --buildtools --arch=$__HaikuArch) + unzip -o "$__RootfsDir/tmp/download/buildtools.zip" -d "$__RootfsDir" + # Cleaning up temporary files echo "Cleaning up temporary files" + popd rm -rf "$__RootfsDir/tmp" - for name in "$__RootfsDir/generated/"*; do - if [[ "$name" =~ "cross-tools-" ]]; then - : # Keep the cross-compiler - else - rm -rf "$name" - fi - done elif [[ -n "$__CodeName" ]]; then - qemu-debootstrap $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" + + if [[ "$__SkipSigCheck" == "0" ]]; then + __Keyring="$__Keyring --force-check-gpg" + fi + + debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" cp "$__CrossDir/$__BuildArch/sources.list.$__CodeName" "$__RootfsDir/etc/apt/sources.list" chroot "$__RootfsDir" apt-get update chroot "$__RootfsDir" apt-get -f -y install diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index ccfb9951a5..a88d643c8a 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -6,6 +6,7 @@ unset(FREEBSD) unset(ILLUMOS) unset(ANDROID) unset(TIZEN) +unset(HAIKU) set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) @@ -16,6 +17,7 @@ elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) set(ILLUMOS 1) elseif(EXISTS ${CROSS_ROOTFS}/boot/system/develop/headers/config/HaikuConfig.h) set(CMAKE_SYSTEM_NAME Haiku) + set(HAIKU 1) else() set(CMAKE_SYSTEM_NAME Linux) set(LINUX 1) @@ -67,16 +69,30 @@ elseif(TARGET_ARCH_NAME STREQUAL "armv6") endif() elseif(TARGET_ARCH_NAME STREQUAL "ppc64le") set(CMAKE_SYSTEM_PROCESSOR ppc64le) - set(TOOLCHAIN "powerpc64le-linux-gnu") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/powerpc64le-alpine-linux-musl) + set(TOOLCHAIN "powerpc64le-alpine-linux-musl") + else() + set(TOOLCHAIN "powerpc64le-linux-gnu") + endif() elseif(TARGET_ARCH_NAME STREQUAL "riscv64") set(CMAKE_SYSTEM_PROCESSOR riscv64) - set(TOOLCHAIN "riscv64-linux-gnu") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/riscv64-alpine-linux-musl) + set(TOOLCHAIN "riscv64-alpine-linux-musl") + else() + set(TOOLCHAIN "riscv64-linux-gnu") + endif() elseif(TARGET_ARCH_NAME STREQUAL "s390x") set(CMAKE_SYSTEM_PROCESSOR s390x) - set(TOOLCHAIN "s390x-linux-gnu") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/s390x-alpine-linux-musl) + set(TOOLCHAIN "s390x-alpine-linux-musl") + else() + set(TOOLCHAIN "s390x-linux-gnu") + endif() elseif(TARGET_ARCH_NAME STREQUAL "x64") set(CMAKE_SYSTEM_PROCESSOR x86_64) - if(LINUX) + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/x86_64-alpine-linux-musl) + set(TOOLCHAIN "x86_64-alpine-linux-musl") + elseif(LINUX) set(TOOLCHAIN "x86_64-linux-gnu") if(TIZEN) set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu/9.2.0") @@ -86,11 +102,15 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64") elseif(ILLUMOS) set(TOOLCHAIN "x86_64-illumos") elseif(HAIKU) - set(TOOLCHAIN "x64_64-unknown-haiku") + set(TOOLCHAIN "x86_64-unknown-haiku") endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") set(CMAKE_SYSTEM_PROCESSOR i686) - set(TOOLCHAIN "i686-linux-gnu") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) + set(TOOLCHAIN "i586-alpine-linux-musl") + else() + set(TOOLCHAIN "i686-linux-gnu") + endif() if(TIZEN) set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0") endif() @@ -196,10 +216,8 @@ elseif(HAIKU) return() endif() - set(SEARCH_PATH "${CROSS_ROOTFS}/generated/cross-tools-x86_64/bin") - find_program(EXEC_LOCATION_${exec} - PATHS ${SEARCH_PATH} + PATHS "${CROSS_ROOTFS}/cross-tools-x86_64/bin" NAMES "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" "${TOOLSET_PREFIX}${exec}") @@ -264,8 +282,11 @@ elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64)$") add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) + add_toolchain_linker_flag("--target=${TOOLCHAIN}") + add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}") + endif() add_toolchain_linker_flag(-m32) - if(TIZEN) add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") @@ -275,11 +296,14 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") elseif(ILLUMOS) add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/amd64/lib") +elseif(HAIKU) + add_toolchain_linker_flag("-lnetwork") + add_toolchain_linker_flag("-lroot") endif() # Specify compile options -if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|ppc64le|riscv64|s390x)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU) +if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) @@ -298,10 +322,16 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") add_definitions (-DCLR_ARM_FPU_CAPABILITY=${CLR_ARM_FPU_CAPABILITY}) + # persist variables across multiple try_compile passes + list(APPEND CMAKE_TRY_COMPILE_PLATFORM_VARIABLES CLR_ARM_FPU_TYPE CLR_ARM_FPU_CAPABILITY) + if(TARGET_ARCH_NAME STREQUAL "armel") add_compile_options(-mfloat-abi=softfp) endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) + add_compile_options(--target=${TOOLCHAIN}) + endif() add_compile_options(-m32) add_compile_options(-Wno-error=unused-command-line-argument) endif() diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index 7aee4213e1..517401b688 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -64,7 +64,7 @@ if [ -z "$CLR_CC" ]; then if [ -z "$majorVersion" ]; then # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. if [ "$compiler" = "clang" ]; then versions="16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" - elif [ "$compiler" = "gcc" ]; then versions="12 11 10 9 8 7 6 5 4.9"; fi + elif [ "$compiler" = "gcc" ]; then versions="13 12 11 10 9 8 7 6 5 4.9"; fi for version in $versions; do _major="${version%%.*}" diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index b214a31db2..44ad26abf5 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -25,7 +25,7 @@ parameters: enablePublishTestResults: false enablePublishUsingPipelines: false enableBuildRetry: false - disableComponentGovernance: false + disableComponentGovernance: '' componentGovernanceIgnoreDirectories: '' mergeTestResults: false testRunTitle: '' @@ -155,11 +155,16 @@ jobs: uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} continueOnError: true - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(parameters.disableComponentGovernance, 'true')) }}: - - task: ComponentGovernanceComponentDetection@0 - continueOnError: true - inputs: - ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + - template: /eng/common/templates/steps/component-governance.yml + parameters: + ${{ if eq(parameters.disableComponentGovernance, '') }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + disableComponentGovernance: false + ${{ else }}: + disableComponentGovernance: true + ${{ else }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: diff --git a/eng/common/templates/steps/component-governance.yml b/eng/common/templates/steps/component-governance.yml new file mode 100644 index 0000000000..0ecec47b0c --- /dev/null +++ b/eng/common/templates/steps/component-governance.yml @@ -0,0 +1,13 @@ +parameters: + disableComponentGovernance: false + componentGovernanceIgnoreDirectories: '' + +steps: +- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: + - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable +- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: + - task: ComponentGovernanceComponentDetection@0 + continueOnError: true + inputs: + ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 8ad03be3ec..38cf94ff88 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -287,6 +287,25 @@ function InstallDotNet([string] $dotnetRoot, [string] $runtimeSourceFeedKey = '', [switch] $noPath) { + $dotnetVersionLabel = "'sdk v$version'" + + if ($runtime -ne '' -and $runtime -ne 'sdk') { + $runtimePath = $dotnetRoot + $runtimePath = $runtimePath + "\shared" + if ($runtime -eq "dotnet") { $runtimePath = $runtimePath + "\Microsoft.NETCore.App" } + if ($runtime -eq "aspnetcore") { $runtimePath = $runtimePath + "\Microsoft.AspNetCore.App" } + if ($runtime -eq "windowsdesktop") { $runtimePath = $runtimePath + "\Microsoft.WindowsDesktop.App" } + $runtimePath = $runtimePath + "\" + $version + + $dotnetVersionLabel = "runtime toolset '$runtime/$architecture v$version'" + + if (Test-Path $runtimePath) { + Write-Host " Runtime toolset '$runtime/$architecture v$version' already installed." + $installSuccess = $true + Exit + } + } + $installScript = GetDotNetInstallScript $dotnetRoot $installParameters = @{ Version = $version @@ -323,18 +342,18 @@ function InstallDotNet([string] $dotnetRoot, } else { $location = "public location"; } - Write-Host "Attempting to install dotnet from $location." + Write-Host " Attempting to install $dotnetVersionLabel from $location." try { & $installScript @variation $installSuccess = $true break } catch { - Write-Host "Failed to install dotnet from $location." + Write-Host " Failed to install $dotnetVersionLabel from $location." } } if (-not $installSuccess) { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from any of the specified locations." + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install $dotnetVersionLabel from any of the specified locations." ExitWithExitCode 1 } } @@ -399,7 +418,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # Locate Visual Studio installation or download x-copy msbuild. $vsInfo = LocateVisualStudio $vsRequirements if ($vsInfo -ne $null) { - $vsInstallDir = $vsInfo.installationPath + # Ensure vsInstallDir has a trailing slash + $vsInstallDir = Join-Path $vsInfo.installationPath "\" $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0] InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion diff --git a/eng/common/tools.sh b/eng/common/tools.sh index cf9fb1ea2d..e8d4789433 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -184,6 +184,35 @@ function InstallDotNetSdk { function InstallDotNet { local root=$1 local version=$2 + local runtime=$4 + + local dotnetVersionLabel="'$runtime v$version'" + if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then + runtimePath="$root" + runtimePath="$runtimePath/shared" + case "$runtime" in + dotnet) + runtimePath="$runtimePath/Microsoft.NETCore.App" + ;; + aspnetcore) + runtimePath="$runtimePath/Microsoft.AspNetCore.App" + ;; + windowsdesktop) + runtimePath="$runtimePath/Microsoft.WindowsDesktop.App" + ;; + *) + ;; + esac + runtimePath="$runtimePath/$version" + + dotnetVersionLabel="runtime toolset '$runtime/$architecture v$version'" + + if [ -d "$runtimePath" ]; then + echo " Runtime toolset '$runtime/$architecture v$version' already installed." + local installSuccess=1 + return + fi + fi GetDotNetInstallScript "$root" local install_script=$_GetDotNetInstallScript @@ -228,17 +257,17 @@ function InstallDotNet { for variationName in "${variations[@]}"; do local name="$variationName[@]" local variation=("${!name}") - echo "Attempting to install dotnet from $variationName." + echo " Attempting to install $dotnetVersionLabel from $variationName." bash "$install_script" "${variation[@]}" && installSuccess=1 if [[ "$installSuccess" -eq 1 ]]; then break fi - echo "Failed to install dotnet from $variationName." + echo " Failed to install $dotnetVersionLabel from $variationName." done if [[ "$installSuccess" -eq 0 ]]; then - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from any of the specified locations." + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install $dotnetVersionLabel from any of the specified locations." ExitWithExitCode 1 fi } diff --git a/eng/docker-build.sh b/eng/docker-build.sh deleted file mode 100755 index 99cfc0d047..0000000000 --- a/eng/docker-build.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. - -source_directory= -docker_image= -docker_container_name= - -while [ $# -ne 0 ]; do - name=$1 - case $name in - -s|--source-directory) - shift - source_directory=$1 - ;; - -i|--docker-image) - shift - docker_image=$1 - ;; - -c|--container-name) - shift - docker_container_name=$1 - ;; - *) - args="$args $1" - ;; - esac - shift -done - -echo "Initialize Docker Container" -if command -v docker > /dev/null; then - docker_bin=$(command -v docker) -else - echo "Unable to find docker" - exit 1 -fi - -$docker_bin --version - -# Get user id -user_name=$(whoami) -echo "user name: $user_name" -user_id=$(id -u $user_name) -echo "user id: $user_id" - -# Download image -$docker_bin pull $docker_image - -# Create local network to avoid port conflicts when multiple agents run on same machine -$docker_bin network create vsts_network_$docker_container_name - -# Create and start container -docker_id="$($docker_bin create -it --rm --security-opt seccomp=unconfined --ulimit core=-1 \ - --name vsts_container_$docker_container_name \ - --network=vsts_network_$docker_container_name \ - --volume $source_directory:$source_directory \ - --workdir=$source_directory $docker_image bash --verbose)" -$docker_bin start $docker_id - -# Create an user with the same uid in the container -container_user_name=vsts_$(echo $user_name | awk '{print tolower($0)}') -echo "container user name: $container_user_name" - -# Add sudo user with same uid that can run any sudo command without password -$docker_bin exec $docker_id useradd -K MAIL_DIR=/dev/null -m -u $user_id $container_user_name -$docker_bin exec $docker_id groupadd sudouser -$docker_bin exec $docker_id usermod -a -G sudouser $container_user_name -$docker_bin exec $docker_id su -c "echo '%sudouser ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers" - -echo "Execute $args" -$docker_bin exec --workdir=$source_directory --user $container_user_name $docker_id $args -lasterrorcode=$? - -echo "Cleanup Docker Container/Network" -$docker_bin container stop $docker_id -$docker_bin network rm vsts_network_$docker_container_name - -exit $lasterrorcode diff --git a/eng/build.yml b/eng/pipelines/build.yml similarity index 73% rename from eng/build.yml rename to eng/pipelines/build.yml index 6115437680..4b5dd121c8 100644 --- a/eng/build.yml +++ b/eng/pipelines/build.yml @@ -3,24 +3,26 @@ parameters: name: '' # Agent OS (Windows_NT, Linux, MacOS, FreeBSD) osGroup: Windows_NT + # Optional: OS suffix like -musl + osSuffix: '' # Additional variables variables: {} # Build strategy - matrix strategy: '' # Optional: Job timeout timeoutInMinutes: 180 - # Optional: Docker image to use - dockerImage: '' - # Optional: ROOTFS_DIR to use - crossrootfsDir: '' - crossbuild: false + # Optional: native build container resource name + nativeBuildContainer: '' + # Optional: container resource name + container: '' + # Optional: build only job if true + buildOnly: false # Optional: test only job if true testOnly: false - buildAndSkipTest: false - # Depends on + # Optional: architecture cross build if true + crossBuild: false + # Depends on dependsOn: '' - artifactsTargetPath: '' - requiresCapPtraceContainer: false isCodeQLRun: false jobs: @@ -62,8 +64,8 @@ jobs: name: NetCore-Public demands: ImageOverride -equals windows.vs2022.amd64.open - ${{ if and(ne(parameters.dockerImage, ''), ne(parameters.requiresCapPtraceContainer, 'true')) }}: - container: ${{ parameters.dockerImage }} + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} ${{ if ne(parameters.strategy, '') }}: strategy: ${{ parameters.strategy }} @@ -76,27 +78,28 @@ jobs: variables: - ${{ insert }}: ${{ parameters.variables }} - - _DockerImageName: ${{ parameters.dockerImage }} - _PhaseName : ${{ parameters.name }} - _HelixType: build/product - _HelixBuildConfig: $(_BuildConfig) - _Pipeline_StreamDumpDir: $(Build.SourcesDirectory)/artifacts/tmp/$(_BuildConfig)/streams + - _BuildDisplayName: 'Build / Test' + - _ExtraBuildParams: '' + - _TestArgs: '-test' + - _Cross: '' + - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - _buildScript: $(Build.SourcesDirectory)\eng\cibuild.cmd + - _buildScript: $(Build.SourcesDirectory)\build.cmd - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - _buildScript: $(Build.SourcesDirectory)/eng/cibuild.sh - - - _TestArgs: '-test' - - _dockerEnv: '' + - _buildScript: $(Build.SourcesDirectory)/build.sh - ${{ if eq(parameters.testOnly, 'true') }}: - _TestArgs: '-test -skipnative' - - ${{ if eq(parameters.requiresCapPtraceContainer, 'true') }}: - - _dockerEnv: $(Build.SourcesDirectory)/eng/docker-build.sh - --docker-image $(_DockerImageName) - --source-directory $(Build.SourcesDirectory) - --container-name diagnostics-$(Build.BuildId) + - _BuildDisplayName: Test + + - ${{ if or(eq(parameters.buildOnly, 'true'), eq(parameters.isCodeQLRun, 'true')) }}: + - _TestArgs: '' + - ${{ if eq(parameters.isCodeQLRun, 'true') }}: - name: Codeql.Enabled value: True @@ -109,11 +112,8 @@ jobs: - name: Codeql.Language value: csharp,cpp - - ${{ if or(eq(parameters.buildAndSkipTest, 'true'), eq(parameters.isCodeQLRun, 'true')) }}: - - _TestArgs: '' - - - _InternalInstallArgs: '' # For testing msrc's and service releases. The RuntimeSourceVersion is either "default" or the service release version to test + - _InternalInstallArgs: '' - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.isCodeQLRun, 'false')) }}: - _InternalInstallArgs: -dotnetruntimeversion '$(DotnetRuntimeVersion)' @@ -128,43 +128,61 @@ jobs: - _HelixSource: pr/dotnet/arcade/$(Build.SourceBranch) # This is only required for cross builds. - - ${{ if and(eq(parameters.crossbuild, false), eq(parameters.crossrootfsDir, '')) }}: - - _Cross: '' - - ${{ if or(eq(parameters.crossbuild, true), ne(parameters.crossrootfsDir, '')) }}: + - ${{ if eq(parameters.crossBuild, true) }}: - _Cross: -cross + # If there is a native build container, build managed in the host vm/container and native in the nativeBuildContainer + - ${{ if ne(parameters.nativeBuildContainer, '') }}: + - _ExtraBuildParams: -skipnative + - _BuildDisplayName: 'Build Managed' + + # Only add the cross build option if a combined build/test managed/native build (i.e. MacOS arm64) + - ${{ if eq(parameters.nativeBuildContainer, '') }}: + - _ExtraBuildParams: $(_Cross) + steps: - ${{ if eq(parameters.osGroup, 'Linux') }}: - ${{ if eq(parameters.testOnly, 'true') }}: - task: DownloadBuildArtifacts@0 - displayName: 'Download release builds' + displayName: 'Download Build Artifacts' inputs: downloadPath: '$(Build.ArtifactStagingDirectory)/__download__' - downloadType: specific + downloadType: specific itemPattern: | - Build_$(_BuildConfig)/bin/Linux.$(_BuildArch).$(_BuildConfig)/** + Build_$(_BuildConfig)/bin/Linux${{ parameters.osSuffix }}.$(_BuildArch).$(_BuildConfig)/** checkDownloadedFiles: true - task: CopyFiles@2 displayName: 'Binplace Product' inputs: - sourceFolder: $(Build.ArtifactStagingDirectory)/__download__/Build_$(_BuildConfig)/bin/Linux.$(_BuildArch).$(_BuildConfig) + sourceFolder: $(Build.ArtifactStagingDirectory)/__download__/Build_$(_BuildConfig)/bin/Linux${{ parameters.osSuffix }}.$(_BuildArch).$(_BuildConfig) targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/Linux.$(_BuildArch).$(_BuildConfig)' - ${{ if eq(parameters.isCodeQLRun, 'true') }}: - task: CodeQL3000Init@0 displayName: CodeQL Initialize - - script: $(_dockerEnv) $(_buildScript) - -configuration $(_BuildConfig) + - script: $(_buildScript) + -ci + -configuration $(_BuildConfig) -architecture $(_BuildArch) - $(_Cross) + $(_ExtraBuildParams) $(_TestArgs) /p:OfficialBuildId=$(BUILD.BUILDNUMBER) $(_InternalInstallArgs) - displayName: Build / Test + displayName: $(_BuildDisplayName) condition: succeeded() - env: - ROOTFS_DIR: ${{ parameters.crossrootfsDir }} + + - ${{ if ne(parameters.nativeBuildContainer, '') }}: + - script: $(_buildScript) + -ci + -configuration $(_BuildConfig) + -architecture $(_BuildArch) + -skipmanaged + $(_Cross) + /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + $(_InternalInstallArgs) + displayName: Build Native + target: ${{ parameters.nativeBuildContainer }} - ${{ if eq(parameters.isCodeQLRun, 'true') }}: - task: CodeQL3000Finalize@0 @@ -176,8 +194,15 @@ jobs: inputs: SourceFolder: '$(Build.SourcesDirectory)/artifacts/$(_PublishArtifacts)' Contents: '**' - TargetFolder: $(Build.ArtifactStagingDirectory)/artifacts/${{ coalesce(parameters.artifactsTargetPath, '$(_PublishArtifacts)') }} - condition: ne(variables['_PublishArtifacts'], '') + TargetFolder: $(Build.ArtifactStagingDirectory)/artifacts/$(_ArtifactsTargetPath) + condition: and(ne(variables['_PublishArtifacts'], ''), ne(variables['_ArtifactsTargetPath'], '')) + - task: CopyFiles@2 + displayName: Gather binaries for publish to artifacts + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/$(_PublishArtifacts)' + Contents: '**' + TargetFolder: $(Build.ArtifactStagingDirectory)/artifacts/$(_PublishArtifacts) + condition: and(ne(variables['_PublishArtifacts'], ''), eq(variables['_ArtifactsTargetPath'], '')) - task: PublishBuildArtifacts@1 displayName: Publish Build Artifacts inputs: @@ -230,12 +255,12 @@ jobs: continueOnError: true condition: always() - - ${{ if and(eq(parameters.buildAndSkipTest, 'false'), eq(parameters.isCodeQLRun, 'false')) }}: + - ${{ if and(eq(parameters.buildOnly, 'false'), eq(parameters.isCodeQLRun, 'false')) }}: # Publish test results to Azure Pipelines - task: PublishTestResults@2 inputs: testResultsFormat: xUnit - testResultsFiles: '**/*UnitTests*.xml' + testResultsFiles: '**/*UnitTests*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults' failTaskOnFailedTests: true testRunTitle: 'Tests $(_PhaseName) $(_BuildArch) $(_BuildConfig)' diff --git a/eng/pipelines/pipeline-resources.yml b/eng/pipelines/pipeline-resources.yml new file mode 100644 index 0000000000..524a3793ce --- /dev/null +++ b/eng/pipelines/pipeline-resources.yml @@ -0,0 +1,61 @@ +parameters: + - name: stages + type: stageList + +resources: + containers: + - container: linux_x64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7 + + - container: linux_arm + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross + env: + ROOTFS_DIR: /crossrootfs/arm + + - container: linux_arm64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64 + env: + ROOTFS_DIR: /crossrootfs/arm64 + + - container: linux_musl_x64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.13-WithNode + + - container: linux_musl_arm + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm-alpine + env: + ROOTFS_DIR: /crossrootfs/arm + + - container: linux_musl_arm64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-alpine + env: + ROOTFS_DIR: /crossrootfs/arm64 + + - container: test_linux_x64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7 + + - container: test_linux_musl_x64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.13-WithNode + options: --cap-add=SYS_PTRACE + + - container: test_debian_11_amd64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 + options: '--env PYTHONPATH=/usr/bin/python3.9' + + - container: test_fedora_36 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-36 + options: --cap-add=SYS_PTRACE + + - container: test_opensuse_15_2 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:opensuse-15.2-helix-amd64 + + - container: test_ubuntu_18_04 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04 + + - container: test_ubuntu_20_04 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04 + options: '--env PYTHONPATH=/usr/bin/python3.8' + + - container: test_ubuntu_22_04 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 + +stages: ${{ parameters.stages }} diff --git a/eng/prepare-release.yml b/eng/pipelines/prepare-release.yml similarity index 100% rename from eng/prepare-release.yml rename to eng/pipelines/prepare-release.yml diff --git a/global.json b/global.json index 879572b3ab..b6a0f5bea8 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "tools": { - "dotnet": "8.0.100-preview.1.23115.2", + "dotnet": "8.0.100-preview.3.23178.7", "runtimes": { "dotnet": [ "$(MicrosoftNETCoreApp60Version)", @@ -16,6 +16,6 @@ }, "msbuild-sdks": { "Microsoft.Build.NoTargets": "3.5.0", - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23168.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23262.5" } } diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/ClrMDHelper.cs b/src/Microsoft.Diagnostics.ExtensionCommands/ClrMDHelper.cs index 074026ac39..76cda04435 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/ClrMDHelper.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/ClrMDHelper.cs @@ -851,37 +851,51 @@ private static bool TryGetSegmentMemoryRange(ClrSegment segment, GCGeneration ge switch (generation) { case GCGeneration.Generation0: - start = segment.Generation0.Start; - end = segment.Generation0.End; - return start != end; + if (segment.Kind == GCSegmentKind.Generation0 || segment.Kind == GCSegmentKind.Ephemeral) + { + start = segment.Generation0.Start; + end = segment.Generation0.End; + } + break; case GCGeneration.Generation1: - start = segment.Generation1.Start; - end = segment.Generation1.End; - return start != end; + if (segment.Kind == GCSegmentKind.Generation1 || segment.Kind == GCSegmentKind.Ephemeral) + { + start = segment.Generation1.Start; + end = segment.Generation1.End; + } + break; case GCGeneration.Generation2: - if (segment.Kind != GCSegmentKind.Large && segment.Kind != GCSegmentKind.Large && segment.Kind != GCSegmentKind.Frozen) + if (segment.Kind == GCSegmentKind.Generation2 || segment.Kind == GCSegmentKind.Ephemeral) { start = segment.Generation2.Start; end = segment.Generation2.End; } - return start != end; + break; case GCGeneration.LargeObjectHeap: if (segment.Kind == GCSegmentKind.Large) { start = segment.Start; end = segment.End; } - return start != end; + break; case GCGeneration.PinnedObjectHeap: - if (segment.Kind == GCSegmentKind.Pinned || segment.Kind == GCSegmentKind.Frozen) + if (segment.Kind == GCSegmentKind.Pinned) { start = segment.Start; end = segment.End; } - return start != end; + break; + case GCGeneration.FrozenObjectHeap: + if (segment.Kind == GCSegmentKind.Frozen) + { + start = segment.Start; + end = segment.End; + } + break; default: return false; } + return start != end; } public IEnumerable EnumerateConcurrentQueue(ulong address) diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/DumpGenCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/DumpGenCommand.cs index ef1908ef3b..db020e7926 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/DumpGenCommand.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/DumpGenCommand.cs @@ -96,22 +96,21 @@ private GCGeneration ParseGenerationArgument(string generation) return GCGeneration.NotSet; } string lowerString = generation.ToLowerInvariant(); - switch (lowerString) + GCGeneration result = lowerString switch { - case "gen0": - return GCGeneration.Generation0; - case "gen1": - return GCGeneration.Generation1; - case "gen2": - return GCGeneration.Generation2; - case "loh": - return GCGeneration.LargeObjectHeap; - case "poh": - return GCGeneration.PinnedObjectHeap; - default: - WriteLine($"{generation} is not a supported generation (gen0, gen1, gen2, loh, poh)"); - return GCGeneration.NotSet; + "gen0" => GCGeneration.Generation0, + "gen1" => GCGeneration.Generation1, + "gen2" => GCGeneration.Generation2, + "loh" => GCGeneration.LargeObjectHeap, + "poh" => GCGeneration.PinnedObjectHeap, + "foh" => GCGeneration.FrozenObjectHeap, + _ => GCGeneration.NotSet, + }; + if (result == GCGeneration.NotSet) + { + WriteLine($"{generation} is not a supported generation (gen0, gen1, gen2, loh, poh, foh)"); } + return result; } @@ -133,6 +132,7 @@ Generation number can take the following values (case insensitive): - gen2 - loh - poh +- foh > dumpgen gen0 Statistics: diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/DumpStackObjectsCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/DumpStackObjectsCommand.cs index 1fb0562d63..131450fe8b 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/DumpStackObjectsCommand.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/DumpStackObjectsCommand.cs @@ -228,7 +228,7 @@ private ClrSegment GetSegment(ulong potentialObject, ref int segmentIndex) // and that all objects passed to it are within the given // range of segment bounds. Debug.Assert(segmentIndex >= 0 && segmentIndex <= segments.Length); - Debug.Assert(segments[0].ObjectRange.Start < potentialObject); + Debug.Assert(segments[0].ObjectRange.Start <= potentialObject); Debug.Assert(potentialObject < segments[segments.Length - 1].ObjectRange.End); for (; segmentIndex < segments.Length; segmentIndex++) diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/GCGeneration.cs b/src/Microsoft.Diagnostics.ExtensionCommands/GCGeneration.cs index 5ebe713150..028e7c413c 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/GCGeneration.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/GCGeneration.cs @@ -10,6 +10,7 @@ public enum GCGeneration Generation1 = 2, Generation2 = 3, LargeObjectHeap = 4, - PinnedObjectHeap = 5 + PinnedObjectHeap = 5, + FrozenObjectHeap = 6 } } diff --git a/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt b/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt index bd1568c039..dbe012fada 100644 --- a/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt +++ b/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt @@ -21,6 +21,9 @@ $(RootBinDir)/TestResults/$(TargetConfiguration)/sos.unittests_$(Timestamp) $(RootBinDir)/tmp/$(TargetConfiguration)\dumps + true + true + true false @@ -140,6 +143,8 @@