diff --git a/Build.cmd b/Build.cmd index 3de9ae405f..0a7b2aa973 100644 --- a/Build.cmd +++ b/Build.cmd @@ -1,3 +1,3 @@ @echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\build.ps1""" -restore %*" +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\build.ps1""" -restore -build %*" exit /b %ErrorLevel% diff --git a/build.sh b/build.sh index 58a2abb50f..75055e9c92 100755 --- a/build.sh +++ b/build.sh @@ -13,4 +13,4 @@ while [[ -h $source ]]; do done scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" -"$scriptroot/eng/build.sh" --restore $@ +"$scriptroot/eng/build.sh" --restore --build $@ diff --git a/diagnostics-codeql.yml b/diagnostics-codeql.yml index ee506ce6de..5cacd47b62 100644 --- a/diagnostics-codeql.yml +++ b/diagnostics-codeql.yml @@ -28,125 +28,128 @@ variables: - name: skipComponentGovernanceDetection value: true -stages: -- stage: build - displayName: Build and Test Diagnostics - jobs: - - template: /eng/build.yml - parameters: - name: Windows - osGroup: Windows_NT - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - Build_Release_x86: - _BuildConfig: Release - _BuildArch: x86 - Build_Release_arm: - _BuildConfig: Release - _BuildArch: arm - Build_Release_arm64: - _BuildConfig: Release - _BuildArch: arm64 +extends: + template: /eng/pipelines/pipeline-resources.yml + parameters: + stages: + - stage: build + displayName: Build and Test Diagnostics + jobs: + - template: /eng/pipelines/build.yml + parameters: + name: Windows + osGroup: Windows_NT + isCodeQLRun: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 + Build_Release_x86: + _BuildConfig: Release + _BuildArch: x86 + Build_Release_arm: + _BuildConfig: Release + _BuildArch: arm + Build_Release_arm64: + _BuildConfig: Release + _BuildArch: arm64 - - template: /eng/build.yml - parameters: - name: CentOS_7 - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343 - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 + - template: /eng/pipelines/build.yml + parameters: + name: Linux_x64 + osGroup: Linux + nativeBuildContainer: linux_x64 + isCodeQLRun: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 - - template: /eng/build.yml - parameters: - name: Alpine3_13 - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.13-WithNode-20210910135845-c401c85 - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 + - template: /eng/pipelines/build.yml + parameters: + name: Linux_musl + osGroup: Linux + nativeBuildContainer: linux_musl_x64 + isCodeQLRun: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 - - template: /eng/build.yml - parameters: - name: MacOS - osGroup: MacOS - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 + - template: /eng/pipelines/build.yml + parameters: + name: MacOS + osGroup: MacOS + isCodeQLRun: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 - - template: /eng/build.yml - parameters: - name: MacOS_arm64 - osGroup: MacOS_cross - crossbuild: true - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 + - template: /eng/pipelines/build.yml + parameters: + name: MacOS_arm64 + osGroup: MacOS_cross + crossBuild: true + isCodeQLRun: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm64 - - template: /eng/build.yml - parameters: - name: Linux_arm - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-20210719121212-8a8d3be - crossrootfsDir: '/crossrootfs/arm' - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm + - template: /eng/pipelines/build.yml + parameters: + name: Linux_arm + osGroup: Linux + nativeBuildContainer: linux_arm + crossBuild: true + isCodeQLRun: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm - - template: /eng/build.yml - parameters: - name: Linux_arm64 - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-20210719121212-8a8d3be - crossrootfsDir: '/crossrootfs/arm64' - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 + - template: /eng/pipelines/build.yml + parameters: + name: Linux_arm64 + osGroup: Linux + nativeBuildContainer: linux_arm64 + crossBuild: true + isCodeQLRun: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm64 - - template: /eng/build.yml - parameters: - name: Linux_musl_arm - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm-alpine-20210923140502-78f7860 - crossrootfsDir: '/crossrootfs/arm' - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm + - template: /eng/pipelines/build.yml + parameters: + name: Linux_musl_arm + osGroup: Linux + nativeBuildContainer: linux_musl_arm + crossBuild: true + isCodeQLRun: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm - - template: /eng/build.yml - parameters: - name: Linux_musl_arm64 - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-alpine-20210923140502-78f7860 - crossrootfsDir: '/crossrootfs/arm64' - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 \ No newline at end of file + - template: /eng/pipelines/build.yml + parameters: + name: Linux_musl_arm64 + osGroup: Linux + nativeBuildContainer: linux_musl_arm64 + crossBuild: true + isCodeQLRun: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm64 diff --git a/diagnostics.yml b/diagnostics.yml index 28b5eeb015..599fce356f 100644 --- a/diagnostics.yml +++ b/diagnostics.yml @@ -70,357 +70,420 @@ variables: - name: RuntimeFeedBase64SasToken value: $(dotnetclimsrc-read-sas-token-base64) -stages: - - stage: build - displayName: Build and Test Diagnostics - jobs: +extends: + template: /eng/pipelines/pipeline-resources.yml + parameters: + stages: + - stage: build + displayName: Build and Test Diagnostics + jobs: - ############################ - # # - # Source Build legs # - # # - ############################ + ############################ + # # + # Source Build legs # + # # + ############################ - - template: /eng/common/templates/job/source-build.yml - parameters: - platform: - name: Complete - buildScript: ./eng/common/build.sh + - template: /eng/common/templates/job/source-build.yml + parameters: + platform: + name: Complete + buildScript: ./eng/common/build.sh - ############################ - # # - # Build legs # - # # - ############################ + ############################ + # # + # Build legs # + # # + ############################ - - template: /eng/build.yml - parameters: - name: Windows - osGroup: Windows_NT - strategy: - matrix: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - _PublishArtifacts: bin - Build_Release_x86: - _BuildConfig: Release - _BuildArch: x86 - _PublishArtifacts: bin/Windows_NT.x86.Release - ${{ if ne(variables['System.TeamProject'], 'public') }}: - Build_Release_arm: - _BuildOnly: true - _BuildConfig: Release - _BuildArch: arm - _PublishArtifacts: bin/Windows_NT.arm.Release - Build_Release_arm64: - _BuildOnly: true - _BuildConfig: Release - _BuildArch: arm64 - _PublishArtifacts: bin/Windows_NT.arm64.Release + - template: /eng/pipelines/build.yml + parameters: + name: Windows + osGroup: Windows_NT + strategy: + matrix: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 + Build_Release: + _BuildConfig: Release + _BuildArch: x64 + _PublishArtifacts: bin + Build_Release_x86: + _BuildConfig: Release + _BuildArch: x86 + _PublishArtifacts: bin/Windows_NT.x86.Release + ${{ if ne(variables['System.TeamProject'], 'public') }}: + Build_Release_arm: + _BuildConfig: Release + _BuildArch: arm + _PublishArtifacts: bin/Windows_NT.arm.Release + Build_Release_arm64: + _BuildConfig: Release + _BuildArch: arm64 + _PublishArtifacts: bin/Windows_NT.arm64.Release - - template: /eng/build.yml - parameters: - name: CentOS_7 - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343 - strategy: - matrix: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 - _PublishArtifacts: bin/Linux.x64.Debug - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - _PublishArtifacts: bin/Linux.x64.Release + - template: /eng/pipelines/build.yml + parameters: + name: Linux_x64 + osGroup: Linux + nativeBuildContainer: linux_x64 + buildOnly: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 + _PublishArtifacts: bin/Linux.x64.Release + ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 + _PublishArtifacts: bin/Linux.x64.Debug - - template: /eng/build.yml - parameters: - name: Alpine3_13 - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.13-WithNode-20210910135845-c401c85 - artifactsTargetPath: bin/Linux-musl.x64.Release - requiresCapPtraceContainer: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - _PublishArtifacts: bin/Linux.x64.Release - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 + - template: /eng/pipelines/build.yml + parameters: + name: Linux_musl + osGroup: Linux + osSuffix: -musl + nativeBuildContainer: linux_musl_x64 + buildOnly: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 + _PublishArtifacts: bin/Linux.x64.Release + _ArtifactsTargetPath: bin/Linux-musl.x64.Release + ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 + _PublishArtifacts: bin/Linux.x64.Debug + _ArtifactsTargetPath: bin/Linux-musl.x64.Debug - - template: /eng/build.yml - parameters: - name: MacOS - osGroup: MacOS - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - _PublishArtifacts: bin/OSX.x64.Release - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 + - template: /eng/pipelines/build.yml + parameters: + name: MacOS + osGroup: MacOS + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 + _PublishArtifacts: bin/OSX.x64.Release + ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 - - template: /eng/build.yml - parameters: - name: MacOS_arm64 - osGroup: MacOS_cross - crossbuild: true - buildAndSkipTest: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 - _PublishArtifacts: bin/OSX.arm64.Release - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - Build_Debug: - _BuildConfig: Debug - _BuildArch: arm64 + - template: /eng/pipelines/build.yml + parameters: + name: MacOS_arm64 + osGroup: MacOS_cross + crossBuild: true + buildOnly: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm64 + _PublishArtifacts: bin/OSX.arm64.Release + ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + Build_Debug: + _BuildConfig: Debug + _BuildArch: arm64 - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - template: /eng/build.yml - parameters: - name: Linux_arm - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-20210719121212-8a8d3be - crossrootfsDir: '/crossrootfs/arm' - buildAndSkipTest: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm - _PublishArtifacts: bin/Linux.arm.Release + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/pipelines/build.yml + parameters: + name: Linux_arm + osGroup: Linux + nativeBuildContainer: linux_arm + crossBuild: true + buildOnly: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm + _PublishArtifacts: bin/Linux.arm.Release - - template: /eng/build.yml - parameters: - name: Linux_arm64 - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-20210719121212-8a8d3be - crossrootfsDir: '/crossrootfs/arm64' - buildAndSkipTest: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 - _PublishArtifacts: bin/Linux.arm64.Release + - template: /eng/pipelines/build.yml + parameters: + name: Linux_arm64 + osGroup: Linux + nativeBuildContainer: linux_arm64 + crossBuild: true + buildOnly: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm64 + _PublishArtifacts: bin/Linux.arm64.Release - - template: /eng/build.yml - parameters: - name: Linux_musl_arm - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm-alpine-20210923140502-78f7860 - crossrootfsDir: '/crossrootfs/arm' - artifactsTargetPath: bin/Linux-musl.arm.Release - buildAndSkipTest: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm - _PublishArtifacts: bin/Linux.arm.Release + - template: /eng/pipelines/build.yml + parameters: + name: Linux_musl_arm + osGroup: Linux + osSuffix: -musl + nativeBuildContainer: linux_musl_arm + crossBuild: true + buildOnly: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm + _PublishArtifacts: bin/Linux.arm.Release + _ArtifactsTargetPath: bin/Linux-musl.arm.Release - - template: /eng/build.yml - parameters: - name: Linux_musl_arm64 - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-alpine-20210923140502-78f7860 - crossrootfsDir: '/crossrootfs/arm64' - artifactsTargetPath: bin/Linux-musl.arm64.Release - buildAndSkipTest: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 - _PublishArtifacts: bin/Linux.arm64.Release + - template: /eng/pipelines/build.yml + parameters: + name: Linux_musl_arm64 + osGroup: Linux + osSuffix: -musl + nativeBuildContainer: linux_musl_arm64 + crossBuild: true + buildOnly: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: arm64 + _PublishArtifacts: bin/Linux.arm64.Release + _ArtifactsTargetPath: bin/Linux-musl.arm64.Release - ############################ - # # - # Test only legs # - # # - ############################ + ############################ + # # + # Test only legs # + # # + ############################ - - template: /eng/build.yml - parameters: - name: Debian_Stretch - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-stretch-3e800f1-20190521154431 - dependsOn: CentOS_7 - testOnly: true - strategy: - matrix: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 + - template: /eng/pipelines/build.yml + parameters: + name: Ubuntu_20_04 + osGroup: Linux + container: test_ubuntu_20_04 + dependsOn: Linux_x64 + testOnly: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 + ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 -# - template: /eng/build.yml -# parameters: -# name: Fedora_34 -# osGroup: Linux -# dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-34-helix-20220331150839-4f64125 -# dependsOn: CentOS_7 -# testOnly: true -# requiresCapPtraceContainer: true -# strategy: -# matrix: -# Build_Debug: -# _BuildConfig: Debug -# _BuildArch: x64 + - template: /eng/pipelines/build.yml + parameters: + name: Alpine3_13 + osGroup: Linux + osSuffix: -musl + container: test_linux_musl_x64 + dependsOn: Linux_musl + testOnly: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 + ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 -# - template: /eng/build.yml -# parameters: -# name: OpenSuse_15_2 -# osGroup: Linux -# dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:opensuse-15.2-helix-amd64-20211018152525-9cc02fe -# dependsOn: CentOS_7 -# testOnly: true -# strategy: -# matrix: -# Build_Debug: -# _BuildConfig: Debug -# _BuildArch: x64 + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/pipelines/build.yml + parameters: + name: Debian_Bullseye + osGroup: Linux + container: test_debian_11_amd64 + dependsOn: Linux_x64 + testOnly: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 + ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 - - template: /eng/build.yml - parameters: - name: Ubuntu_16_04 - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-09ca40b-20190520220842 - dependsOn: CentOS_7 - testOnly: true - strategy: - matrix: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 + - template: /eng/pipelines/build.yml + parameters: + name: Fedora_36 + osGroup: Linux + container: test_fedora_36 + dependsOn: Linux_x64 + testOnly: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 + ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 - - template: /eng/build.yml - parameters: - name: Ubuntu_18_04 - osGroup: Linux - dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-3e800f1-20190508143252 - dependsOn: CentOS_7 - testOnly: true - strategy: - matrix: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 + #- template: /eng/pipelines/build.yml + # parameters: + # name: OpenSuse_15_2 + # osGroup: Linux + # container: test_opensuse_15_2 + # dependsOn: Linux_x64 + # testOnly: true + # strategy: + # matrix: + # Build_Release: + # _BuildConfig: Release + # _BuildArch: x64 + # ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + # Build_Debug: + # _BuildConfig: Debug + # _BuildArch: x64 - # Download, sign, package and publish - - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - - template: /eng/common/templates/job/job.yml - parameters: - name: Sign_Package_Publish - displayName: Sign, Package, and Generate BAR Manifests - dependsOn: - - Windows - - CentOS_7 - - Alpine3_13 - - MacOS - - MacOS_arm64 - - Linux_arm - - Linux_arm64 - - Linux_musl_arm - - Linux_musl_arm64 - condition: succeeded() - pool: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals windows.vs2022.amd64 - enablePublishUsingPipelines: true - enableMicrobuild: true - artifacts: - publish: - logs: - name: Logs_Packaging_Signing - steps: - - task: DownloadBuildArtifacts@0 - displayName: 'Download release builds' - inputs: - downloadPath: '$(Build.ArtifactStagingDirectory)/__download__' - artifactName: Build_Release - checkDownloadedFiles: true - - task: CopyFiles@2 - displayName: 'Binplace Product' - inputs: - sourceFolder: $(Build.ArtifactStagingDirectory)/__download__/Build_Release - targetFolder: '$(Build.SourcesDirectory)/artifacts/' + #- template: /eng/pipelines/build.yml + # parameters: + # name: Ubuntu_18_04 + # osGroup: Linux + # container: test_ubuntu_18_04 + # dependsOn: Linux_x64 + # testOnly: true + # strategy: + # matrix: + # Build_Release: + # _BuildConfig: Release + # _BuildArch: x64 + # ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + # Build_Debug: + # _BuildConfig: Debug + # _BuildArch: x64 - # Windows x64 download. Everything under "bin" is published for the Windows x64 build. - # Create nuget packages, sign binaries and publish to blob feed - - script: $(Build.SourcesDirectory)\eng\ci-prepare-artifacts.cmd $(_InternalBuildArgs) - displayName: Package, Sign, and Publish - continueOnError: false + - template: /eng/pipelines/build.yml + parameters: + name: Ubuntu_22_04 + osGroup: Linux + container: test_ubuntu_22_04 + dependsOn: Linux_x64 + testOnly: true + strategy: + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 + ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 + + # Download, sign, package and publish + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates/job/job.yml + parameters: + name: Sign_Package_Publish + displayName: Sign, Package, and Generate BAR Manifests + dependsOn: + - Windows + - MacOS + - MacOS_arm64 + - Linux_x64 + - Linux_musl + - Linux_arm + - Linux_arm64 + - Linux_musl_arm + - Linux_musl_arm64 condition: succeeded() + pool: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals windows.vs2022.amd64 + enablePublishUsingPipelines: true + enableMicrobuild: true + artifacts: + publish: + logs: + name: Logs_Packaging_Signing + steps: + - task: DownloadBuildArtifacts@0 + displayName: 'Download release builds' + inputs: + downloadPath: '$(Build.ArtifactStagingDirectory)/__download__' + artifactName: Build_Release + checkDownloadedFiles: true + - task: CopyFiles@2 + displayName: 'Binplace Product' + inputs: + sourceFolder: $(Build.ArtifactStagingDirectory)/__download__/Build_Release + targetFolder: '$(Build.SourcesDirectory)/artifacts/' - # Publish package and log build artifacts - - task: PublishBuildArtifacts@1 - displayName: Publish Package Artifacts - inputs: - publishLocation: Container - pathtoPublish: '$(Build.SourcesDirectory)/artifacts/packages' - artifactName: Packages - continueOnError: true - condition: always() + # Windows x64 download. Everything under "bin" is published for the Windows x64 build. + # Create nuget packages, sign binaries and publish to blob feed + - script: $(Build.SourcesDirectory)\eng\ci-prepare-artifacts.cmd $(_InternalBuildArgs) + displayName: Package, Sign, and Publish + continueOnError: false + condition: succeeded() - - task: PublishBuildArtifacts@1 - displayName: Publish Bundled Tools - inputs: - publishLocation: Container - pathtoPublish: '$(Build.SourcesDirectory)/artifacts/bundledtools' - artifactName: BundledTools - continueOnError: true - condition: always() + # Publish package and log build artifacts + - task: PublishBuildArtifacts@1 + displayName: Publish Package Artifacts + inputs: + publishLocation: Container + pathtoPublish: '$(Build.SourcesDirectory)/artifacts/packages' + artifactName: Packages + continueOnError: true + condition: always() - - template: /eng/common/templates/job/publish-build-assets.yml - parameters: - configuration: Release - dependsOn: Sign_Package_Publish - publishUsingPipelines: true - pool: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals windows.vs2022.amd64 + - task: PublishBuildArtifacts@1 + displayName: Publish Bundled Tools + inputs: + publishLocation: Container + pathtoPublish: '$(Build.SourcesDirectory)/artifacts/bundledtools' + artifactName: BundledTools + continueOnError: true + condition: always() - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/common/templates/post-build/post-build.yml - parameters: - # This is to enable SDL runs part of Post-Build Validation Stage. - # as well as NuGet, SourceLink, and signing validation. - # The variables get imported from group dotnet-diagnostics-sdl-params - publishingInfraVersion: 3 - enableSourceLinkValidation: true - enableSigningValidation: false - enableSymbolValidation: false - enableNugetValidation: true - symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=false' - publishInstallersAndChecksums: true - SDLValidationParameters: - enable: true - continueOnError: true - params: ' -SourceToolsList @("policheck","credscan") - -TsaInstanceURL $(_TsaInstanceURL) - -TsaProjectName $(_TsaProjectName) - -TsaNotificationEmail $(_TsaNotificationEmail) - -TsaCodebaseAdmin $(_TsaCodebaseAdmin) - -TsaBugAreaPath $(_TsaBugAreaPath) - -TsaIterationPath $(_TsaIterationPath) - -TsaRepositoryName "diagnostics" - -TsaCodebaseName "diagnostics" - -TsaPublish $True' - artifactNames: - - 'Packages' + - template: /eng/common/templates/job/publish-build-assets.yml + parameters: + configuration: Release + dependsOn: Sign_Package_Publish + publishUsingPipelines: true + pool: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals windows.vs2022.amd64 + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/common/templates/post-build/post-build.yml + parameters: + # This is to enable SDL runs part of Post-Build Validation Stage. + # as well as NuGet, SourceLink, and signing validation. + # The variables get imported from group dotnet-diagnostics-sdl-params + publishingInfraVersion: 3 + enableSourceLinkValidation: true + enableSigningValidation: false + enableSymbolValidation: false + enableNugetValidation: true + symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=false' + publishInstallersAndChecksums: true + SDLValidationParameters: + enable: true + continueOnError: true + params: ' -SourceToolsList @("policheck","credscan") + -TsaInstanceURL $(_TsaInstanceURL) + -TsaProjectName $(_TsaProjectName) + -TsaNotificationEmail $(_TsaNotificationEmail) + -TsaCodebaseAdmin $(_TsaCodebaseAdmin) + -TsaBugAreaPath $(_TsaBugAreaPath) + -TsaIterationPath $(_TsaIterationPath) + -TsaRepositoryName "diagnostics" + -TsaCodebaseName "diagnostics" + -TsaPublish $True' + artifactNames: + - 'Packages' - # This sets up the bits to do a Release. - - template: /eng/prepare-release.yml + # This sets up the bits to do a Release. + - template: /eng/pipelines/prepare-release.yml diff --git a/eng/CIBuild.cmd b/eng/CIBuild.cmd deleted file mode 100644 index df9ae6479c..0000000000 --- a/eng/CIBuild.cmd +++ /dev/null @@ -1,3 +0,0 @@ -@echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0build.ps1""" -restore -ci -prepareMachine %*" -exit /b %ErrorLevel% diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6950b9c51c..6c4edc2c12 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,60 +1,60 @@ - + https://github.com/dotnet/symstore - e09f81a0b38786cb20f66b589a8b88b6997a62da + 00f6edae1666690960cd207fd2b7a51232af9605 - + https://github.com/microsoft/clrmd - 3368bf4451a9441076595022fdff0f2bbea57b1b + 272986369826a777686ba616a00acd48febc2546 - + https://github.com/microsoft/clrmd - 3368bf4451a9441076595022fdff0f2bbea57b1b + 272986369826a777686ba616a00acd48febc2546 - + https://github.com/dotnet/arcade - b12f035e893c34ec2c965d75f6e21b7a2667e98d + 234e0726c7384ee84bf08550f2d16a1ff2d5c543 - + https://github.com/dotnet/arcade - b12f035e893c34ec2c965d75f6e21b7a2667e98d + 234e0726c7384ee84bf08550f2d16a1ff2d5c543 https://github.com/dotnet/arcade ccfe6da198c5f05534863bbb1bff66e830e0c6ab - + https://github.com/dotnet/installer - 51e06f6931e859f56564556fa6ba519761fa7141 + 18dc2cf11a2daaaa1633afd0c4225e188ce6c239 - + https://github.com/dotnet/aspnetcore - c0acf059eddd7e70498804dcc99a7c7b33732417 + c2488eead6ead7208f543d0a57104b5d167b93f9 - + https://github.com/dotnet/aspnetcore - c0acf059eddd7e70498804dcc99a7c7b33732417 + c2488eead6ead7208f543d0a57104b5d167b93f9 - + https://github.com/dotnet/runtime - a64420c79cb63c485138f4f1352b8730f27d7b19 + 2bf8f1aa83e192a307d5846424880cd61bec1a4f - + https://github.com/dotnet/runtime - a64420c79cb63c485138f4f1352b8730f27d7b19 + 2bf8f1aa83e192a307d5846424880cd61bec1a4f - + https://github.com/dotnet/source-build-reference-packages - dc842f8fab4bd38db9334a312a990d198b971fc2 + 4a3b4b6b37bdafe501477bf2e564380e1962ce61 - + https://github.com/dotnet/sourcelink - 3f43bf1b2dead2cb51f20dc47f6dfd7981248820 + 54eb3b811c57f5e94617d31a102fc9cb664ccdd5 diff --git a/eng/Versions.props b/eng/Versions.props index b513c48f5d..23f0ae30aa 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -16,26 +16,26 @@ - 1.0.417001 + 1.0.430201 - 8.0.0-preview.3.23155.1 - 8.0.0-preview.3.23155.1 + 8.0.0-preview.6.23304.2 + 8.0.0-preview.6.23304.2 - 8.0.0-preview.4.23179.5 - 8.0.0-preview.4.23179.5 + 8.0.0-preview.6.23302.1 + 8.0.0-preview.6.23302.1 - 8.0.100-preview.3.23156.1 + 8.0.100-preview.6.23305.2 - 6.0.14 + 6.0.16 $(MicrosoftNETCoreApp60Version) - 7.0.3 + 7.0.5 $(MicrosoftNETCoreApp70Version) $(MicrosoftNETCoreApp60Version) - 7.0.2 - 8.0.0-preview.2.23127.4 + $(MicrosoftNETCoreApp70Version) + 8.0.0-preview.6.23302.2 @@ -44,13 +44,10 @@ 5.0.0 - 1.1.0 - 3.0.0-beta.23205.1 + 6.0.0 + 3.0.0-beta.23302.1 16.9.0-beta1.21055.5 3.0.7 - - 2.1.1 - 6.0.0 6.0.0 @@ -58,18 +55,19 @@ 2.0.0-beta1.20468.1 2.0.0-beta1.20074.1 5.0.0 - 4.5.4 + 4.5.1 + 4.5.5 4.3.0 4.7.2 4.7.1 2.0.3 - 8.0.0-beta.23168.1 + 8.0.0-beta.23302.3 1.2.0-beta.406 7.0.0-beta.22316.2 10.0.18362 13.0.1 - 8.0.0-alpha.1.23178.3 - 1.2.0-beta-23165-02 + 8.0.0-alpha.1.23302.3 + 8.0.0-beta.23252.2 3.11.0 diff --git a/eng/build.ps1 b/eng/build.ps1 index b107d42088..6bf2d4f694 100644 --- a/eng/build.ps1 +++ b/eng/build.ps1 @@ -59,7 +59,7 @@ if ($cleanupprivatebuild) { # Install sdk for building, restore and build managed components. if (-not $skipmanaged) { - Invoke-Expression "& `"$engroot\common\build.ps1`" -build -configuration $configuration -verbosity $verbosity /p:BuildArch=$architecture /p:TestArchitectures=$architecture $remainingargs" + Invoke-Expression "& `"$engroot\common\build.ps1`" -configuration $configuration -verbosity $verbosity /p:BuildArch=$architecture /p:TestArchitectures=$architecture $remainingargs" if ($lastExitCode -ne 0) { exit $lastExitCode } diff --git a/eng/build.sh b/eng/build.sh index 08fa93f245..ae11013d07 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -41,14 +41,14 @@ usage_list+=("-test: run xunit tests") handle_arguments() { - lowerI="$(echo "$1" | tr "[:upper:]" "[:lower:]")" + lowerI="$(echo "${1/--/-}" | tr "[:upper:]" "[:lower:]")" case "$lowerI" in architecture|-architecture|-a) __BuildArch="$(echo "$2" | tr "[:upper:]" "[:lower:]")" __ShiftArgs=1 ;; - -binarylog|-bl|-clean|-integrationtest|-pack|-performancetest|-pipelineslog|-pl|-preparemachine|-publish|-r|-rebuild|-restore|-sign|-sb) + -binarylog|-bl|-clean|-integrationtest|-pack|-performancetest|-pipelineslog|-pl|-preparemachine|-publish|-r|-rebuild|-build|-restore|-sign|-sb) __ManagedBuildArgs="$__ManagedBuildArgs $1" ;; @@ -63,10 +63,6 @@ handle_arguments() { __ShiftArgs=1 ;; - -clean|-binarylog|-bl|-pipelineslog|-pl|-restore|-r|-rebuild|-pack|-integrationtest|-performancetest|-sign|-publish|-preparemachine|-sb) - __ManagedBuildArgs="$__ManagedBuildArgs $1" - ;; - -dotnetruntimeversion) __DotnetRuntimeVersion="$2" __ShiftArgs=1 @@ -148,11 +144,32 @@ fi # if [[ "$__ManagedBuild" == 1 ]]; then + echo "Commencing managed build for $__BuildType in $__RootBinDir/bin" - "$__RepoRootDir/eng/common/build.sh" --build --configuration "$__BuildType" $__CommonMSBuildArgs $__ManagedBuildArgs $__UnprocessedBuildArgs + "$__RepoRootDir/eng/common/build.sh" --configuration "$__BuildType" $__CommonMSBuildArgs $__ManagedBuildArgs $__UnprocessedBuildArgs + if [ "$?" != 0 ]; then exit 1 fi + + echo "Generating Version Source File" + __GenerateVersionLog="$__LogsDir/GenerateVersion.binlog" + + "$__RepoRootDir/eng/common/msbuild.sh" \ + $__RepoRootDir/eng/CreateVersionFile.proj \ + /bl:$__GenerateVersionLog \ + /t:GenerateVersionFiles \ + /restore \ + /p:GenerateVersionSourceFile=true \ + /p:NativeVersionSourceFile="$__ArtifactsIntermediatesDir/_version.c" \ + /p:Configuration="$__BuildType" \ + /p:Platform="$__BuildArch" \ + $__UnprocessedBuildArgs + + if [ $? != 0 ]; then + echo "Generating Version Source File FAILED" + exit 1 + fi fi # @@ -198,25 +215,6 @@ fi # Build native components # if [[ "$__NativeBuild" == 1 ]]; then - echo "Generating Version Source File" - __GenerateVersionLog="$__LogsDir/GenerateVersion.binlog" - - "$__RepoRootDir/eng/common/msbuild.sh" \ - $__RepoRootDir/eng/CreateVersionFile.proj \ - /bl:$__GenerateVersionLog \ - /t:GenerateVersionFiles \ - /restore \ - /p:GenerateVersionSourceFile=true \ - /p:NativeVersionSourceFile="$__ArtifactsIntermediatesDir/_version.c" \ - /p:Configuration="$__BuildType" \ - /p:Platform="$__BuildArch" \ - $__UnprocessedBuildArgs - - if [ $? != 0 ]; then - echo "Generating Version Source File FAILED" - exit 1 - fi - build_native "$__TargetOS" "$__BuildArch" "$__RepoRootDir" "$__IntermediatesDir" "install" "$__ExtraCmakeArgs" "diagnostic component" | tee "$__LogsDir"/make.log if [ "$?" != 0 ]; then @@ -251,19 +249,35 @@ fi if [[ "$__Test" == 1 ]]; then if [[ "$__CrossBuild" == 0 ]]; then if [[ -z "$LLDB_PATH" ]]; then - export LLDB_PATH="$(which lldb-3.9.1 2> /dev/null)" - if [[ -z "$LLDB_PATH" ]]; then - export LLDB_PATH="$(which lldb-3.9 2> /dev/null)" - if [[ -z "$LLDB_PATH" ]]; then - export LLDB_PATH="$(which lldb-4.0 2> /dev/null)" - if [[ -z "$LLDB_PATH" ]]; then - export LLDB_PATH="$(which lldb-5.0 2> /dev/null)" - if [[ -z "$LLDB_PATH" ]]; then - export LLDB_PATH="$(which lldb 2> /dev/null)" - fi - fi - fi + check_version_exists() { + desired_version=-1 + + # Set up the environment to be used for building with the desired debugger. + if command -v "lldb-$1.$2" > /dev/null; then + desired_version="-$1.$2" + elif command -v "lldb$1$2" > /dev/null; then + desired_version="$1$2" + elif command -v "lldb-$1$2" > /dev/null; then + desired_version="-$1$2" fi + + echo "$desired_version" + } + + # note: clang versions higher than 6 do not have minor version in file name, if it is zero. + versions="16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9" + for version in $versions; do + _major="${version%%.*}" + [ -z "${version##*.*}" ] && _minor="${version#*.}" + desired_version="$(check_version_exists "$_major" "$_minor")" + if [ "$desired_version" != "-1" ]; then majorVersion="$_major"; break; fi + done + + if [ -z "$majorVersion" ]; then + export LLDB_PATH="$(command -v "lldb")" + else + export LLDB_PATH="$(command -v "lldb$desired_version")" + fi fi if [[ -z "$GDB_PATH" ]]; then diff --git a/eng/ci-prepare-artifacts.cmd b/eng/ci-prepare-artifacts.cmd index 5632c47db3..af95f7c3aa 100644 --- a/eng/ci-prepare-artifacts.cmd +++ b/eng/ci-prepare-artifacts.cmd @@ -9,7 +9,7 @@ powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0common\Build.p if NOT '%ERRORLEVEL%' == '0' goto ExitWithCode echo Creating bundles -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" %_commonArgs% -bundletools %*" +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" %_commonArgs% -build -bundletools %*" if NOT '%ERRORLEVEL%' == '0' goto ExitWithCode echo Creating dbgshim packages diff --git a/eng/cibuild.sh b/eng/cibuild.sh deleted file mode 100755 index ffc534ba0a..0000000000 --- a/eng/cibuild.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. - -source="${BASH_SOURCE[0]}" - -# resolve $SOURCE until the file is no longer a symlink -while [[ -h $source ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - - # if $source was a relative symlink, we need to resolve it relative to the path where - # the symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done - -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -# Fix any CI lab docker image problems - -__osname=$(uname -s) -if [ "$__osname" == "Linux" ]; then - if [ -e /etc/os-release ]; then - source /etc/os-release - if [[ $ID == "ubuntu" ]]; then - if [[ $VERSION_ID == "18.04" ]]; then - # Fix the CI lab's ubuntu 18.04 docker image: install curl. - sudo apt-get update - sudo apt-get install -y curl - fi - fi - elif [ -e /etc/redhat-release ]; then - __redhatRelease=$( - #System.Exception: 'process launch -s' FAILED - # - # so we will keep using old image for now and install newer cmake as a workaround instead.. - # FIXME: delete this comment and the next `if` block once centos image is upgraded. - if [ "$ID" = "centos" ]; then - # upgrade cmake - requiredversion=3.6.2 - cmakeversion="$(cmake --version | head -1)" - currentversion="${cmakeversion##* }" - if ! printf '%s\n' "$requiredversion" "$currentversion" | sort --version-sort --check 2>/dev/null; then - echo "Old cmake version found: $currentversion, minimal requirement is $requiredversion. Upgrading to 3.15.5 .." - curl -sSL -o /tmp/cmake-install.sh https://github.com/Kitware/CMake/releases/download/v3.15.5/cmake-3.15.5-Linux-$(uname -m).sh - mkdir "$HOME/.cmake" - bash /tmp/cmake-install.sh --skip-license --exclude-subdir --prefix="$HOME/.cmake" - PATH="$HOME/.cmake/bin:$PATH" - export PATH - cmakeversion="$(cmake --version | head -1)" - newversion="${cmakeversion##* }" - echo "New cmake version is: $newversion" - fi - fi -fi - -"$scriptroot/build.sh" -restore -prepareMachine -ci $@ -if [[ $? != 0 ]]; then - exit 1 -fi diff --git a/eng/common/cross/arm/sources.list.xenial b/eng/common/cross/arm/sources.list.xenial index eacd86b7df..56fbb36a59 100644 --- a/eng/common/cross/arm/sources.list.xenial +++ b/eng/common/cross/arm/sources.list.xenial @@ -8,4 +8,4 @@ deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse \ No newline at end of file +deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse diff --git a/eng/common/cross/arm64/sources.list.xenial b/eng/common/cross/arm64/sources.list.xenial index eacd86b7df..56fbb36a59 100644 --- a/eng/common/cross/arm64/sources.list.xenial +++ b/eng/common/cross/arm64/sources.list.xenial @@ -8,4 +8,4 @@ deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse \ No newline at end of file +deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index ff113f7335..9caf9b021d 100644 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -14,6 +14,7 @@ usage() echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" echo "llvmx[.y] - optional, LLVM version for LLVM related packages." echo "--skipunmount - optional, will skip the unmount of rootfs folder." + echo "--skipsigcheck - optional, will skip package signature checks (allowing untrusted packages)." echo "--use-mirror - optional, use mirror URL to fetch resources, when available." echo "--jobs N - optional, restrict to N jobs." exit 1 @@ -26,6 +27,7 @@ __AlpineArch=armv7 __FreeBSDArch=arm __FreeBSDMachineArch=armv7 __IllumosArch=arm7 +__HaikuArch=arm __QEMUArch=arm __UbuntuArch=armhf __UbuntuRepo="http://ports.ubuntu.com/" @@ -69,7 +71,7 @@ __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" -__FreeBSDBase="12.3-RELEASE" +__FreeBSDBase="12.4-RELEASE" __FreeBSDPkg="1.17.0" __FreeBSDABI="12" __FreeBSDPackages="libunwind" @@ -84,8 +86,12 @@ __IllumosPackages+=" mit-krb5" __IllumosPackages+=" openssl" __IllumosPackages+=" zlib" -__HaikuPackages="gmp" +__HaikuPackages="gcc_syslibs" +__HaikuPackages+=" gcc_syslibs_devel" +__HaikuPackages+=" gmp" __HaikuPackages+=" gmp_devel" +__HaikuPackages+=" icu66" +__HaikuPackages+=" icu66_devel" __HaikuPackages+=" krb5" __HaikuPackages+=" krb5_devel" __HaikuPackages+=" libiconv" @@ -94,12 +100,36 @@ __HaikuPackages+=" llvm12_libunwind" __HaikuPackages+=" llvm12_libunwind_devel" __HaikuPackages+=" mpfr" __HaikuPackages+=" mpfr_devel" +__HaikuPackages+=" openssl" +__HaikuPackages+=" openssl_devel" +__HaikuPackages+=" zlib" +__HaikuPackages+=" zlib_devel" # ML.NET dependencies __UbuntuPackages+=" libomp5" __UbuntuPackages+=" libomp-dev" +# Taken from https://github.com/alpinelinux/alpine-chroot-install/blob/6d08f12a8a70dd9b9dc7d997c88aa7789cc03c42/alpine-chroot-install#L85-L133 +__AlpineKeys=' +4a6a0840:MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1yHJxQgsHQREclQu4Ohe\nqxTxd1tHcNnvnQTu/UrTky8wWvgXT+jpveroeWWnzmsYlDI93eLI2ORakxb3gA2O\nQ0Ry4ws8vhaxLQGC74uQR5+/yYrLuTKydFzuPaS1dK19qJPXB8GMdmFOijnXX4SA\njixuHLe1WW7kZVtjL7nufvpXkWBGjsfrvskdNA/5MfxAeBbqPgaq0QMEfxMAn6/R\nL5kNepi/Vr4S39Xvf2DzWkTLEK8pcnjNkt9/aafhWqFVW7m3HCAII6h/qlQNQKSo\nGuH34Q8GsFG30izUENV9avY7hSLq7nggsvknlNBZtFUcmGoQrtx3FmyYsIC8/R+B\nywIDAQAB +5243ef4b:MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvNijDxJ8kloskKQpJdx+\nmTMVFFUGDoDCbulnhZMJoKNkSuZOzBoFC94omYPtxnIcBdWBGnrm6ncbKRlR+6oy\nDO0W7c44uHKCFGFqBhDasdI4RCYP+fcIX/lyMh6MLbOxqS22TwSLhCVjTyJeeH7K\naA7vqk+QSsF4TGbYzQDDpg7+6aAcNzg6InNePaywA6hbT0JXbxnDWsB+2/LLSF2G\nmnhJlJrWB1WGjkz23ONIWk85W4S0XB/ewDefd4Ly/zyIciastA7Zqnh7p3Ody6Q0\nsS2MJzo7p3os1smGjUF158s6m/JbVh4DN6YIsxwl2OjDOz9R0OycfJSDaBVIGZzg\ncQIDAQAB +524d27bb:MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAr8s1q88XpuJWLCZALdKj\nlN8wg2ePB2T9aIcaxryYE/Jkmtu+ZQ5zKq6BT3y/udt5jAsMrhHTwroOjIsF9DeG\ne8Y3vjz+Hh4L8a7hZDaw8jy3CPag47L7nsZFwQOIo2Cl1SnzUc6/owoyjRU7ab0p\niWG5HK8IfiybRbZxnEbNAfT4R53hyI6z5FhyXGS2Ld8zCoU/R4E1P0CUuXKEN4p0\n64dyeUoOLXEWHjgKiU1mElIQj3k/IF02W89gDj285YgwqA49deLUM7QOd53QLnx+\nxrIrPv3A+eyXMFgexNwCKQU9ZdmWa00MjjHlegSGK8Y2NPnRoXhzqSP9T9i2HiXL\nVQIDAQAB +5261cecb:MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwlzMkl7b5PBdfMzGdCT0\ncGloRr5xGgVmsdq5EtJvFkFAiN8Ac9MCFy/vAFmS8/7ZaGOXoCDWbYVLTLOO2qtX\nyHRl+7fJVh2N6qrDDFPmdgCi8NaE+3rITWXGrrQ1spJ0B6HIzTDNEjRKnD4xyg4j\ng01FMcJTU6E+V2JBY45CKN9dWr1JDM/nei/Pf0byBJlMp/mSSfjodykmz4Oe13xB\nCa1WTwgFykKYthoLGYrmo+LKIGpMoeEbY1kuUe04UiDe47l6Oggwnl+8XD1MeRWY\nsWgj8sF4dTcSfCMavK4zHRFFQbGp/YFJ/Ww6U9lA3Vq0wyEI6MCMQnoSMFwrbgZw\nwwIDAQAB +58199dcc:MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3v8/ye/V/t5xf4JiXLXa\nhWFRozsnmn3hobON20GdmkrzKzO/eUqPOKTpg2GtvBhK30fu5oY5uN2ORiv2Y2ht\neLiZ9HVz3XP8Fm9frha60B7KNu66FO5P2o3i+E+DWTPqqPcCG6t4Znk2BypILcit\nwiPKTsgbBQR2qo/cO01eLLdt6oOzAaF94NH0656kvRewdo6HG4urbO46tCAizvCR\nCA7KGFMyad8WdKkTjxh8YLDLoOCtoZmXmQAiwfRe9pKXRH/XXGop8SYptLqyVVQ+\ntegOD9wRs2tOlgcLx4F/uMzHN7uoho6okBPiifRX+Pf38Vx+ozXh056tjmdZkCaV\naQIDAQAB +58cbb476:MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoSPnuAGKtRIS5fEgYPXD\n8pSGvKAmIv3A08LBViDUe+YwhilSHbYXUEAcSH1KZvOo1WT1x2FNEPBEFEFU1Eyc\n+qGzbA03UFgBNvArurHQ5Z/GngGqE7IarSQFSoqewYRtFSfp+TL9CUNBvM0rT7vz\n2eMu3/wWG+CBmb92lkmyWwC1WSWFKO3x8w+Br2IFWvAZqHRt8oiG5QtYvcZL6jym\nY8T6sgdDlj+Y+wWaLHs9Fc+7vBuyK9C4O1ORdMPW15qVSl4Lc2Wu1QVwRiKnmA+c\nDsH/m7kDNRHM7TjWnuj+nrBOKAHzYquiu5iB3Qmx+0gwnrSVf27Arc3ozUmmJbLj\nzQIDAQAB +58e4f17d:MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvBxJN9ErBgdRcPr5g4hV\nqyUSGZEKuvQliq2Z9SRHLh2J43+EdB6A+yzVvLnzcHVpBJ+BZ9RV30EM9guck9sh\nr+bryZcRHyjG2wiIEoduxF2a8KeWeQH7QlpwGhuobo1+gA8L0AGImiA6UP3LOirl\nI0G2+iaKZowME8/tydww4jx5vG132JCOScMjTalRsYZYJcjFbebQQolpqRaGB4iG\nWqhytWQGWuKiB1A22wjmIYf3t96l1Mp+FmM2URPxD1gk/BIBnX7ew+2gWppXOK9j\n1BJpo0/HaX5XoZ/uMqISAAtgHZAqq+g3IUPouxTphgYQRTRYpz2COw3NF43VYQrR\nbQIDAQAB +60ac2099:MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwR4uJVtJOnOFGchnMW5Y\nj5/waBdG1u5BTMlH+iQMcV5+VgWhmpZHJCBz3ocD+0IGk2I68S5TDOHec/GSC0lv\n6R9o6F7h429GmgPgVKQsc8mPTPtbjJMuLLs4xKc+viCplXc0Nc0ZoHmCH4da6fCV\ntdpHQjVe6F9zjdquZ4RjV6R6JTiN9v924dGMAkbW/xXmamtz51FzondKC52Gh8Mo\n/oA0/T0KsCMCi7tb4QNQUYrf+Xcha9uus4ww1kWNZyfXJB87a2kORLiWMfs2IBBJ\nTmZ2Fnk0JnHDb8Oknxd9PvJPT0mvyT8DA+KIAPqNvOjUXP4bnjEHJcoCP9S5HkGC\nIQIDAQAB +6165ee59:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAutQkua2CAig4VFSJ7v54\nALyu/J1WB3oni7qwCZD3veURw7HxpNAj9hR+S5N/pNeZgubQvJWyaPuQDm7PTs1+\ntFGiYNfAsiibX6Rv0wci3M+z2XEVAeR9Vzg6v4qoofDyoTbovn2LztaNEjTkB+oK\ntlvpNhg1zhou0jDVYFniEXvzjckxswHVb8cT0OMTKHALyLPrPOJzVtM9C1ew2Nnc\n3848xLiApMu3NBk0JqfcS3Bo5Y2b1FRVBvdt+2gFoKZix1MnZdAEZ8xQzL/a0YS5\nHd0wj5+EEKHfOd3A75uPa/WQmA+o0cBFfrzm69QDcSJSwGpzWrD1ScH3AK8nWvoj\nv7e9gukK/9yl1b4fQQ00vttwJPSgm9EnfPHLAtgXkRloI27H6/PuLoNvSAMQwuCD\nhQRlyGLPBETKkHeodfLoULjhDi1K2gKJTMhtbnUcAA7nEphkMhPWkBpgFdrH+5z4\nLxy+3ek0cqcI7K68EtrffU8jtUj9LFTUC8dERaIBs7NgQ/LfDbDfGh9g6qVj1hZl\nk9aaIPTm/xsi8v3u+0qaq7KzIBc9s59JOoA8TlpOaYdVgSQhHHLBaahOuAigH+VI\nisbC9vmqsThF2QdDtQt37keuqoda2E6sL7PUvIyVXDRfwX7uMDjlzTxHTymvq2Ck\nhtBqojBnThmjJQFgZXocHG8CAwEAAQ== +61666e3f:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAlEyxkHggKCXC2Wf5Mzx4\nnZLFZvU2bgcA3exfNPO/g1YunKfQY+Jg4fr6tJUUTZ3XZUrhmLNWvpvSwDS19ZmC\nIXOu0+V94aNgnhMsk9rr59I8qcbsQGIBoHzuAl8NzZCgdbEXkiY90w1skUw8J57z\nqCsMBydAueMXuWqF5nGtYbi5vHwK42PffpiZ7G5Kjwn8nYMW5IZdL6ZnMEVJUWC9\nI4waeKg0yskczYDmZUEAtrn3laX9677ToCpiKrvmZYjlGl0BaGp3cxggP2xaDbUq\nqfFxWNgvUAb3pXD09JM6Mt6HSIJaFc9vQbrKB9KT515y763j5CC2KUsilszKi3mB\nHYe5PoebdjS7D1Oh+tRqfegU2IImzSwW3iwA7PJvefFuc/kNIijfS/gH/cAqAK6z\nbhdOtE/zc7TtqW2Wn5Y03jIZdtm12CxSxwgtCF1NPyEWyIxAQUX9ACb3M0FAZ61n\nfpPrvwTaIIxxZ01L3IzPLpbc44x/DhJIEU+iDt6IMTrHOphD9MCG4631eIdB0H1b\n6zbNX1CXTsafqHRFV9XmYYIeOMggmd90s3xIbEujA6HKNP/gwzO6CDJ+nHFDEqoF\nSkxRdTkEqjTjVKieURW7Swv7zpfu5PrsrrkyGnsRrBJJzXlm2FOOxnbI2iSL1B5F\nrO5kbUxFeZUIDq+7Yv4kLWcCAwEAAQ== +616a9724:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnC+bR4bHf/L6QdU4puhQ\ngl1MHePszRC38bzvVFDUJsmCaMCL2suCs2A2yxAgGb9pu9AJYLAmxQC4mM3jNqhg\n/E7yuaBbek3O02zN/ctvflJ250wZCy+z0ZGIp1ak6pu1j14IwHokl9j36zNfGtfv\nADVOcdpWITFFlPqwq1qt/H3UsKVmtiF3BNWWTeUEQwKvlU8ymxgS99yn0+4OPyNT\nL3EUeS+NQJtDS01unau0t7LnjUXn+XIneWny8bIYOQCuVR6s/gpIGuhBaUqwaJOw\n7jkJZYF2Ij7uPb4b5/R3vX2FfxxqEHqssFSg8FFUNTZz3qNZs0CRVyfA972g9WkJ\nhPfn31pQYil4QGRibCMIeU27YAEjXoqfJKEPh4UWMQsQLrEfdGfb8VgwrPbniGfU\nL3jKJR3VAafL9330iawzVQDlIlwGl6u77gEXMl9K0pfazunYhAp+BMP+9ot5ckK+\nosmrqj11qMESsAj083GeFdfV3pXEIwUytaB0AKEht9DbqUfiE/oeZ/LAXgySMtVC\nsbC4ESmgVeY2xSBIJdDyUap7FR49GGrw0W49NUv9gRgQtGGaNVQQO9oGL2PBC41P\niWF9GLoX30HIz1P8PF/cZvicSSPkQf2Z6TV+t0ebdGNS5DjapdnCrq8m9Z0pyKsQ\nuxAL2a7zX8l5i1CZh1ycUGsCAwEAAQ== +616abc23:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA0MfCDrhODRCIxR9Dep1s\neXafh5CE5BrF4WbCgCsevyPIdvTeyIaW4vmO3bbG4VzhogDZju+R3IQYFuhoXP5v\nY+zYJGnwrgz3r5wYAvPnLEs1+dtDKYOgJXQj+wLJBW1mzRDL8FoRXOe5iRmn1EFS\nwZ1DoUvyu7/J5r0itKicZp3QKED6YoilXed+1vnS4Sk0mzN4smuMR9eO1mMCqNp9\n9KTfRDHTbakIHwasECCXCp50uXdoW6ig/xUAFanpm9LtK6jctNDbXDhQmgvAaLXZ\nLvFqoaYJ/CvWkyYCgL6qxvMvVmPoRv7OPcyni4xR/WgWa0MSaEWjgPx3+yj9fiMA\n1S02pFWFDOr5OUF/O4YhFJvUCOtVsUPPfA/Lj6faL0h5QI9mQhy5Zb9TTaS9jB6p\nLw7u0dJlrjFedk8KTJdFCcaGYHP6kNPnOxMylcB/5WcztXZVQD5WpCicGNBxCGMm\nW64SgrV7M07gQfL/32QLsdqPUf0i8hoVD8wfQ3EpbQzv6Fk1Cn90bZqZafg8XWGY\nwddhkXk7egrr23Djv37V2okjzdqoyLBYBxMz63qQzFoAVv5VoY2NDTbXYUYytOvG\nGJ1afYDRVWrExCech1mX5ZVUB1br6WM+psFLJFoBFl6mDmiYt0vMYBddKISsvwLl\nIJQkzDwtXzT2cSjoj3T5QekCAwEAAQ== +616ac3bc:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvaaoSLab+IluixwKV5Od\n0gib2YurjPatGIbn5Ov2DLUFYiebj2oJINXJSwUOO+4WcuHFEqiL/1rya+k5hLZt\nhnPL1tn6QD4rESznvGSasRCQNT2vS/oyZbTYJRyAtFkEYLlq0t3S3xBxxHWuvIf0\nqVxVNYpQWyM3N9RIeYBR/euXKJXileSHk/uq1I5wTC0XBIHWcthczGN0m9wBEiWS\n0m3cnPk4q0Ea8mUJ91Rqob19qETz6VbSPYYpZk3qOycjKosuwcuzoMpwU8KRiMFd\n5LHtX0Hx85ghGsWDVtS0c0+aJa4lOMGvJCAOvDfqvODv7gKlCXUpgumGpLdTmaZ8\n1RwqspAe3IqBcdKTqRD4m2mSg23nVx2FAY3cjFvZQtfooT7q1ItRV5RgH6FhQSl7\n+6YIMJ1Bf8AAlLdRLpg+doOUGcEn+pkDiHFgI8ylH1LKyFKw+eXaAml/7DaWZk1d\ndqggwhXOhc/UUZFQuQQ8A8zpA13PcbC05XxN2hyP93tCEtyynMLVPtrRwDnHxFKa\nqKzs3rMDXPSXRn3ZZTdKH3069ApkEjQdpcwUh+EmJ1Ve/5cdtzT6kKWCjKBFZP/s\n91MlRrX2BTRdHaU5QJkUheUtakwxuHrdah2F94lRmsnQlpPr2YseJu6sIE+Dnx4M\nCfhdVbQL2w54R645nlnohu8CAwEAAQ== +616adfeb:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAq0BFD1D4lIxQcsqEpQzU\npNCYM3aP1V/fxxVdT4DWvSI53JHTwHQamKdMWtEXetWVbP5zSROniYKFXd/xrD9X\n0jiGHey3lEtylXRIPxe5s+wXoCmNLcJVnvTcDtwx/ne2NLHxp76lyc25At+6RgE6\nADjLVuoD7M4IFDkAsd8UQ8zM0Dww9SylIk/wgV3ZkifecvgUQRagrNUdUjR56EBZ\nraQrev4hhzOgwelT0kXCu3snbUuNY/lU53CoTzfBJ5UfEJ5pMw1ij6X0r5S9IVsy\nKLWH1hiO0NzU2c8ViUYCly4Fe9xMTFc6u2dy/dxf6FwERfGzETQxqZvSfrRX+GLj\n/QZAXiPg5178hT/m0Y3z5IGenIC/80Z9NCi+byF1WuJlzKjDcF/TU72zk0+PNM/H\nKuppf3JT4DyjiVzNC5YoWJT2QRMS9KLP5iKCSThwVceEEg5HfhQBRT9M6KIcFLSs\nmFjx9kNEEmc1E8hl5IR3+3Ry8G5/bTIIruz14jgeY9u5jhL8Vyyvo41jgt9sLHR1\n/J1TxKfkgksYev7PoX6/ZzJ1ksWKZY5NFoDXTNYUgzFUTOoEaOg3BAQKadb3Qbbq\nXIrxmPBdgrn9QI7NCgfnAY3Tb4EEjs3ON/BNyEhUENcXOH6I1NbcuBQ7g9P73kE4\nVORdoc8MdJ5eoKBpO8Ww8HECAwEAAQ== +616ae350:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyduVzi1mWm+lYo2Tqt/0\nXkCIWrDNP1QBMVPrE0/ZlU2bCGSoo2Z9FHQKz/mTyMRlhNqTfhJ5qU3U9XlyGOPJ\npiM+b91g26pnpXJ2Q2kOypSgOMOPA4cQ42PkHBEqhuzssfj9t7x47ppS94bboh46\nxLSDRff/NAbtwTpvhStV3URYkxFG++cKGGa5MPXBrxIp+iZf9GnuxVdST5PGiVGP\nODL/b69sPJQNbJHVquqUTOh5Ry8uuD2WZuXfKf7/C0jC/ie9m2+0CttNu9tMciGM\nEyKG1/Xhk5iIWO43m4SrrT2WkFlcZ1z2JSf9Pjm4C2+HovYpihwwdM/OdP8Xmsnr\nDzVB4YvQiW+IHBjStHVuyiZWc+JsgEPJzisNY0Wyc/kNyNtqVKpX6dRhMLanLmy+\nf53cCSI05KPQAcGj6tdL+D60uKDkt+FsDa0BTAobZ31OsFVid0vCXtsbplNhW1IF\nHwsGXBTVcfXg44RLyL8Lk/2dQxDHNHzAUslJXzPxaHBLmt++2COa2EI1iWlvtznk\nOk9WP8SOAIj+xdqoiHcC4j72BOVVgiITIJNHrbppZCq6qPR+fgXmXa+sDcGh30m6\n9Wpbr28kLMSHiENCWTdsFij+NQTd5S47H7XTROHnalYDuF1RpS+DpQidT5tUimaT\nJZDr++FjKrnnijbyNF8b98UCAwEAAQ== +616db30d:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnpUpyWDWjlUk3smlWeA0\nlIMW+oJ38t92CRLHH3IqRhyECBRW0d0aRGtq7TY8PmxjjvBZrxTNDpJT6KUk4LRm\na6A6IuAI7QnNK8SJqM0DLzlpygd7GJf8ZL9SoHSH+gFsYF67Cpooz/YDqWrlN7Vw\ntO00s0B+eXy+PCXYU7VSfuWFGK8TGEv6HfGMALLjhqMManyvfp8hz3ubN1rK3c8C\nUS/ilRh1qckdbtPvoDPhSbTDmfU1g/EfRSIEXBrIMLg9ka/XB9PvWRrekrppnQzP\nhP9YE3x/wbFc5QqQWiRCYyQl/rgIMOXvIxhkfe8H5n1Et4VAorkpEAXdsfN8KSVv\nLSMazVlLp9GYq5SUpqYX3KnxdWBgN7BJoZ4sltsTpHQ/34SXWfu3UmyUveWj7wp0\nx9hwsPirVI00EEea9AbP7NM2rAyu6ukcm4m6ATd2DZJIViq2es6m60AE6SMCmrQF\nwmk4H/kdQgeAELVfGOm2VyJ3z69fQuywz7xu27S6zTKi05Qlnohxol4wVb6OB7qG\nLPRtK9ObgzRo/OPumyXqlzAi/Yvyd1ZQk8labZps3e16bQp8+pVPiumWioMFJDWV\nGZjCmyMSU8V6MB6njbgLHoyg2LCukCAeSjbPGGGYhnKLm1AKSoJh3IpZuqcKCk5C\n8CM1S15HxV78s9dFntEqIokCAwEAAQ== +' __Keyring= +__SkipSigCheck=0 __UseMirror=0 __UnprocessedBuildArgs= @@ -184,11 +214,13 @@ while :; do ;; x64) __BuildArch=x64 + __AlpineArch=x86_64 __UbuntuArch=amd64 __FreeBSDArch=amd64 __FreeBSDMachineArch=amd64 __illumosArch=x86_64 - __UbuntuRepo= + __HaikuArch=x86_64 + __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" ;; x86) __BuildArch=x86 @@ -308,7 +340,7 @@ while :; do ;; freebsd13) __CodeName=freebsd - __FreeBSDBase="13.0-RELEASE" + __FreeBSDBase="13.2-RELEASE" __FreeBSDABI="13" __SkipUnmount=1 ;; @@ -318,12 +350,14 @@ while :; do ;; haiku) __CodeName=haiku - __BuildArch=x64 __SkipUnmount=1 ;; --skipunmount) __SkipUnmount=1 ;; + --skipsigcheck) + __SkipSigCheck=1 + ;; --rootfsdir|-rootfsdir) shift __RootfsDir="$1" @@ -351,7 +385,6 @@ case "$__AlpineVersion" in edge) __AlpineLlvmLibsLookup=1 ;; *) if [[ "$__AlpineArch" =~ s390x|ppc64le ]]; then - echo boo __AlpineVersion=3.15 # minimum version that supports lldb-dev __AlpinePackages+=" llvm12-libs" elif [[ "$__AlpineArch" == "x86" ]]; then @@ -380,6 +413,11 @@ if [[ "$__BuildArch" == "armel" ]]; then __LLDB_Package="lldb-3.5-dev" fi +if [[ "$__CodeName" == "xenial" && "$__UbuntuArch" == "armhf" ]]; then + # libnuma-dev is not available on armhf for xenial + __UbuntuPackages="${__UbuntuPackages//libnuma-dev/}" +fi + __UbuntuPackages+=" ${__LLDB_Package:-}" if [[ -n "$__LLVM_MajorVersion" ]]; then @@ -406,13 +444,18 @@ __RootfsDir="$( cd "$__RootfsDir" && pwd )" if [[ "$__CodeName" == "alpine" ]]; then __ApkToolsVersion=2.12.11 + __ApkToolsSHA512SUM=53e57b49230da07ef44ee0765b9592580308c407a8d4da7125550957bb72cb59638e04f8892a18b584451c8d841d1c7cb0f0ab680cc323a3015776affaa3be33 __ApkToolsDir="$(mktemp -d)" + __ApkKeysDir="$(mktemp -d)" wget "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic//v$__ApkToolsVersion/x86_64/apk.static" -P "$__ApkToolsDir" + echo "$__ApkToolsSHA512SUM $__ApkToolsDir/apk.static" | sha512sum -c chmod +x "$__ApkToolsDir/apk.static" - mkdir -p "$__RootfsDir"/usr/bin - cp -v "/usr/bin/qemu-$__QEMUArch-static" "$__RootfsDir/usr/bin" + if [[ -f "/usr/bin/qemu-$__QEMUArch-static" ]]; then + mkdir -p "$__RootfsDir"/usr/bin + cp -v "/usr/bin/qemu-$__QEMUArch-static" "$__RootfsDir/usr/bin" + fi if [[ "$__AlpineVersion" == "edge" ]]; then version=edge @@ -420,17 +463,30 @@ if [[ "$__CodeName" == "alpine" ]]; then version="v$__AlpineVersion" fi + for line in $__AlpineKeys; do + id="${line%%:*}" + content="${line#*:}" + + echo -e "-----BEGIN PUBLIC KEY-----\n$content\n-----END PUBLIC KEY-----" > "$__ApkKeysDir/alpine-devel@lists.alpinelinux.org-$id.rsa.pub" + done + + if [[ "$__SkipSigCheck" == "1" ]]; then + __ApkSignatureArg="--allow-untrusted" + else + __ApkSignatureArg="--keys-dir $__ApkKeysDir" + fi + # initialize DB "$__ApkToolsDir/apk.static" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ - -U --allow-untrusted --root "$__RootfsDir" --arch "$__AlpineArch" --initdb add + -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" --initdb add if [[ "$__AlpineLlvmLibsLookup" == 1 ]]; then __AlpinePackages+=" $("$__ApkToolsDir/apk.static" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ - -U --allow-untrusted --root "$__RootfsDir" --arch "$__AlpineArch" \ + -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \ search 'llvm*-libs' | sort | tail -1 | sed 's/-[^-]*//2g')" fi @@ -438,7 +494,7 @@ if [[ "$__CodeName" == "alpine" ]]; then "$__ApkToolsDir/apk.static" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ - -U --allow-untrusted --root "$__RootfsDir" --arch "$__AlpineArch" \ + -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \ add $__AlpinePackages rm -r "$__ApkToolsDir" @@ -512,69 +568,61 @@ elif [[ "$__CodeName" == "illumos" ]]; then elif [[ "$__CodeName" == "haiku" ]]; then JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} - echo "Building Haiku sysroot for x86_64" + echo "Building Haiku sysroot for $__HaikuArch" mkdir -p "$__RootfsDir/tmp" - cd "$__RootfsDir/tmp" - git clone -b hrev56235 https://review.haiku-os.org/haiku - git clone -b btrev43195 https://review.haiku-os.org/buildtools - cd "$__RootfsDir/tmp/buildtools" && git checkout 7487388f5110021d400b9f3b88e1a7f310dc066d - - # Fetch some unmerged patches - cd "$__RootfsDir/tmp/haiku" - ## Add development build profile (slimmer than nightly) - git fetch origin refs/changes/64/4164/1 && git -c commit.gpgsign=false cherry-pick FETCH_HEAD - - # Build jam - cd "$__RootfsDir/tmp/buildtools/jam" - make - - # Configure cross tools - echo "Building cross-compiler" - mkdir -p "$__RootfsDir/generated" - cd "$__RootfsDir/generated" - "$__RootfsDir/tmp/haiku/configure" -j"$JOBS" --sysroot "$__RootfsDir" --cross-tools-source "$__RootfsDir/tmp/buildtools" --build-cross-tools x86_64 - - # Build Haiku packages - echo "Building Haiku" - echo 'HAIKU_BUILD_PROFILE = "development-raw" ;' > UserProfileConfig - "$__RootfsDir/tmp/buildtools/jam/jam0" -j"$JOBS" -q 'package' 'Haiku' - - BaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg" - - # Download additional packages - echo "Downloading additional required packages" + pushd "$__RootfsDir/tmp" + + mkdir "$__RootfsDir/tmp/download" + + echo "Downloading Haiku package tool" + git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 $__RootfsDir/tmp/script + wget -O "$__RootfsDir/tmp/download/hosttools.zip" $($__RootfsDir/tmp/script/fetch.sh --hosttools) + unzip -o "$__RootfsDir/tmp/download/hosttools.zip" -d "$__RootfsDir/tmp/bin" + + DepotBaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg" + HpkgBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current" + + # Download Haiku packages + echo "Downloading Haiku packages" read -ra array <<<"$__HaikuPackages" for package in "${array[@]}"; do echo "Downloading $package..." # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60 # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598 - hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_x86_64","versionType":"LATEST","naturalLanguageCode":"en"}' \ - --header='Content-Type:application/json' "$BaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" - wget -P "$__RootfsDir/generated/download" "$hpkgDownloadUrl" + hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \ + --header='Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" + wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl" + done + for package in haiku haiku_devel; do + echo "Downloading $package..." + hpkgVersion="$(wget -qO- $HpkgBaseUrl | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" + wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" done - # Setup the sysroot - echo "Setting up sysroot and extracting needed packages" + # Set up the sysroot + echo "Setting up sysroot and extracting required packages" mkdir -p "$__RootfsDir/boot/system" - for file in "$__RootfsDir/generated/objects/haiku/x86_64/packaging/packages/"*.hpkg; do - "$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file" - done - for file in "$__RootfsDir/generated/download/"*.hpkg; do - "$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file" + for file in "$__RootfsDir/tmp/download/"*.hpkg; do + echo "Extracting $file..." + LD_LIBRARY_PATH="$__RootfsDir/tmp/bin" "$__RootfsDir/tmp/bin/package" extract -C "$__RootfsDir/boot/system" "$file" done + # Download buildtools + echo "Downloading Haiku buildtools" + wget -O "$__RootfsDir/tmp/download/buildtools.zip" $($__RootfsDir/tmp/script/fetch.sh --buildtools --arch=$__HaikuArch) + unzip -o "$__RootfsDir/tmp/download/buildtools.zip" -d "$__RootfsDir" + # Cleaning up temporary files echo "Cleaning up temporary files" + popd rm -rf "$__RootfsDir/tmp" - for name in "$__RootfsDir/generated/"*; do - if [[ "$name" =~ "cross-tools-" ]]; then - : # Keep the cross-compiler - else - rm -rf "$name" - fi - done elif [[ -n "$__CodeName" ]]; then - qemu-debootstrap $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" + + if [[ "$__SkipSigCheck" == "0" ]]; then + __Keyring="$__Keyring --force-check-gpg" + fi + + debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" cp "$__CrossDir/$__BuildArch/sources.list.$__CodeName" "$__RootfsDir/etc/apt/sources.list" chroot "$__RootfsDir" apt-get update chroot "$__RootfsDir" apt-get -f -y install diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index ccfb9951a5..a88d643c8a 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -6,6 +6,7 @@ unset(FREEBSD) unset(ILLUMOS) unset(ANDROID) unset(TIZEN) +unset(HAIKU) set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) @@ -16,6 +17,7 @@ elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) set(ILLUMOS 1) elseif(EXISTS ${CROSS_ROOTFS}/boot/system/develop/headers/config/HaikuConfig.h) set(CMAKE_SYSTEM_NAME Haiku) + set(HAIKU 1) else() set(CMAKE_SYSTEM_NAME Linux) set(LINUX 1) @@ -67,16 +69,30 @@ elseif(TARGET_ARCH_NAME STREQUAL "armv6") endif() elseif(TARGET_ARCH_NAME STREQUAL "ppc64le") set(CMAKE_SYSTEM_PROCESSOR ppc64le) - set(TOOLCHAIN "powerpc64le-linux-gnu") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/powerpc64le-alpine-linux-musl) + set(TOOLCHAIN "powerpc64le-alpine-linux-musl") + else() + set(TOOLCHAIN "powerpc64le-linux-gnu") + endif() elseif(TARGET_ARCH_NAME STREQUAL "riscv64") set(CMAKE_SYSTEM_PROCESSOR riscv64) - set(TOOLCHAIN "riscv64-linux-gnu") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/riscv64-alpine-linux-musl) + set(TOOLCHAIN "riscv64-alpine-linux-musl") + else() + set(TOOLCHAIN "riscv64-linux-gnu") + endif() elseif(TARGET_ARCH_NAME STREQUAL "s390x") set(CMAKE_SYSTEM_PROCESSOR s390x) - set(TOOLCHAIN "s390x-linux-gnu") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/s390x-alpine-linux-musl) + set(TOOLCHAIN "s390x-alpine-linux-musl") + else() + set(TOOLCHAIN "s390x-linux-gnu") + endif() elseif(TARGET_ARCH_NAME STREQUAL "x64") set(CMAKE_SYSTEM_PROCESSOR x86_64) - if(LINUX) + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/x86_64-alpine-linux-musl) + set(TOOLCHAIN "x86_64-alpine-linux-musl") + elseif(LINUX) set(TOOLCHAIN "x86_64-linux-gnu") if(TIZEN) set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu/9.2.0") @@ -86,11 +102,15 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64") elseif(ILLUMOS) set(TOOLCHAIN "x86_64-illumos") elseif(HAIKU) - set(TOOLCHAIN "x64_64-unknown-haiku") + set(TOOLCHAIN "x86_64-unknown-haiku") endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") set(CMAKE_SYSTEM_PROCESSOR i686) - set(TOOLCHAIN "i686-linux-gnu") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) + set(TOOLCHAIN "i586-alpine-linux-musl") + else() + set(TOOLCHAIN "i686-linux-gnu") + endif() if(TIZEN) set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0") endif() @@ -196,10 +216,8 @@ elseif(HAIKU) return() endif() - set(SEARCH_PATH "${CROSS_ROOTFS}/generated/cross-tools-x86_64/bin") - find_program(EXEC_LOCATION_${exec} - PATHS ${SEARCH_PATH} + PATHS "${CROSS_ROOTFS}/cross-tools-x86_64/bin" NAMES "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" "${TOOLSET_PREFIX}${exec}") @@ -264,8 +282,11 @@ elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64)$") add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) + add_toolchain_linker_flag("--target=${TOOLCHAIN}") + add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}") + endif() add_toolchain_linker_flag(-m32) - if(TIZEN) add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") @@ -275,11 +296,14 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") elseif(ILLUMOS) add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/amd64/lib") +elseif(HAIKU) + add_toolchain_linker_flag("-lnetwork") + add_toolchain_linker_flag("-lroot") endif() # Specify compile options -if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|ppc64le|riscv64|s390x)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU) +if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) @@ -298,10 +322,16 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") add_definitions (-DCLR_ARM_FPU_CAPABILITY=${CLR_ARM_FPU_CAPABILITY}) + # persist variables across multiple try_compile passes + list(APPEND CMAKE_TRY_COMPILE_PLATFORM_VARIABLES CLR_ARM_FPU_TYPE CLR_ARM_FPU_CAPABILITY) + if(TARGET_ARCH_NAME STREQUAL "armel") add_compile_options(-mfloat-abi=softfp) endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) + add_compile_options(--target=${TOOLCHAIN}) + endif() add_compile_options(-m32) add_compile_options(-Wno-error=unused-command-line-argument) endif() diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index 7aee4213e1..517401b688 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -64,7 +64,7 @@ if [ -z "$CLR_CC" ]; then if [ -z "$majorVersion" ]; then # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. if [ "$compiler" = "clang" ]; then versions="16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" - elif [ "$compiler" = "gcc" ]; then versions="12 11 10 9 8 7 6 5 4.9"; fi + elif [ "$compiler" = "gcc" ]; then versions="13 12 11 10 9 8 7 6 5 4.9"; fi for version in $versions; do _major="${version%%.*}" diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index e10a596879..6c4ac6fec1 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.4.1" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.6.0-2" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index b214a31db2..e20ee3a983 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -25,7 +25,7 @@ parameters: enablePublishTestResults: false enablePublishUsingPipelines: false enableBuildRetry: false - disableComponentGovernance: false + disableComponentGovernance: '' componentGovernanceIgnoreDirectories: '' mergeTestResults: false testRunTitle: '' @@ -75,6 +75,10 @@ jobs: - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: - name: EnableRichCodeNavigation value: 'true' + # Retry signature validation up to three times, waiting 2 seconds between attempts. + # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures + - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY + value: 3,2000 - ${{ each variable in parameters.variables }}: # handle name-value variable syntax # example: @@ -83,7 +87,7 @@ jobs: - ${{ if ne(variable.name, '') }}: - name: ${{ variable.name }} value: ${{ variable.value }} - + # handle variable groups - ${{ if ne(variable.group, '') }}: - group: ${{ variable.group }} @@ -155,16 +159,21 @@ jobs: uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} continueOnError: true - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(parameters.disableComponentGovernance, 'true')) }}: - - task: ComponentGovernanceComponentDetection@0 - continueOnError: true - inputs: - ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + - template: /eng/common/templates/steps/component-governance.yml + parameters: + ${{ if eq(parameters.disableComponentGovernance, '') }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + disableComponentGovernance: false + ${{ else }}: + disableComponentGovernance: true + ${{ else }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: MicroBuildCleanup@1 - displayName: Execute Microbuild cleanup tasks + displayName: Execute Microbuild cleanup tasks condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} env: @@ -214,7 +223,7 @@ jobs: displayName: Publish XUnit Test Results inputs: testResultsFormat: 'xUnit' - testResultsFiles: '*.xml' + testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit mergeTestResults: ${{ parameters.mergeTestResults }} @@ -225,7 +234,7 @@ jobs: displayName: Publish TRX Test Results inputs: testResultsFormat: 'VSTest' - testResultsFiles: '*.trx' + testResultsFiles: '*.trx' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx mergeTestResults: ${{ parameters.mergeTestResults }} diff --git a/eng/common/templates/steps/component-governance.yml b/eng/common/templates/steps/component-governance.yml new file mode 100644 index 0000000000..0ecec47b0c --- /dev/null +++ b/eng/common/templates/steps/component-governance.yml @@ -0,0 +1,13 @@ +parameters: + disableComponentGovernance: false + componentGovernanceIgnoreDirectories: '' + +steps: +- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: + - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable +- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: + - task: ComponentGovernanceComponentDetection@0 + continueOnError: true + inputs: + ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index a97a185a36..1100521834 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -68,6 +68,11 @@ steps: runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' fi + baseOsArgs= + if [ '${{ parameters.platform.baseOS }}' != '' ]; then + baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' + fi + publishArgs= if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then publishArgs='--publish' @@ -86,6 +91,7 @@ steps: $internalRestoreArgs \ $targetRidArgs \ $runtimeOsArgs \ + $baseOsArgs \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ /p:ArcadeBuildFromSource=true \ /p:AssetManifestFileName=$assetManifestFileName diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 8ad03be3ec..ffe0b4e2df 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -287,6 +287,25 @@ function InstallDotNet([string] $dotnetRoot, [string] $runtimeSourceFeedKey = '', [switch] $noPath) { + $dotnetVersionLabel = "'sdk v$version'" + + if ($runtime -ne '' -and $runtime -ne 'sdk') { + $runtimePath = $dotnetRoot + $runtimePath = $runtimePath + "\shared" + if ($runtime -eq "dotnet") { $runtimePath = $runtimePath + "\Microsoft.NETCore.App" } + if ($runtime -eq "aspnetcore") { $runtimePath = $runtimePath + "\Microsoft.AspNetCore.App" } + if ($runtime -eq "windowsdesktop") { $runtimePath = $runtimePath + "\Microsoft.WindowsDesktop.App" } + $runtimePath = $runtimePath + "\" + $version + + $dotnetVersionLabel = "runtime toolset '$runtime/$architecture v$version'" + + if (Test-Path $runtimePath) { + Write-Host " Runtime toolset '$runtime/$architecture v$version' already installed." + $installSuccess = $true + Exit + } + } + $installScript = GetDotNetInstallScript $dotnetRoot $installParameters = @{ Version = $version @@ -323,18 +342,18 @@ function InstallDotNet([string] $dotnetRoot, } else { $location = "public location"; } - Write-Host "Attempting to install dotnet from $location." + Write-Host " Attempting to install $dotnetVersionLabel from $location." try { & $installScript @variation $installSuccess = $true break } catch { - Write-Host "Failed to install dotnet from $location." + Write-Host " Failed to install $dotnetVersionLabel from $location." } } if (-not $installSuccess) { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from any of the specified locations." + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install $dotnetVersionLabel from any of the specified locations." ExitWithExitCode 1 } } @@ -365,8 +384,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=17.4.1&view=overview - $defaultXCopyMSBuildVersion = '17.4.1' + # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.6.0-2 + $defaultXCopyMSBuildVersion = '17.6.0-2' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { @@ -399,7 +418,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # Locate Visual Studio installation or download x-copy msbuild. $vsInfo = LocateVisualStudio $vsRequirements if ($vsInfo -ne $null) { - $vsInstallDir = $vsInfo.installationPath + # Ensure vsInstallDir has a trailing slash + $vsInstallDir = Join-Path $vsInfo.installationPath "\" $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0] InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion diff --git a/eng/common/tools.sh b/eng/common/tools.sh index cf9fb1ea2d..e8d4789433 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -184,6 +184,35 @@ function InstallDotNetSdk { function InstallDotNet { local root=$1 local version=$2 + local runtime=$4 + + local dotnetVersionLabel="'$runtime v$version'" + if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then + runtimePath="$root" + runtimePath="$runtimePath/shared" + case "$runtime" in + dotnet) + runtimePath="$runtimePath/Microsoft.NETCore.App" + ;; + aspnetcore) + runtimePath="$runtimePath/Microsoft.AspNetCore.App" + ;; + windowsdesktop) + runtimePath="$runtimePath/Microsoft.WindowsDesktop.App" + ;; + *) + ;; + esac + runtimePath="$runtimePath/$version" + + dotnetVersionLabel="runtime toolset '$runtime/$architecture v$version'" + + if [ -d "$runtimePath" ]; then + echo " Runtime toolset '$runtime/$architecture v$version' already installed." + local installSuccess=1 + return + fi + fi GetDotNetInstallScript "$root" local install_script=$_GetDotNetInstallScript @@ -228,17 +257,17 @@ function InstallDotNet { for variationName in "${variations[@]}"; do local name="$variationName[@]" local variation=("${!name}") - echo "Attempting to install dotnet from $variationName." + echo " Attempting to install $dotnetVersionLabel from $variationName." bash "$install_script" "${variation[@]}" && installSuccess=1 if [[ "$installSuccess" -eq 1 ]]; then break fi - echo "Failed to install dotnet from $variationName." + echo " Failed to install $dotnetVersionLabel from $variationName." done if [[ "$installSuccess" -eq 0 ]]; then - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from any of the specified locations." + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install $dotnetVersionLabel from any of the specified locations." ExitWithExitCode 1 fi } diff --git a/eng/docker-build.sh b/eng/docker-build.sh deleted file mode 100755 index 99cfc0d047..0000000000 --- a/eng/docker-build.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. - -source_directory= -docker_image= -docker_container_name= - -while [ $# -ne 0 ]; do - name=$1 - case $name in - -s|--source-directory) - shift - source_directory=$1 - ;; - -i|--docker-image) - shift - docker_image=$1 - ;; - -c|--container-name) - shift - docker_container_name=$1 - ;; - *) - args="$args $1" - ;; - esac - shift -done - -echo "Initialize Docker Container" -if command -v docker > /dev/null; then - docker_bin=$(command -v docker) -else - echo "Unable to find docker" - exit 1 -fi - -$docker_bin --version - -# Get user id -user_name=$(whoami) -echo "user name: $user_name" -user_id=$(id -u $user_name) -echo "user id: $user_id" - -# Download image -$docker_bin pull $docker_image - -# Create local network to avoid port conflicts when multiple agents run on same machine -$docker_bin network create vsts_network_$docker_container_name - -# Create and start container -docker_id="$($docker_bin create -it --rm --security-opt seccomp=unconfined --ulimit core=-1 \ - --name vsts_container_$docker_container_name \ - --network=vsts_network_$docker_container_name \ - --volume $source_directory:$source_directory \ - --workdir=$source_directory $docker_image bash --verbose)" -$docker_bin start $docker_id - -# Create an user with the same uid in the container -container_user_name=vsts_$(echo $user_name | awk '{print tolower($0)}') -echo "container user name: $container_user_name" - -# Add sudo user with same uid that can run any sudo command without password -$docker_bin exec $docker_id useradd -K MAIL_DIR=/dev/null -m -u $user_id $container_user_name -$docker_bin exec $docker_id groupadd sudouser -$docker_bin exec $docker_id usermod -a -G sudouser $container_user_name -$docker_bin exec $docker_id su -c "echo '%sudouser ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers" - -echo "Execute $args" -$docker_bin exec --workdir=$source_directory --user $container_user_name $docker_id $args -lasterrorcode=$? - -echo "Cleanup Docker Container/Network" -$docker_bin container stop $docker_id -$docker_bin network rm vsts_network_$docker_container_name - -exit $lasterrorcode diff --git a/eng/build.yml b/eng/pipelines/build.yml similarity index 68% rename from eng/build.yml rename to eng/pipelines/build.yml index aaf24175b3..48d483e8e1 100644 --- a/eng/build.yml +++ b/eng/pipelines/build.yml @@ -3,24 +3,26 @@ parameters: name: '' # Agent OS (Windows_NT, Linux, MacOS, FreeBSD) osGroup: Windows_NT + # Optional: OS suffix like -musl + osSuffix: '' # Additional variables variables: {} # Build strategy - matrix strategy: '' # Optional: Job timeout timeoutInMinutes: 180 - # Optional: Docker image to use - dockerImage: '' - # Optional: ROOTFS_DIR to use - crossrootfsDir: '' - crossbuild: false + # Optional: native build container resource name + nativeBuildContainer: '' + # Optional: container resource name + container: '' + # Optional: build only job if true + buildOnly: false # Optional: test only job if true testOnly: false - buildAndSkipTest: false - # Depends on + # Optional: architecture cross build if true + crossBuild: false + # Depends on dependsOn: '' - artifactsTargetPath: '' - requiresCapPtraceContainer: false isCodeQLRun: false jobs: @@ -62,8 +64,8 @@ jobs: name: NetCore-Svc-Public demands: ImageOverride -equals windows.vs2022.amd64.open - ${{ if and(ne(parameters.dockerImage, ''), ne(parameters.requiresCapPtraceContainer, 'true')) }}: - container: ${{ parameters.dockerImage }} + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} ${{ if ne(parameters.strategy, '') }}: strategy: ${{ parameters.strategy }} @@ -76,27 +78,28 @@ jobs: variables: - ${{ insert }}: ${{ parameters.variables }} - - _DockerImageName: ${{ parameters.dockerImage }} - _PhaseName : ${{ parameters.name }} - _HelixType: build/product - _HelixBuildConfig: $(_BuildConfig) - _Pipeline_StreamDumpDir: $(Build.SourcesDirectory)/artifacts/tmp/$(_BuildConfig)/streams + - _BuildDisplayName: 'Build / Test' + - _ExtraBuildParams: '' + - _TestArgs: '-test' + - _Cross: '' + - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - _buildScript: $(Build.SourcesDirectory)\eng\cibuild.cmd + - _buildScript: $(Build.SourcesDirectory)\build.cmd - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - _buildScript: $(Build.SourcesDirectory)/eng/cibuild.sh - - - _TestArgs: '-test' - - _dockerEnv: '' + - _buildScript: $(Build.SourcesDirectory)/build.sh - ${{ if eq(parameters.testOnly, 'true') }}: - _TestArgs: '-test -skipnative' - - ${{ if eq(parameters.requiresCapPtraceContainer, 'true') }}: - - _dockerEnv: $(Build.SourcesDirectory)/eng/docker-build.sh - --docker-image $(_DockerImageName) - --source-directory $(Build.SourcesDirectory) - --container-name diagnostics-$(Build.BuildId) + - _BuildDisplayName: Test + + - ${{ if or(eq(parameters.buildOnly, 'true'), eq(parameters.isCodeQLRun, 'true')) }}: + - _TestArgs: '' + - ${{ if eq(parameters.isCodeQLRun, 'true') }}: - name: Codeql.Enabled value: True @@ -109,11 +112,8 @@ jobs: - name: Codeql.Language value: csharp,cpp - - ${{ if or(eq(parameters.buildAndSkipTest, 'true'), eq(parameters.isCodeQLRun, 'true')) }}: - - _TestArgs: '' - - - _InternalInstallArgs: '' # For testing msrc's and service releases. The RuntimeSourceVersion is either "default" or the service release version to test + - _InternalInstallArgs: '' - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.isCodeQLRun, 'false')) }}: - _InternalInstallArgs: -dotnetruntimeversion '$(DotnetRuntimeVersion)' @@ -128,62 +128,88 @@ jobs: - _HelixSource: pr/dotnet/arcade/$(Build.SourceBranch) # This is only required for cross builds. - - ${{ if and(eq(parameters.crossbuild, false), eq(parameters.crossrootfsDir, '')) }}: - - _Cross: '' - - ${{ if or(eq(parameters.crossbuild, true), ne(parameters.crossrootfsDir, '')) }}: + - ${{ if eq(parameters.crossBuild, true) }}: - _Cross: -cross + # If there is a native build container, build managed in the host vm/container and native in the nativeBuildContainer + - ${{ if ne(parameters.nativeBuildContainer, '') }}: + - _ExtraBuildParams: -skipnative + - _BuildDisplayName: 'Build Managed' + + # Only add the cross build option if a combined build/test managed/native build (i.e. MacOS arm64) + - ${{ if eq(parameters.nativeBuildContainer, '') }}: + - _ExtraBuildParams: $(_Cross) + steps: - ${{ if eq(parameters.osGroup, 'Linux') }}: - ${{ if eq(parameters.testOnly, 'true') }}: - task: DownloadBuildArtifacts@0 - displayName: 'Download release builds' + displayName: 'Download Build Artifacts' inputs: downloadPath: '$(Build.ArtifactStagingDirectory)/__download__' - downloadType: specific + downloadType: specific itemPattern: | - Build_$(_BuildConfig)/bin/Linux.$(_BuildArch).$(_BuildConfig)/** + Build_$(_BuildConfig)/bin/Linux${{ parameters.osSuffix }}.$(_BuildArch).$(_BuildConfig)/** checkDownloadedFiles: true - task: CopyFiles@2 displayName: 'Binplace Product' inputs: - sourceFolder: $(Build.ArtifactStagingDirectory)/__download__/Build_$(_BuildConfig)/bin/Linux.$(_BuildArch).$(_BuildConfig) + sourceFolder: $(Build.ArtifactStagingDirectory)/__download__/Build_$(_BuildConfig)/bin/Linux${{ parameters.osSuffix }}.$(_BuildArch).$(_BuildConfig) targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/Linux.$(_BuildArch).$(_BuildConfig)' - ${{ if eq(parameters.isCodeQLRun, 'true') }}: - task: CodeQL3000Init@0 displayName: CodeQL Initialize - - script: $(_dockerEnv) $(_buildScript) - -configuration $(_BuildConfig) + - script: $(_buildScript) + -ci + -configuration $(_BuildConfig) -architecture $(_BuildArch) - $(_Cross) + $(_ExtraBuildParams) $(_TestArgs) /p:OfficialBuildId=$(BUILD.BUILDNUMBER) $(_InternalInstallArgs) - displayName: Build / Test + displayName: $(_BuildDisplayName) condition: succeeded() - env: - ROOTFS_DIR: ${{ parameters.crossrootfsDir }} + + - ${{ if ne(parameters.nativeBuildContainer, '') }}: + - script: $(_buildScript) + -ci + -configuration $(_BuildConfig) + -architecture $(_BuildArch) + -skipmanaged + $(_Cross) + /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + $(_InternalInstallArgs) + displayName: Build Native + target: ${{ parameters.nativeBuildContainer }} - ${{ if eq(parameters.isCodeQLRun, 'true') }}: - task: CodeQL3000Finalize@0 displayName: CodeQL Finalize - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: CopyFiles@2 - displayName: Gather binaries for publish to artifacts - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/$(_PublishArtifacts)' - Contents: '**' - TargetFolder: $(Build.ArtifactStagingDirectory)/artifacts/${{ coalesce(parameters.artifactsTargetPath, '$(_PublishArtifacts)') }} - condition: ne(variables['_PublishArtifacts'], '') - - task: PublishBuildArtifacts@1 - displayName: Publish Build Artifacts - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - artifactName: Build_$(_BuildConfig) - condition: ne(variables['_PublishArtifacts'], '') + - task: CopyFiles@2 + displayName: Gather binaries for publish to special artifacts path + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/$(_PublishArtifacts)' + Contents: '**' + TargetFolder: $(Build.ArtifactStagingDirectory)/artifacts/$(_ArtifactsTargetPath) + condition: and(ne(variables['_PublishArtifacts'], ''), ne(variables['_ArtifactsTargetPath'], '')) + + - task: CopyFiles@2 + displayName: Gather binaries for publish to artifacts + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/$(_PublishArtifacts)' + Contents: '**' + TargetFolder: $(Build.ArtifactStagingDirectory)/artifacts/$(_PublishArtifacts) + condition: and(ne(variables['_PublishArtifacts'], ''), eq(variables['_ArtifactsTargetPath'], '')) + + - task: PublishBuildArtifacts@1 + displayName: Publish Build Artifacts + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: Build_$(_BuildConfig) + condition: ne(variables['_PublishArtifacts'], '') - task: PublishBuildArtifacts@1 displayName: Publish Artifacts on failure @@ -230,12 +256,12 @@ jobs: continueOnError: true condition: always() - - ${{ if and(eq(parameters.buildAndSkipTest, 'false'), eq(parameters.isCodeQLRun, 'false')) }}: + - ${{ if and(eq(parameters.buildOnly, 'false'), eq(parameters.isCodeQLRun, 'false')) }}: # Publish test results to Azure Pipelines - task: PublishTestResults@2 inputs: testResultsFormat: xUnit - testResultsFiles: '**/*UnitTests*.xml' + testResultsFiles: '**/*UnitTests*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults' failTaskOnFailedTests: true testRunTitle: 'Tests $(_PhaseName) $(_BuildArch) $(_BuildConfig)' @@ -243,4 +269,4 @@ jobs: mergeTestResults: true buildConfiguration: ${{ parameters.name }} continueOnError: true - condition: ne(variables['_BuildOnly'], 'true') + condition: always() diff --git a/eng/pipelines/pipeline-resources.yml b/eng/pipelines/pipeline-resources.yml new file mode 100644 index 0000000000..0b3a5b51ed --- /dev/null +++ b/eng/pipelines/pipeline-resources.yml @@ -0,0 +1,62 @@ +parameters: + - name: stages + type: stageList + +resources: + containers: + - container: linux_x64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7 + + - container: linux_arm + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross + env: + ROOTFS_DIR: /crossrootfs/arm + + - container: linux_arm64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64 + env: + ROOTFS_DIR: /crossrootfs/arm64 + + - container: linux_musl_x64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.13-WithNode + + - container: linux_musl_arm + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm-alpine + env: + ROOTFS_DIR: /crossrootfs/arm + + - container: linux_musl_arm64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-alpine + env: + ROOTFS_DIR: /crossrootfs/arm64 + + - container: test_linux_x64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7 + + - container: test_linux_musl_x64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.13-WithNode + options: --cap-add=SYS_PTRACE + + - container: test_debian_11_amd64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 + options: '--env PYTHONPATH=/usr/bin/python3.9' + + - container: test_fedora_36 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-36 + options: --cap-add=SYS_PTRACE + + - container: test_opensuse_15_2 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:opensuse-15.2-helix-amd64 + + - container: test_ubuntu_18_04 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04 + + - container: test_ubuntu_20_04 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04 + options: '--env PYTHONPATH=/usr/bin/python3.8' + + - container: test_ubuntu_22_04 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 + options: '--env PYTHONPATH=/usr/bin/python3.10' + +stages: ${{ parameters.stages }} diff --git a/eng/prepare-release.yml b/eng/pipelines/prepare-release.yml similarity index 100% rename from eng/prepare-release.yml rename to eng/pipelines/prepare-release.yml diff --git a/global.json b/global.json index 879572b3ab..d3ebd457b7 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "tools": { - "dotnet": "8.0.100-preview.1.23115.2", + "dotnet": "8.0.100-preview.4.23260.5", "runtimes": { "dotnet": [ "$(MicrosoftNETCoreApp60Version)", @@ -16,6 +16,6 @@ }, "msbuild-sdks": { "Microsoft.Build.NoTargets": "3.5.0", - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23168.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23302.3" } } diff --git a/src/Microsoft.Diagnostics.DebugServices.Implementation/DataReader.cs b/src/Microsoft.Diagnostics.DebugServices.Implementation/DataReader.cs index b32038d540..ba1ecc6d21 100644 --- a/src/Microsoft.Diagnostics.DebugServices.Implementation/DataReader.cs +++ b/src/Microsoft.Diagnostics.DebugServices.Implementation/DataReader.cs @@ -55,10 +55,10 @@ bool IDataReader.GetThreadContext(uint threadId, uint contextFlags, Span c try { byte[] registerContext = ThreadService.GetThreadFromId(threadId).GetThreadContext(); - context = new Span(registerContext); + registerContext.AsSpan().Slice(0, context.Length).CopyTo(context); return true; } - catch (DiagnosticsException ex) + catch (Exception ex) when (ex is DiagnosticsException or ArgumentException) { Trace.TraceError($"GetThreadContext: {threadId} exception {ex.Message}"); } @@ -83,7 +83,7 @@ int IMemoryReader.Read(ulong address, Span buffer) bool IMemoryReader.Read(ulong address, out T value) { - Span buffer = stackalloc byte[Marshal.SizeOf()]; + Span buffer = stackalloc byte[Unsafe.SizeOf()]; if (((IMemoryReader)this).Read(address, buffer) == buffer.Length) { value = Unsafe.As(ref MemoryMarshal.GetReference(buffer)); diff --git a/src/Microsoft.Diagnostics.DebugServices.Implementation/Runtime.cs b/src/Microsoft.Diagnostics.DebugServices.Implementation/Runtime.cs index 40b237f884..771e0d14c2 100644 --- a/src/Microsoft.Diagnostics.DebugServices.Implementation/Runtime.cs +++ b/src/Microsoft.Diagnostics.DebugServices.Implementation/Runtime.cs @@ -20,7 +20,6 @@ namespace Microsoft.Diagnostics.DebugServices.Implementation public class Runtime : IRuntime, IDisposable { private readonly ClrInfo _clrInfo; - private readonly IDisposable _onFlushEvent; private readonly ISymbolService _symbolService; private Version _runtimeVersion; private string _dacFilePath; @@ -52,24 +51,19 @@ public Runtime(IServiceProvider services, int id, ClrInfo clrInfo) _serviceContainer.AddService(this); _serviceContainer.AddService(clrInfo); - _onFlushEvent = Target.OnFlushEvent.Register(Flush); - Trace.TraceInformation($"Created runtime #{id} {clrInfo.Flavor} {clrInfo}"); } void IDisposable.Dispose() - { - _serviceContainer.RemoveService(typeof(IRuntime)); - _serviceContainer.DisposeServices(); - _onFlushEvent.Dispose(); - } - - private void Flush() { if (_serviceContainer.TryGetCachedService(typeof(ClrRuntime), out object service)) { - ((ClrRuntime)service).FlushCachedData(); + // The DataTarget created in the RuntimeProvider is disposed here. The ClrRuntime + // instance is disposed below in DisposeServices(). + ((ClrRuntime)service).DataTarget.Dispose(); } + _serviceContainer.RemoveService(typeof(IRuntime)); + _serviceContainer.DisposeServices(); } #region IRuntime diff --git a/src/Microsoft.Diagnostics.DebugServices.Implementation/RuntimeProvider.cs b/src/Microsoft.Diagnostics.DebugServices.Implementation/RuntimeProvider.cs index a755a13d6d..3ea0ab90be 100644 --- a/src/Microsoft.Diagnostics.DebugServices.Implementation/RuntimeProvider.cs +++ b/src/Microsoft.Diagnostics.DebugServices.Implementation/RuntimeProvider.cs @@ -28,6 +28,9 @@ public RuntimeProvider(IServiceProvider services) /// The starting runtime id for this provider public IEnumerable EnumerateRuntimes(int startingRuntimeId) { + // The ClrInfo and DataTarget instances are disposed when Runtime instance is disposed. Runtime instances are + // not flushed when the Target/RuntimeService is flushed; they are all disposed and the list cleared. They are + // all re-created the next time the IRuntime or ClrRuntime instance is queried. DataTarget dataTarget = new(new CustomDataTarget(_services.GetService())) { FileLocator = null diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/ClrMDHelper.cs b/src/Microsoft.Diagnostics.ExtensionCommands/ClrMDHelper.cs index 074026ac39..76cda04435 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/ClrMDHelper.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/ClrMDHelper.cs @@ -851,37 +851,51 @@ private static bool TryGetSegmentMemoryRange(ClrSegment segment, GCGeneration ge switch (generation) { case GCGeneration.Generation0: - start = segment.Generation0.Start; - end = segment.Generation0.End; - return start != end; + if (segment.Kind == GCSegmentKind.Generation0 || segment.Kind == GCSegmentKind.Ephemeral) + { + start = segment.Generation0.Start; + end = segment.Generation0.End; + } + break; case GCGeneration.Generation1: - start = segment.Generation1.Start; - end = segment.Generation1.End; - return start != end; + if (segment.Kind == GCSegmentKind.Generation1 || segment.Kind == GCSegmentKind.Ephemeral) + { + start = segment.Generation1.Start; + end = segment.Generation1.End; + } + break; case GCGeneration.Generation2: - if (segment.Kind != GCSegmentKind.Large && segment.Kind != GCSegmentKind.Large && segment.Kind != GCSegmentKind.Frozen) + if (segment.Kind == GCSegmentKind.Generation2 || segment.Kind == GCSegmentKind.Ephemeral) { start = segment.Generation2.Start; end = segment.Generation2.End; } - return start != end; + break; case GCGeneration.LargeObjectHeap: if (segment.Kind == GCSegmentKind.Large) { start = segment.Start; end = segment.End; } - return start != end; + break; case GCGeneration.PinnedObjectHeap: - if (segment.Kind == GCSegmentKind.Pinned || segment.Kind == GCSegmentKind.Frozen) + if (segment.Kind == GCSegmentKind.Pinned) { start = segment.Start; end = segment.End; } - return start != end; + break; + case GCGeneration.FrozenObjectHeap: + if (segment.Kind == GCSegmentKind.Frozen) + { + start = segment.Start; + end = segment.End; + } + break; default: return false; } + return start != end; } public IEnumerable EnumerateConcurrentQueue(ulong address) diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/DumpAsyncCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/DumpAsyncCommand.cs index 7e80e59d18..0acbca362e 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/DumpAsyncCommand.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/DumpAsyncCommand.cs @@ -52,13 +52,24 @@ public sealed class DumpAsyncCommand : ExtensionCommandBase [ServiceImport(Optional = true)] public ClrRuntime? Runtime { get; set; } + /// Gets whether to only show stacks that include the object with the specified address. [Option(Name = "--address", Aliases = new string[] { "-addr" }, Help = "Only show stacks that include the object with the specified address.")] - public ulong? ObjectAddress { get; set; } + public string? ObjectAddress + { + get => _objectAddress?.ToString(); + set => _objectAddress = ParseAddress(value); + } + private ulong? _objectAddress; /// Gets whether to only show stacks that include objects with the specified method table. [Option(Name = "--methodtable", Aliases = new string[] { "-mt" }, Help = "Only show stacks that include objects with the specified method table.")] - public ulong? MethodTableAddress { get; set; } + public string? MethodTableAddress + { + get => _methodTableAddress?.ToString(); + set => _methodTableAddress = ParseAddress(value); + } + private ulong? _methodTableAddress; /// Gets whether to only show stacks that include objects whose type includes the specified name in its name. [Option(Name = "--type", Help = "Only show stacks that include objects whose type includes the specified name in its name.")] @@ -532,14 +543,14 @@ string Describe(ClrObject obj) // Determines whether the specified object is of interest to the user based on their criteria provided as command arguments. bool IncludeInOutput(ClrObject obj) { - if (ObjectAddress is ulong addr && obj.Address != addr) + if (_objectAddress is ulong addr && obj.Address != addr) { return false; } if (obj.Type is not null) { - if (MethodTableAddress is ulong mt && obj.Type.MethodTable != mt) + if (_methodTableAddress is ulong mt && obj.Type.MethodTable != mt) { return false; } diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/DumpGenCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/DumpGenCommand.cs index ef1908ef3b..db020e7926 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/DumpGenCommand.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/DumpGenCommand.cs @@ -96,22 +96,21 @@ private GCGeneration ParseGenerationArgument(string generation) return GCGeneration.NotSet; } string lowerString = generation.ToLowerInvariant(); - switch (lowerString) + GCGeneration result = lowerString switch { - case "gen0": - return GCGeneration.Generation0; - case "gen1": - return GCGeneration.Generation1; - case "gen2": - return GCGeneration.Generation2; - case "loh": - return GCGeneration.LargeObjectHeap; - case "poh": - return GCGeneration.PinnedObjectHeap; - default: - WriteLine($"{generation} is not a supported generation (gen0, gen1, gen2, loh, poh)"); - return GCGeneration.NotSet; + "gen0" => GCGeneration.Generation0, + "gen1" => GCGeneration.Generation1, + "gen2" => GCGeneration.Generation2, + "loh" => GCGeneration.LargeObjectHeap, + "poh" => GCGeneration.PinnedObjectHeap, + "foh" => GCGeneration.FrozenObjectHeap, + _ => GCGeneration.NotSet, + }; + if (result == GCGeneration.NotSet) + { + WriteLine($"{generation} is not a supported generation (gen0, gen1, gen2, loh, poh, foh)"); } + return result; } @@ -133,6 +132,7 @@ Generation number can take the following values (case insensitive): - gen2 - loh - poh +- foh > dumpgen gen0 Statistics: diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/DumpHeapCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/DumpHeapCommand.cs index c62435f1d3..2f8c47256b 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/DumpHeapCommand.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/DumpHeapCommand.cs @@ -72,6 +72,14 @@ public override void Invoke() ParseArguments(); IEnumerable objectsToPrint = FilteredHeap.EnumerateFilteredObjects(Console.CancellationToken); + + bool? liveObjectWarning = null; + if ((Live || Dead) && Short) + { + liveObjectWarning = LiveObjects.PrintWarning; + LiveObjects.PrintWarning = false; + } + if (Live) { objectsToPrint = objectsToPrint.Where(LiveObjects.IsLive); @@ -148,6 +156,11 @@ public override void Invoke() } DumpHeap.PrintHeap(objectsToPrint, displayKind, StatOnly, printFragmentation); + + if (liveObjectWarning is bool original) + { + LiveObjects.PrintWarning = original; + } } private void ParseArguments() diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/DumpHeapService.cs b/src/Microsoft.Diagnostics.ExtensionCommands/DumpHeapService.cs index f9687f8f33..d4587697d6 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/DumpHeapService.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/DumpHeapService.cs @@ -6,8 +6,8 @@ using System.Diagnostics; using System.Linq; using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.ExtensionCommands.Output; using Microsoft.Diagnostics.Runtime; -using static Microsoft.Diagnostics.ExtensionCommands.TableOutput; namespace Microsoft.Diagnostics.ExtensionCommands { @@ -41,16 +41,14 @@ public void PrintHeap(IEnumerable objects, DisplayKind displayKind, b Dictionary<(string String, ulong Size), uint> stringTable = null; Dictionary stats = new(); - TableOutput thinLockOutput = null; - TableOutput objectTable = new(Console, (12, "x12"), (12, "x12"), (12, ""), (0, "")); - if (!statsOnly && (displayKind is DisplayKind.Normal or DisplayKind.Strings)) - { - objectTable.WriteRow("Address", "MT", "Size"); - } + Table thinLockOutput = null; + Table objectTable = null; ClrObject lastFreeObject = default; foreach (ClrObject obj in objects) { + Console.CancellationToken.ThrowIfCancellationRequested(); + if (displayKind == DisplayKind.ThinLock) { ClrThinLock thinLock = obj.GetThinLock(); @@ -58,11 +56,11 @@ public void PrintHeap(IEnumerable objects, DisplayKind displayKind, b { if (thinLockOutput is null) { - thinLockOutput = new(Console, (12, "x"), (16, "x"), (16, "x"), (10, "n0")); - thinLockOutput.WriteRow("Object", "Thread", "OSId", "Recursion"); + thinLockOutput = new(Console, ColumnKind.DumpObj, ColumnKind.Pointer, ColumnKind.HexValue, ColumnKind.Integer); + thinLockOutput.WriteHeader("Object", "Thread", "OSId", "Recursion"); } - thinLockOutput.WriteRow(new DmlDumpObj(obj), thinLock.Thread?.Address ?? 0, thinLock.Thread?.OSThreadId ?? 0, thinLock.Recursion); + thinLockOutput.WriteRow(obj, thinLock.Thread, thinLock.Thread?.OSThreadId ?? 0, thinLock.Recursion); } continue; @@ -77,7 +75,16 @@ public void PrintHeap(IEnumerable objects, DisplayKind displayKind, b ulong size = obj.IsValid ? obj.Size : 0; if (!statsOnly) { - objectTable.WriteRow(new DmlDumpObj(obj), new DmlDumpHeap(obj.Type?.MethodTable ?? 0), size, obj.IsFree ? "Free" : ""); + if (objectTable is null) + { + objectTable = new(Console, ColumnKind.DumpObj, ColumnKind.DumpHeap, ColumnKind.ByteCount, ColumnKind.Text); + if (displayKind is DisplayKind.Normal or DisplayKind.Strings) + { + objectTable.WriteHeader("Address", "MT", "Size"); + } + } + + objectTable.WriteRow(obj, obj.Type, obj.IsValid ? size : null, obj.IsFree ? "Free" : ""); } if (printFragmentation) @@ -170,7 +177,7 @@ public void PrintHeap(IEnumerable objects, DisplayKind displayKind, b } Console.WriteLine("Statistics:"); - TableOutput statsTable = new(Console, (countLen, "n0"), (sizeLen, "n0"), (0, "")); + Table statsTable = new(Console, ColumnKind.Integer, ColumnKind.ByteCount, ColumnKind.Text); var stringsSorted = from item in stringTable let Count = item.Value @@ -186,12 +193,15 @@ orderby TotalSize foreach (var item in stringsSorted) { + Console.CancellationToken.ThrowIfCancellationRequested(); + statsTable.WriteRow(item.Count, item.TotalSize, item.String); } } } else if (displayKind == DisplayKind.Normal) { + // Print statistics table if (stats.Count != 0) { // Print statistics table @@ -200,16 +210,17 @@ orderby TotalSize Console.WriteLine(); } - int countLen = stats.Values.Max(ts => ts.Count).ToString("n0").Length; - countLen = Math.Max(countLen, "Count".Length); + Console.WriteLine("Statistics:"); - int sizeLen = stats.Values.Max(ts => ts.Size).ToString("n0").Length; - sizeLen = Math.Max(sizeLen, "TotalSize".Length); + Column countColumn = ColumnKind.Integer; + countColumn = countColumn.GetAppropriateWidth(stats.Values.Select(ts => ts.Count)); - TableOutput statsTable = new(Console, (12, "x12"), (countLen, "n0"), (sizeLen, "n0"), (0, "")); + Column sizeColumn = ColumnKind.ByteCount; + sizeColumn = sizeColumn.GetAppropriateWidth(stats.Values.Select(ts => ts.Size)); - Console.WriteLine("Statistics:"); - statsTable.WriteRow("MT", "Count", "TotalSize", "Class Name"); + Column methodTableColumn = ColumnKind.DumpHeap.GetAppropriateWidth(stats.Keys); + Table statsTable = new(Console, methodTableColumn, countColumn, sizeColumn, ColumnKind.TypeName); + statsTable.WriteHeader("MT", "Count", "TotalSize", "Class Name"); var statsSorted = from item in stats let MethodTable = item.Key @@ -224,7 +235,9 @@ orderby Size foreach (var item in statsSorted) { - statsTable.WriteRow(new DmlDumpHeap(item.MethodTable), item.Count, item.Size, item.TypeName); + Console.CancellationToken.ThrowIfCancellationRequested(); + + statsTable.WriteRow(item.MethodTable, item.Count, item.Size, item.TypeName); } Console.WriteLine($"Total {stats.Values.Sum(r => r.Count):n0} objects, {stats.Values.Sum(r => (long)r.Size):n0} bytes"); @@ -242,15 +255,17 @@ private void PrintFragmentation(List<(ClrObject Free, ClrObject Next)> fragmenta return; } - TableOutput output = new(Console, (16, "x12"), (12, "n0"), (16, "x12")); - Console.WriteLine(); Console.WriteLine("Fragmented blocks larger than 0.5 MB:"); - output.WriteRow("Address", "Size", "Followed By"); + + Table output = new(Console, ColumnKind.ListNearObj, ColumnKind.ByteCount, ColumnKind.DumpObj, ColumnKind.TypeName); + output.WriteHeader("Address", "Size", "Followed By"); foreach ((ClrObject free, ClrObject next) in fragmentation) { - output.WriteRow(free.Address, free.Size, new DmlDumpObj(next.Address), next.Type?.Name ?? ""); + Console.CancellationToken.ThrowIfCancellationRequested(); + + output.WriteRow(free.Address, free.Size, next.Address, next.Type); } } diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/DumpObjGCRefsHelper.cs b/src/Microsoft.Diagnostics.ExtensionCommands/DumpObjGCRefsHelper.cs new file mode 100644 index 0000000000..adae249ab2 --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/DumpObjGCRefsHelper.cs @@ -0,0 +1,100 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Linq; +using System.Text; +using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.ExtensionCommands.Output; +using Microsoft.Diagnostics.Runtime; + +namespace Microsoft.Diagnostics.ExtensionCommands +{ + [Command(Name = "dumpobjgcrefs", Help = "A helper command to implement !dumpobj -refs")] + public sealed class DumpObjGCRefsHelper : CommandBase + { + private readonly StringBuilderPool _stringBuilderPool = new(260); + + [ServiceImport] + public ClrRuntime Runtime { get; set; } + + [Argument(Name = "object")] + public string ObjectAddress { get; set; } + + public override void Invoke() + { + if (!TryParseAddress(ObjectAddress, out ulong objAddress)) + { + throw new ArgumentException($"Invalid object address: '{ObjectAddress}'", nameof(ObjectAddress)); + } + + ClrObject obj = Runtime.Heap.GetObject(objAddress); + if (!obj.IsValid) + { + Console.WriteLine($"Unable to walk object references, invalid object."); + return; + } + + ClrReference[] refs = obj.EnumerateReferencesWithFields(carefully: false, considerDependantHandles: false).ToArray(); + if (refs.Length == 0) + { + Console.WriteLine("GC Refs: none"); + return; + } + + Console.WriteLine("GC Refs:"); + + Column fieldNameColumn = ColumnKind.Text.GetAppropriateWidth(refs.Select(r => GetFieldName(r))); + Column offsetName = ColumnKind.HexOffset.GetAppropriateWidth(refs.Select(r => r.Offset)); + + Table output = new(Console, fieldNameColumn, offsetName, ColumnKind.DumpObj, ColumnKind.TypeName); + output.WriteHeader("Field", "Offset", "Object", "Type"); + foreach (ClrReference objRef in refs) + { + output.WriteRow(GetFieldName(objRef), objRef.Offset, objRef.Object, objRef.Object.Type); + } + } + + private string GetFieldName(ClrReference objRef) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + if (objRef.Field is null) + { + return null; + } + + if (objRef.InnerField is null) + { + return objRef.Field?.Name; + } + + StringBuilder sb = _stringBuilderPool.Rent(); + bool foundOneFieldName = false; + + for (ClrReference? curr = objRef; curr.HasValue; curr = curr.Value.InnerField) + { + if (sb.Length > 0) + { + sb.Append('.'); + } + + string fieldName = curr.Value.Field?.Name; + if (string.IsNullOrWhiteSpace(fieldName)) + { + sb.Append("???"); + } + else + { + sb.Append(fieldName); + foundOneFieldName = true; + } + } + + // Make sure we don't just return "???.???.???" + string result = foundOneFieldName ? sb.ToString() : null; + _stringBuilderPool.Return(sb); + return result; + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/DumpRuntimeTypeCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/DumpRuntimeTypeCommand.cs new file mode 100644 index 0000000000..cfc8579e6f --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/DumpRuntimeTypeCommand.cs @@ -0,0 +1,70 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.ExtensionCommands.Output; +using Microsoft.Diagnostics.Runtime; +using static Microsoft.Diagnostics.ExtensionCommands.Output.ColumnKind; + +namespace Microsoft.Diagnostics.ExtensionCommands +{ + [Command(Name = "dumpruntimetypes", Help = "Finds all System.RuntimeType objects in the GC heap and prints the type name and MethodTable they refer too.")] + public sealed class DumpRuntimeTypeCommand : CommandBase + { + [ServiceImport] + public ClrRuntime Runtime { get; set; } + + public override void Invoke() + { + Table output = null; + + foreach (ClrObject runtimeType in Runtime.Heap.EnumerateObjects()) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + if (!runtimeType.IsValid || !runtimeType.IsRuntimeType) + { + continue; + } + + if (!runtimeType.TryReadField("m_handle", out nuint m_handle)) + { + continue; + } + + ClrAppDomain domain = null; + object typeName = m_handle; + bool isMethodTable = (m_handle & 2) == 0; + if (isMethodTable) + { + // Only lookup the type if we have a MethodTable. + ClrType type = Runtime.GetTypeByMethodTable(m_handle); + if (type is not null) + { + typeName = type; + domain = type.Module?.AppDomain; + } + } + else + { + typeName = $"typehandle: {m_handle:x} (SOS does not support resolving typehandle names.)"; + } + + if (output is null) + { + output = new(Console, DumpObj, DumpDomain, DumpHeap, TypeName); + output.WriteHeader("Address", "Domain", "MT", "Type Name"); + } + + // We pass .Address here instead of the ClrObject because every type is a RuntimeType, we don't need + // or want the alt-text. + output.WriteRow(runtimeType.Address, domain, m_handle, typeName); + } + + if (output is null) + { + Console.WriteLine("No System.RuntimeType objects found."); + } + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/DumpStackObjectsCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/DumpStackObjectsCommand.cs new file mode 100644 index 0000000000..131450fe8b --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/DumpStackObjectsCommand.cs @@ -0,0 +1,383 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.Linq; +using System.Runtime.CompilerServices; +using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.ExtensionCommands.Output; +using Microsoft.Diagnostics.Runtime; +using static Microsoft.Diagnostics.ExtensionCommands.Output.ColumnKind; + +namespace Microsoft.Diagnostics.ExtensionCommands +{ + [Command(Name = "dumpstackobjects", Aliases = new string[] { "dso" }, Help = "Displays all managed objects found within the bounds of the current stack.")] + public class DumpStackObjectsCommand : CommandBase + { + [ServiceImport] + public IMemoryService MemoryService { get; set; } + + [ServiceImport] + public IThread CurrentThread { get; set; } + + [ServiceImport] + public IThreadService ThreadService { get; set; } + + [ServiceImport] + public ClrRuntime Runtime { get; set; } + + [Option(Name = "-verify", Help = "Verify each object and only print ones that are valid objects.")] + public bool Verify { get; set; } + + [Argument(Name = "StackBounds", Help = "The top and bottom of the stack (in hex).")] + public string[] Bounds { get; set; } + + public override void Invoke() + { + if (Runtime.Heap.Segments.Length == 0) + { + throw new DiagnosticsException("Cannot walk heap."); + } + + MemoryRange range; + if (Bounds is null || Bounds.Length == 0) + { + range = GetStackRange(); + } + else if (Bounds.Length == 2) + { + ulong start = ParseAddress(Bounds[0]) ?? throw new ArgumentException($"Failed to parse start address '{Bounds[0]}'."); + ulong end = ParseAddress(Bounds[1]) ?? throw new ArgumentException($"Failed to parse end address '{Bounds[1]}'."); + if (start > end) + { + (start, end) = (end, start); + } + + range = new(AlignDown(start), AlignUp(end)); + } + else + { + throw new ArgumentException("Invalid arguments."); + } + + if (range.Start == 0 || range.End == 0) + { + throw new ArgumentException($"Invalid range {range.Start:x} - {range.End:x}"); + } + + PrintStackObjects(range); + } + + private void PrintStackObjects(MemoryRange stack) + { + Console.WriteLine($"OS Thread Id: 0x{CurrentThread.ThreadId:x} ({CurrentThread.ThreadIndex})"); + + Table output = new(Console, Pointer, DumpObj, TypeName); + output.WriteHeader("SP/REG", "Object", "Name"); + + int regCount = ThreadService.Registers.Count(); + foreach ((ulong address, ClrObject obj) in EnumerateValidObjectsWithinRange(stack).OrderBy(r => r.StackAddress)) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + if (address < (ulong)regCount) + { + string registerName; + if (ThreadService.TryGetRegisterInfo((int)address, out RegisterInfo regInfo)) + { + registerName = regInfo.RegisterName; + } + else + { + registerName = $"reg{address}"; + } + + output.WriteRow(registerName, obj, obj.Type); + } + else + { + output.WriteRow(address, obj, obj.Type); + } + } + } + + /// + /// Enumerates all valid objects (and the address they came from) within the given range. + /// + private IEnumerable<(ulong StackAddress, ClrObject Object)> EnumerateValidObjectsWithinRange(MemoryRange range) + { + // Note: This implementation is careful to enumerate only real objects and not generate a lot of native + // exceptions within the dac. A naïve implementation could simply read every pointer aligned address + // and call ClrHeap.GetObject(objAddr).IsValid. That approach will generate a lot of exceptions + // within the dac trying to validate wild pointers as MethodTables, and it will often find old + // pointers which the GC has already swept but not zeroed yet. + + // Sort the list of potential objects so that we can go through each in segment order. + // Sorting this array saves us a lot of time by not searching for segments. + IEnumerable<(ulong StackAddress, ulong PotentialObject)> potentialObjects = EnumeratePointersWithinHeapBounds(range); + potentialObjects = potentialObjects.Concat(EnumerateRegistersWithinHeapBounds()); + potentialObjects = potentialObjects.OrderBy(r => r.PotentialObject); + + ClrSegment currSegment = null; + List<(ulong StackAddress, ulong PotentialObject)> withinCurrSegment = new(64); + int segmentIndex = 0; + foreach ((ulong _, ulong PotentialObject) entry in potentialObjects) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + // Find the segment of the current potential object, or null if it doesn't live + // within a segment. + ClrSegment segment = GetSegment(entry.PotentialObject, ref segmentIndex); + if (segment is null) + { + continue; + } + + // If we are already processing this segment, just add the entry to the list + if (currSegment == segment) + { + withinCurrSegment.Add(entry); + continue; + } + + // We are finished walking objects from "currSegment". If we found any pointers + // within its range, walk the segment and return every valid object. + if (withinCurrSegment.Count > 0) + { + foreach ((ulong StackAddress, ClrObject Object) validObject in EnumerateObjectsOnSegment(withinCurrSegment, currSegment)) + { + yield return validObject; + } + + withinCurrSegment.Clear(); + } + + // Update currSegment and add this entry to the processing list. + currSegment = segment; + withinCurrSegment.Add(entry); + } + + // Process leftover items + if (withinCurrSegment.Count > 0) + { + foreach ((ulong StackAddress, ClrObject Object) validObject in EnumerateObjectsOnSegment(withinCurrSegment, currSegment)) + { + yield return validObject; + } + } + } + + /// + /// Simultaneously walks the withinCurrSegment list and objects on segment returning valid objects found. + /// + private IEnumerable<(ulong StackAddress, ClrObject Object)> EnumerateObjectsOnSegment(List<(ulong StackAddress, ulong PotentialObject)> withinCurrSegment, ClrSegment segment) + { + if (withinCurrSegment.Count == 0) + { + yield break; + } + + int index = 0; + MemoryRange range = new(withinCurrSegment[0].PotentialObject, withinCurrSegment[withinCurrSegment.Count - 1].PotentialObject + 1); + foreach (ClrObject obj in segment.EnumerateObjects(range, carefully: true)) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + if (index >= withinCurrSegment.Count) + { + yield break; + } + + while (index < withinCurrSegment.Count && withinCurrSegment[index].PotentialObject < obj) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + index++; + } + + while (index < withinCurrSegment.Count && obj == withinCurrSegment[index].PotentialObject) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + if (Verify) + { + if (!Runtime.Heap.IsObjectCorrupted(obj, out _)) + { + yield return (withinCurrSegment[index].StackAddress, obj); + } + } + else + { + yield return (withinCurrSegment[index].StackAddress, obj); + } + + index++; + } + } + } + + private ClrSegment GetSegment(ulong potentialObject, ref int segmentIndex) + { + ImmutableArray segments = Runtime.Heap.Segments; + + // This function assumes that segmentIndex is always within the bounds of segments + // and that all objects passed to it are within the given + // range of segment bounds. + Debug.Assert(segmentIndex >= 0 && segmentIndex <= segments.Length); + Debug.Assert(segments[0].ObjectRange.Start <= potentialObject); + Debug.Assert(potentialObject < segments[segments.Length - 1].ObjectRange.End); + + for (; segmentIndex < segments.Length; segmentIndex++) + { + ClrSegment curr = segments[segmentIndex]; + if (potentialObject < curr.Start) + { + return null; + } + else if (potentialObject < curr.ObjectRange.End) + { + return segments[segmentIndex]; + } + } + + // Unreachable. + Debug.Fail("Reached the end of the segment array."); + return null; + } + + private IEnumerable<(ulong RegisterIndex, ulong PotentialObject)> EnumerateRegistersWithinHeapBounds() + { + ClrHeap heap = Runtime.Heap; + + // Segments are always sorted by address + ulong minAddress = heap.Segments[0].ObjectRange.Start; + ulong maxAddress = heap.Segments[heap.Segments.Length - 1].ObjectRange.End - (uint)MemoryService.PointerSize; + + int regCount = ThreadService.Registers.Count(); + for (int i = 0; i < regCount; i++) + { + if (CurrentThread.TryGetRegisterValue(i, out ulong value)) + { + if (minAddress <= value && value < maxAddress) + { + yield return ((ulong)i, value); + } + } + } + } + + private IEnumerable<(ulong StackAddress, ulong PotentialObject)> EnumeratePointersWithinHeapBounds(MemoryRange stack) + { + Debug.Assert(AlignDown(stack.Start) == stack.Start); + Debug.Assert(AlignUp(stack.End) == stack.End); + + uint pointerSize = (uint)MemoryService.PointerSize; + ClrHeap heap = Runtime.Heap; + + // Segments are always sorted by address + ulong minAddress = heap.Segments[0].ObjectRange.Start; + ulong maxAddress = heap.Segments[heap.Segments.Length - 1].ObjectRange.End - pointerSize; + + // Read in 64k chunks + byte[] buffer = ArrayPool.Shared.Rent(64 * 1024); + try + { + ulong address = stack.Start; + while (stack.Contains(address)) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + if (!MemoryService.ReadMemory(address, buffer, out int read)) + { + break; + } + + read = AlignDown(read); + if (read < pointerSize) + { + break; + } + + for (int i = 0; i < read; i += (int)pointerSize) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + ulong stackAddress = address + (uint)i; + if (!stack.Contains(stackAddress)) + { + yield break; + } + + ulong potentialObj = GetIndex(buffer, i); + if (minAddress <= potentialObj && potentialObj < maxAddress) + { + yield return (stackAddress, potentialObj); + } + } + + address += (uint)read; + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + private static ulong GetIndex(Span buffer, int i) => Unsafe.As(ref buffer[i]); + + private MemoryRange GetStackRange() + { + ulong end = 0; + + int spIndex = ThreadService.StackPointerIndex; + if (!CurrentThread.TryGetRegisterValue(spIndex, out ulong stackPointer)) + { + throw new DiagnosticsException($"Unable to get the stack pointer for thread {CurrentThread.ThreadId:x}."); + } + + // On Windows we have the TEB to know where to end the walk. + ulong teb = CurrentThread.GetThreadTeb(); + if (teb != 0) + { + // The stack base is after the first pointer, see TEB and NT_TIB. + MemoryService.ReadPointer(teb + (uint)MemoryService.PointerSize, out end); + } + + if (end == 0) + { + end = stackPointer + 0xFFFF; + } + + return new(AlignDown(stackPointer), AlignUp(end)); + } + + private ulong AlignDown(ulong address) + { + ulong mask = ~((ulong)MemoryService.PointerSize - 1); + return address & mask; + } + + private int AlignDown(int value) + { + int mask = ~(MemoryService.PointerSize - 1); + return value & mask; + } + + private ulong AlignUp(ulong address) + { + ulong pointerSize = (ulong)MemoryService.PointerSize; + if (address > ulong.MaxValue - pointerSize) + { + return AlignDown(address); + } + + ulong mask = ~(pointerSize - 1); + return (address + pointerSize - 1) & mask; + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/EEHeapCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/EEHeapCommand.cs index 90afe5dc43..ac2f9507f6 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/EEHeapCommand.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/EEHeapCommand.cs @@ -7,8 +7,9 @@ using System.Linq; using System.Text; using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.ExtensionCommands.Output; using Microsoft.Diagnostics.Runtime; -using static Microsoft.Diagnostics.ExtensionCommands.TableOutput; +using static Microsoft.Diagnostics.ExtensionCommands.Output.ColumnKind; namespace Microsoft.Diagnostics.ExtensionCommands { @@ -72,10 +73,8 @@ public override void Invoke() private ulong PrintOneRuntime(ClrRuntime clrRuntime) { StringBuilder stringBuilder = null; - TableOutput output = new(Console, (21, "x12"), (0, "x12")) - { - AlignLeft = true - }; + Table output = new(Console, Text.WithWidth(21), Pointer.WithWidth(-1)); + output.SetAlignment(Align.Left); HashSet seen = new(); @@ -105,7 +104,7 @@ private ulong PrintOneRuntime(ClrRuntime clrRuntime) return totalSize; } - private ulong PrintAppDomains(TableOutput output, ClrRuntime clrRuntime, HashSet loaderAllocatorsSeen) + private ulong PrintAppDomains(Table output, ClrRuntime clrRuntime, HashSet loaderAllocatorsSeen) { Console.WriteLine("Loader Heap:"); WriteDivider(); @@ -117,6 +116,8 @@ private ulong PrintAppDomains(TableOutput output, ClrRuntime clrRuntime, HashSet for (int i = 0; i < clrRuntime.AppDomains.Length; i++) { + Console.CancellationToken.ThrowIfCancellationRequested(); + ClrAppDomain appDomain = clrRuntime.AppDomains[i]; totalBytes += PrintAppDomain(output, appDomain, $"Domain {i + 1}:", loaderAllocatorsSeen); } @@ -124,7 +125,7 @@ private ulong PrintAppDomains(TableOutput output, ClrRuntime clrRuntime, HashSet return totalBytes; } - private ulong PrintAppDomain(TableOutput output, ClrAppDomain appDomain, string name, HashSet loaderAllocatorsSeen) + private ulong PrintAppDomain(Table output, ClrAppDomain appDomain, string name, HashSet loaderAllocatorsSeen) { if (appDomain is null) { @@ -151,7 +152,7 @@ private ulong PrintAppDomain(TableOutput output, ClrAppDomain appDomain, string IOrderedEnumerable> filteredHeapsByKind = from heap in appDomain.EnumerateLoaderAllocatorHeaps() where IsIncludedInFilter(heap) - where loaderAllocatorsSeen.Add(heap.Address) + where loaderAllocatorsSeen.Add(heap.MemoryRange.Start) group heap by heap.Kind into g orderby GetSortOrder(g.Key) select g; @@ -182,7 +183,7 @@ private static int GetSortOrder(NativeHeapKind key) }; } - private ulong PrintAppDomainHeapsByKind(TableOutput output, IOrderedEnumerable> filteredHeapsByKind) + private ulong PrintAppDomainHeapsByKind(Table output, IOrderedEnumerable> filteredHeapsByKind) { // Just build and print the table. ulong totalSize = 0; @@ -191,6 +192,8 @@ private ulong PrintAppDomainHeapsByKind(TableOutput output, IOrderedEnumerable item in filteredHeapsByKind) { + Console.CancellationToken.ThrowIfCancellationRequested(); + text.Clear(); NativeHeapKind kind = item.Key; ulong heapSize = 0; @@ -236,15 +239,17 @@ private ulong PrintAppDomainHeapsByKind(TableOutput output, IOrderedEnumerable heaps = jitManager.EnumerateNativeHeaps().Where(IsIncludedInFilter).OrderBy(r => r.Kind).ThenBy(r => r.Address); + IEnumerable heaps = jitManager.EnumerateNativeHeaps().Where(IsIncludedInFilter).OrderBy(r => r.Kind).ThenBy(r => r.MemoryRange.Start); ulong jitMgrSize = 0, jitMgrWasted = 0; foreach (ClrNativeHeapInfo heap in heaps) @@ -284,15 +289,15 @@ private bool IsIncludedInFilter(ClrNativeHeapInfo info) return true; } - if (filterRange.Contains(info.Address)) + if (filterRange.Contains(info.MemoryRange.Start)) { return true; } - if (info.Size is ulong size && size > 0) + if (info.MemoryRange.Length > 0) { // Check for the last valid address in the range - return filterRange.Contains(info.Address + size - 1); + return filterRange.Contains(info.MemoryRange.End - 1); } return false; @@ -300,19 +305,20 @@ private bool IsIncludedInFilter(ClrNativeHeapInfo info) private (ulong Size, ulong Wasted) CalculateSizeAndWasted(StringBuilder sb, ClrNativeHeapInfo heap) { - sb.Append(heap.Address.ToString("x12")); + sb.Append(heap.MemoryRange.Start.ToString("x12")); - if (heap.Size is ulong size) + ulong size = heap.MemoryRange.Length; + if (size > 0) { sb.Append('('); sb.Append(size.ToString("x")); sb.Append(':'); - ulong actualSize = GetActualSize(heap.Address, size); + ulong actualSize = GetActualSize(heap.MemoryRange.Start, size); sb.Append(actualSize.ToString("x")); sb.Append(')'); ulong wasted = 0; - if (actualSize < size && !heap.IsCurrentBlock) + if (actualSize < size && heap.State != ClrNativeHeapState.Active) { wasted = size - actualSize; } @@ -323,7 +329,7 @@ private bool IsIncludedInFilter(ClrNativeHeapInfo info) return (0, 0); } - private ulong PrintModuleThunkTable(TableOutput output, ref StringBuilder text, ClrRuntime clrRuntime) + private ulong PrintModuleThunkTable(Table output, ref StringBuilder text, ClrRuntime clrRuntime) { IEnumerable modulesWithThunks = clrRuntime.EnumerateModules().Where(r => r.ThunkHeap != 0); if (!modulesWithThunks.Any()) @@ -337,7 +343,7 @@ private ulong PrintModuleThunkTable(TableOutput output, ref StringBuilder text, return PrintModules(output, ref text, modulesWithThunks); } - private ulong PrintModuleLoaderAllocators(TableOutput output, ref StringBuilder text, ClrRuntime clrRuntime, HashSet loaderAllocatorsSeen) + private ulong PrintModuleLoaderAllocators(Table output, ref StringBuilder text, ClrRuntime clrRuntime, HashSet loaderAllocatorsSeen) { // On .Net Core, modules share their LoaderAllocator with their AppDomain (and AppDomain shares theirs // with SystemDomain). Only collectable assemblies have unique loader allocators, and that's what we @@ -358,17 +364,21 @@ private ulong PrintModuleLoaderAllocators(TableOutput output, ref StringBuilder return PrintModules(output, ref text, collectable); } - private ulong PrintModules(TableOutput output, ref StringBuilder text, IEnumerable modules) + private ulong PrintModules(Table output, ref StringBuilder text, IEnumerable modules) { text ??= new(128); ulong totalSize = 0, totalWasted = 0; foreach (ClrModule module in modules) { + Console.CancellationToken.ThrowIfCancellationRequested(); + ulong moduleSize = 0, moduleWasted = 0; text.Clear(); foreach (ClrNativeHeapInfo info in module.EnumerateThunkHeap().Where(IsIncludedInFilter)) { + Console.CancellationToken.ThrowIfCancellationRequested(); + if (text.Length > 0) { text.Append(' '); @@ -438,14 +448,8 @@ private ulong PrintGCHeap(ClrRuntime clrRuntime) Console.WriteLine(); ClrHeap heap = clrRuntime.Heap; - int pointerWidth = 16; - string pointerToStringFormat = "x16"; - (int pointerWidth, string pointerToStringFormat) pointerFormat = (pointerWidth, pointerToStringFormat); - - int sizeWidth = Math.Max(15, heap.Segments.Max(seg => FormatMemorySize(seg.CommittedMemory.Length).Length)); - (int sizeWidth, string) sizeFormat = (sizeWidth, ""); - - TableOutput gcOutput = new(Console, pointerFormat, pointerFormat, pointerFormat, pointerFormat, sizeFormat, sizeFormat); + Column sizeColumn = Text.GetAppropriateWidth(heap.Segments.Select(seg => FormatMemorySize(seg.CommittedMemory.Length)), max: 32); + Table gcOutput = new(Console, DumpHeap, Pointer, Pointer, Pointer, sizeColumn, sizeColumn); WriteDivider('='); Console.WriteLine($"Number of GC Heaps: {heap.SubHeaps.Length}"); @@ -453,6 +457,8 @@ private ulong PrintGCHeap(ClrRuntime clrRuntime) foreach (ClrSubHeap gc_heap in HeapWithFilters.EnumerateFilteredSubHeaps()) { + Console.CancellationToken.ThrowIfCancellationRequested(); + if (heap.IsServer) { Console.Write("Heap "); @@ -595,15 +601,14 @@ private ulong PrintGCHeap(ClrRuntime clrRuntime) return totalCommitted; } - private static void WriteSegmentHeader(TableOutput gcOutput) + private static void WriteSegmentHeader(Table gcOutput) { - gcOutput.WriteRow("segment", "begin", "allocated", "committed", "allocated size", "committed size"); + gcOutput.WriteHeader("segment", "begin", "allocated", "committed", "allocated size", "committed size"); } - private static void WriteSegment(TableOutput gcOutput, ClrSegment segment) + private static void WriteSegment(Table gcOutput, ClrSegment segment) { - gcOutput.WriteRow(new DmlDumpHeapSegment(segment), - segment.ObjectRange.Start, segment.ObjectRange.End, segment.CommittedMemory.End, + gcOutput.WriteRow(segment, segment.ObjectRange.Start, segment.ObjectRange.End, segment.CommittedMemory.End, FormatMemorySize(segment.ObjectRange.Length), FormatMemorySize(segment.CommittedMemory.Length)); } diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/ExtensionMethodHelpers.cs b/src/Microsoft.Diagnostics.ExtensionCommands/ExtensionMethodHelpers.cs index 5f35b39381..348a23eb84 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/ExtensionMethodHelpers.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/ExtensionMethodHelpers.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Collections.Generic; using System.Linq; using Microsoft.Diagnostics.Runtime; @@ -32,6 +33,8 @@ public static string ConvertToHumanReadable(this double totalBytes) return $"{updated:0.00}gb"; } + public static string ToSignedHexString(this int offset) => offset < 0 ? $"-{Math.Abs(offset):x2}" : offset.ToString("x2"); + internal static ulong FindMostCommonPointer(this IEnumerable enumerable) => (from ptr in enumerable group ptr by ptr into g diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/FinalizeQueueCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/FinalizeQueueCommand.cs new file mode 100644 index 0000000000..9d4efb2ede --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/FinalizeQueueCommand.cs @@ -0,0 +1,226 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.ExtensionCommands.Output; +using Microsoft.Diagnostics.Runtime; +using static Microsoft.Diagnostics.ExtensionCommands.Output.ColumnKind; + +namespace Microsoft.Diagnostics.ExtensionCommands +{ + [Command(Name = "finalizequeue", Help = "Displays all objects registered for finalization.")] + public class FinalizeQueueCommand : CommandBase + { + [Option(Name = "-detail", Help = "Will display extra information on any SyncBlocks that need to be cleaned up, and on any RuntimeCallableWrappers (RCWs) that await cleanup. Both of these data structures are cached and cleaned up by the finalizer thread when it gets a chance to run.")] + public bool Detail { get; set; } + + [Option(Name = "-allReady", Help = "Specifying this argument will allow for the display of all objects that are ready for finalization, whether they are already marked by the GC as such, or whether the next GC will. The objects that are not in the \"Ready for finalization\" list are finalizable objects that are no longer rooted. This option can be very expensive, as it verifies whether all the objects in the finalizable queues are still rooted or not.")] + public bool AllReady { get; set; } + + [Option(Name = "-short", Help = "Limits the output to just the address of each object. If used in conjunction with -allReady it enumerates all objects that have a finalizer that are no longer rooted. If used independently it lists all objects in the finalizable and \"ready for finalization\" queues.")] + public bool Short { get; set; } + + [Option(Name = "-mt", Help = "Limits the search for finalizable objects to only those matching the given MethodTable.")] + public string MethodTable { get; set; } + + [Option(Name = "-stat", Aliases = new string[] { "-summary" }, Help = "Only print object statistics, not the list of all objects.")] + public bool Stat { get; set; } + + [ServiceImport] + public LiveObjectService LiveObjects { get; set; } + + [ServiceImport] + public RootCacheService RootCache { get; set; } + + [ServiceImport] + public DumpHeapService DumpHeap { get; set; } + + [ServiceImport] + public ClrRuntime Runtime { get; set; } + + public override void Invoke() + { + ulong mt = 0; + if (!string.IsNullOrWhiteSpace(MethodTable)) + { + mt = ParseAddress(MethodTable) ?? throw new ArgumentException($"Could not parse MethodTable: '{MethodTable}'"); + } + + if (Short && Stat) + { + throw new ArgumentException("Cannot specify both -short and -stat."); + } + + // If we are going to search for only live objects, be sure to print a warning first + // in the output of the command instead of in between the rest of the output. + if (AllReady) + { + LiveObjects.PrintWarning = true; + LiveObjects.Initialize(); + } + + if (!Short) + { + PrintSyncBlockCleanupData(); + PrintRcwCleanupData(); + Console.WriteLine("----------------------------------"); + Console.WriteLine(); + + PrintGenerationalRanges(); + + if (AllReady) + { + Console.WriteLine("Statistics for all finalizable objects that are no longer rooted:"); + } + else + { + Console.WriteLine("Statistics for all finalizable objects (including all objects ready for finalization):"); + } + } + + IEnumerable objects = EnumerateFinalizableObjects(AllReady, mt); + DumpHeapService.DisplayKind displayKind = Short ? DumpHeapService.DisplayKind.Short : DumpHeapService.DisplayKind.Normal; + + DumpHeap.PrintHeap(objects, displayKind, Stat, printFragmentation: false); + + } + private IEnumerable EnumerateFinalizableObjects(bool allReady, ulong mt) + { + IEnumerable result = EnumerateValidFinalizableObjectsWithTypeFilter(mt); + + if (allReady) + { + HashSet rootedByFinalizer = new(); + foreach (ClrRoot root in Runtime.Heap.EnumerateFinalizerRoots()) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + ClrObject obj = root.Object; + if (obj.IsValid) + { + rootedByFinalizer.Add(obj); + } + } + + // We are trying to find all objects that are ready to be finalized, which is essentially + // all dead objects. However, objects which were previously collected but waiting on + // the finalizer thread to process them are considered "live" because they are rooted by + // the finalizer queue. So our result needs to be either dead objects or directly rooted + // by the finalizer queue. + result = result.Where(obj => rootedByFinalizer.Contains(obj) || !LiveObjects.IsLive(obj)); + } + + return result; + } + + private IEnumerable EnumerateValidFinalizableObjectsWithTypeFilter(ulong mt) + { + foreach (ClrObject obj in Runtime.Heap.EnumerateFinalizableObjects()) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + if (!obj.IsValid) + { + continue; + } + + if (mt != 0 && obj.Type.MethodTable != mt) + { + continue; + } + + yield return obj; + } + } + + private void PrintSyncBlockCleanupData() + { + Table output = null; + int total = 0; + foreach (ClrSyncBlockCleanupData cleanup in Runtime.EnumerateSyncBlockCleanupData()) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + if (output is null) + { + output = new(Console, Pointer, Pointer, Pointer, Pointer); + output.WriteHeader("SyncBlock", "RCW", "CCW", "ComClassFactory"); + } + + output.WriteRow(cleanup.SyncBlock, cleanup.Rcw, cleanup.Ccw, cleanup.ClassFactory); + total++; + } + + Console.WriteLine($"SyncBlocks to be cleaned up: {total:n0}"); + } + + private void PrintRcwCleanupData() + { + Table output = null; + int freeThreadedCount = 0; + int mtaCount = 0; + int staCount = 0; + + foreach (ClrRcwCleanupData cleanup in Runtime.EnumerateRcwCleanupData()) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + if (output is null) + { + output = new(Console, Pointer, Pointer, Thread, Text); + output.WriteHeader("RCW", "Context", "Thread", "Apartment"); + } + + string apartment; + if (cleanup.IsFreeThreaded) + { + freeThreadedCount++; + apartment = "(FreeThreaded)"; + } + else if (cleanup.Thread == 0) + { + mtaCount++; + apartment = "(MTA)"; + } + else + { + staCount++; + apartment = "(STA)"; + } + + output.WriteRow(cleanup.Rcw, cleanup.Context, cleanup.Thread, apartment); + } + + Console.WriteLine($"Free-Threaded Interfaces to be released: {freeThreadedCount:n0}"); + Console.WriteLine($"MTA Interfaces to be released: {mtaCount:n0}"); + Console.WriteLine($"STA Interfaces to be released: {staCount:n0}"); + } + + private void PrintGenerationalRanges() + { + foreach (ClrSubHeap heap in Runtime.Heap.SubHeaps) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + Console.WriteLine($"Heap {heap.Index}"); + + WriteGeneration(heap, 0); + WriteGeneration(heap, 1); + WriteGeneration(heap, 2); + + Console.WriteLine($"Ready for finalization {heap.FinalizerQueueRoots.Length / (uint)IntPtr.Size:n0} objects ({heap.FinalizerQueueRoots.Start:x}->{heap.FinalizerQueueRoots.End:x})"); + + Console.WriteLine("------------------------------"); + } + } + + private void WriteGeneration(ClrSubHeap heap, int gen) + { + MemoryRange range = heap.GenerationalFinalizableObjects[gen]; + Console.WriteLine($"generation {gen} has {range.Length / (uint)IntPtr.Size:n0} objects ({range.Start:x}->{range.End:x})"); + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/FindEphemeralReferencesToLOHCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/FindEphemeralReferencesToLOHCommand.cs index c7fa399b83..e1715396da 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/FindEphemeralReferencesToLOHCommand.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/FindEphemeralReferencesToLOHCommand.cs @@ -5,8 +5,9 @@ using System.Diagnostics; using System.Linq; using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.ExtensionCommands.Output; using Microsoft.Diagnostics.Runtime; -using static Microsoft.Diagnostics.ExtensionCommands.TableOutput; +using static Microsoft.Diagnostics.ExtensionCommands.Output.ColumnKind; namespace Microsoft.Diagnostics.ExtensionCommands { @@ -28,7 +29,7 @@ public override void Invoke() Console.WriteLineWarning($"Walking {segments:n0} {gcSegKind}, this may take a moment..."); } - TableOutput output = new(Console, (16, "x12"), (64, ""), (16, "x12")); + Table output = new(Console, DumpObj, TypeName.WithWidth(64), DumpObj, TypeName); // Ephemeral -> Large List<(ClrObject From, ClrObject To)> ephToLoh = FindEphemeralToLOH().OrderBy(i => i.From.Address).ThenBy(i => i.To.Address).ToList(); @@ -40,11 +41,11 @@ public override void Invoke() { Console.WriteLine("Ephemeral objects pointing to the Large objects:"); Console.WriteLine(); - output.WriteRow("Ephemeral", "Ephemeral Type", "Large Object", "Large Object Type"); + output.WriteHeader("Ephemeral", "Ephemeral Type", "Large Object", "Large Object Type"); foreach ((ClrObject from, ClrObject to) in ephToLoh) { - output.WriteRow(new DmlDumpObj(from), from.Type?.Name, new DmlDumpObj(to), to.Type?.Name); + output.WriteRow(from, from.Type, to, to.Type); } Console.WriteLine(); @@ -60,11 +61,11 @@ public override void Invoke() { Console.WriteLine("Large objects pointing to Ephemeral objects:"); Console.WriteLine(); - output.WriteRow("Ephemeral", "Ephemeral Type", "Large Object", "Large Object Type"); + output.WriteHeader("Ephemeral", "Ephemeral Type", "Large Object", "Large Object Type"); foreach ((ClrObject from, ClrObject to) in lohToEph) { - output.WriteRow(new DmlDumpObj(from), from.Type?.Name, new DmlDumpObj(to), to.Type?.Name); + output.WriteRow(from, from.Type, to, to.Type); } Console.WriteLine(); @@ -90,8 +91,8 @@ public override void Invoke() { Console.WriteLine($"Ephemeral objects which point to Large objects which point to Ephemeral objects:"); Console.WriteLine(); - output = new(Console, (16, "x12"), (64, ""), (16, "x12"), (64, ""), (16, "x12")); - output.WriteRow(new DmlDumpObj(from), from.Type?.Name, new DmlDumpObj(to), to.Type?.Name, new DmlDumpObj(ephEnd), ephEnd.Type?.Name); + output = new(Console, DumpObj, TypeName.WithWidth(64), DumpObj, TypeName.WithWidth(64), DumpObj, TypeName); + output.WriteRow(from, from.Type, to, to.Type, ephEnd, ephEnd.Type); } } @@ -104,14 +105,6 @@ public override void Invoke() Console.WriteLine(); } } - - foreach ((ClrObject From, ClrObject To) item in ephToLoh) - { - if (lohToEph.Any(r => item.To.Address == r.From.Address)) - { - Console.WriteLine("error!"); - } - } } private IEnumerable<(ClrObject From, ClrObject To)> FindEphemeralToLOH() diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/FindPointersInCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/FindPointersInCommand.cs index 66de79e8e7..752bb338c3 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/FindPointersInCommand.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/FindPointersInCommand.cs @@ -6,9 +6,10 @@ using System.IO; using System.Linq; using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.ExtensionCommands.Output; using Microsoft.Diagnostics.Runtime; -using Microsoft.Diagnostics.Runtime.Interfaces; using static Microsoft.Diagnostics.ExtensionCommands.NativeAddressHelper; +using static Microsoft.Diagnostics.ExtensionCommands.Output.ColumnKind; namespace Microsoft.Diagnostics.ExtensionCommands { @@ -239,9 +240,10 @@ private void PrintPointerTable(string nameColumn, string truncatedName, bool for int nameLen = Math.Min(80, maxNameLen); nameLen = Math.Max(nameLen, truncatedName.Length); - TableOutput table = new(Console, (nameLen, ""), (12, "n0"), (12, "n0"), (12, "x")); - table.Divider = " "; - table.WriteRowWithSpacing('-', nameColumn, "Unique", "Count", "RndPtr"); + using BorderedTable table = new(Console, TypeName.WithWidth(nameLen), Integer, Integer, Pointer); + table.Columns[0] = table.Columns[0].WithAlignment(Align.Center); + table.WriteHeader(nameColumn, "Unique", "Count", "RndPtr"); + table.Columns[0] = table.Columns[0].WithAlignment(Align.Left); IEnumerable<(string Name, int Count, int Unique, IEnumerable Pointers)> items = truncate ? resolved.Take(multi) : resolved; foreach ((string Name, int Count, int Unique, IEnumerable Pointers) in items) @@ -254,7 +256,8 @@ private void PrintPointerTable(string nameColumn, string truncatedName, bool for table.WriteRow(truncatedName, single, single); } - table.WriteRowWithSpacing('-', " [ TOTALS ] ", resolved.Sum(r => r.Unique), resolved.Sum(r => r.Count), ""); + table.Columns[0] = table.Columns[0].WithAlignment(Align.Center); + table.WriteFooter("TOTALS", resolved.Sum(r => r.Unique), resolved.Sum(r => r.Count)); } private static string FixTypeName(string typeName, HashSet offsets) diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/FindReferencesToEphemeralCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/FindReferencesToEphemeralCommand.cs index 8c5f3c0eac..6614438d78 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/FindReferencesToEphemeralCommand.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/FindReferencesToEphemeralCommand.cs @@ -4,8 +4,9 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.ExtensionCommands.Output; using Microsoft.Diagnostics.Runtime; -using static Microsoft.Diagnostics.ExtensionCommands.TableOutput; +using static Microsoft.Diagnostics.ExtensionCommands.Output.ColumnKind; namespace Microsoft.Diagnostics.ExtensionCommands { @@ -20,7 +21,7 @@ public class FindReferencesToEphemeralCommand : CommandBase public override void Invoke() { - TableOutput output = new(Console, (16, "x12"), (16, "x12"), (10, "n0"), (8, ""), (8, ""), (12, "n0"), (12, "n0")); + Table output = new(Console, DumpObj, DumpHeap, ByteCount, Column.ForEnum(), Column.ForEnum(), ByteCount, Integer, TypeName); var generationGroup = from item in FindObjectsWithEphemeralReferences() group item by (item.ObjectGeneration, item.ReferenceGeneration) into g @@ -54,7 +55,7 @@ group item by (item.ObjectGeneration, item.ReferenceGeneration) into g Console.WriteLine($"References from {objGen} to {refGen}:"); Console.WriteLine(); - output.WriteRow("Object", "MethodTable", "Size", "Obj Gen", "Ref Gen", "Obj Count", "Obj Size", "Type"); + output.WriteHeader("Object", "MethodTable", "Size", "Obj Gen", "Ref Gen", "Obj Count", "Obj Size", "Type"); } foreach (EphemeralRefCount erc in item.Objects) @@ -62,7 +63,7 @@ group item by (item.ObjectGeneration, item.ReferenceGeneration) into g Console.CancellationToken.ThrowIfCancellationRequested(); objCount++; - output.WriteRow(new DmlDumpObj(erc.Object), erc.Object.Type.MethodTable, erc.Object.Size, erc.ObjectGeneration, erc.ReferenceGeneration, erc.Count, erc.Size, erc.Object.Type.Name); + output.WriteRow(erc.Object, erc.Object.Type, erc.Object.Size, erc.ObjectGeneration, erc.ReferenceGeneration, erc.Count, erc.Size, erc.Object.Type); } } diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/GCGeneration.cs b/src/Microsoft.Diagnostics.ExtensionCommands/GCGeneration.cs index 5ebe713150..028e7c413c 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/GCGeneration.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/GCGeneration.cs @@ -10,6 +10,7 @@ public enum GCGeneration Generation1 = 2, Generation2 = 3, LargeObjectHeap = 4, - PinnedObjectHeap = 5 + PinnedObjectHeap = 5, + FrozenObjectHeap = 6 } } diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/GCHeapStatCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/GCHeapStatCommand.cs index a7c24db50a..7a3da01995 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/GCHeapStatCommand.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/GCHeapStatCommand.cs @@ -5,7 +5,9 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.ExtensionCommands.Output; using Microsoft.Diagnostics.Runtime; +using static Microsoft.Diagnostics.ExtensionCommands.Output.ColumnKind; namespace Microsoft.Diagnostics.ExtensionCommands { @@ -23,25 +25,29 @@ public class GCHeapStatCommand : CommandBase public override void Invoke() { - HeapInfo[] heaps = Runtime.Heap.SubHeaps.Select(h => GetHeapInfo(h)).ToArray(); bool printFrozen = heaps.Any(h => h.Frozen.Committed != 0); - - List<(int, string)> formats = new() + List formats = new() { - (8, "x"), (12, ""), (12, ""), (12, ""), (12, ""), (12, ""), (8, ""), (8, ""), (8, "") + Text.WithWidth(8), + IntegerWithoutCommas, + IntegerWithoutCommas, + IntegerWithoutCommas, + IntegerWithoutCommas, + IntegerWithoutCommas, + Text.WithWidth(8), + Text.WithWidth(8), + Text.WithWidth(8) }; if (printFrozen) { - formats.Insert(1, (12, "")); + formats.Insert(1, IntegerWithoutCommas); } - TableOutput output = new(Console, formats.ToArray()) - { - AlignLeft = true, - }; + Table output = new(Console, formats.ToArray()); + output.SetAlignment(Align.Left); // Write allocated WriteHeader(output, heaps, printFrozen); @@ -91,13 +97,13 @@ public override void Invoke() } total = GetTotal(heaps); - WriteRow(output, total, (info) => info.Committed, printFrozen); + WriteRow(output, total, (info) => info.Committed, printFrozen, printPercentage: false, footer: true); Console.WriteLine(); } - private static void WriteHeader(TableOutput output, HeapInfo[] heaps, bool printFrozen) + private static void WriteHeader(Table output, HeapInfo[] heaps, bool printFrozen) { - List row = new(8) { "Heap", "Gen0", "Gen1", "Gen2", "LOH", "POH" }; + List row = new(8) { "Heap", "Gen0", "Gen1", "Gen2", "LOH", "POH" }; if (printFrozen) { @@ -110,10 +116,10 @@ private static void WriteHeader(TableOutput output, HeapInfo[] heaps, bool print row.Insert(1, "EPH"); } - output.WriteRow(row.ToArray()); + output.WriteHeader(row.ToArray()); } - private static void WriteRow(TableOutput output, HeapInfo heapInfo, Func select, bool printFrozen, bool printPercentage = false) + private static void WriteRow(Table output, HeapInfo heapInfo, Func select, bool printFrozen, bool printPercentage = false, bool footer = false) { List row = new(11) { @@ -171,7 +177,14 @@ private static void WriteRow(TableOutput output, HeapInfo heapInfo, Func {gen} = condemned generation."); + return; + } + if (gen < 0 || gen > 1) { // If not gen0 or gen1, treat it as a normal !gcroot @@ -127,7 +146,7 @@ private int PrintOlderGenerationRoots(GCRoot gcroot, int gen) } Console.WriteLine($" {objAddress:x}"); - PrintPath(Console, RootCache, Runtime.Heap, path); + PrintPath(Console, RootCache, StaticVariables, Runtime.Heap, path); Console.WriteLine(); count++; @@ -200,25 +219,73 @@ private int PrintNonStackRoots(GCRoot gcroot) private void PrintPath(ClrRoot root, GCRoot.ChainLink link) { PrintRoot(root); - PrintPath(Console, RootCache, Runtime.Heap, link); + PrintPath(Console, RootCache, StaticVariables, Runtime.Heap, link); Console.WriteLine(); } - public static void PrintPath(IConsoleService console, RootCacheService rootCache, ClrHeap heap, GCRoot.ChainLink link) + public static void PrintPath(IConsoleService console, RootCacheService rootCache, StaticVariableService statics, ClrHeap heap, GCRoot.ChainLink link) { - TableOutput objectOutput = new(console, (2, ""), (16, "x16")) + Table objectOutput = new(console, Text.WithWidth(2), DumpObj, TypeName, Text) { - AlignLeft = true, Indent = new(' ', 10) }; + objectOutput.SetAlignment(Align.Left); + + bool first = true; + bool isPossibleStatic = true; + + ClrObject firstObj = default; + ulong prevObj = 0; while (link != null) { - bool isDependentHandleLink = rootCache.IsDependentHandleLink(prevObj, link.Object); ClrObject obj = heap.GetObject(link.Object); - objectOutput.WriteRow("->", obj.IsValid ? new DmlDumpObj(obj) : obj.Address, obj.Type?.Name ?? "", (isDependentHandleLink ? " (dependent handle)" : "")); + // Check whether this link is a dependent handle + string extraText = ""; + bool isDependentHandleLink = rootCache.IsDependentHandleLink(prevObj, link.Object); + if (isDependentHandleLink) + { + extraText = "(dependent handle)"; + } + + // Print static variable info. In all versions of the runtime, static variables are stored in + // a pinned object array. We check if the first link in the chain is an object[], and if so we + // check if the second object's address is the location of a static variable. We could further + // narrow this by checking the root type, but that needlessly complicates this code...we can't + // get false positives or negatives here (as nothing points to static variable object[] other + // than the root). + if (first) + { + firstObj = obj; + isPossibleStatic = firstObj.IsValid && firstObj.IsArray && firstObj.Type.Name == "System.Object[]"; + first = false; + } + else if (isPossibleStatic) + { + if (statics is not null && !isDependentHandleLink) + { + foreach (ClrReference reference in firstObj.EnumerateReferencesWithFields(carefully: false, considerDependantHandles: false)) + { + if (reference.Object == obj) + { + ulong address = firstObj + (uint)reference.Offset; + + if (statics.TryGetStaticByAddress(address, out ClrStaticField field)) + { + extraText = $"(static variable: {field.Type?.Name ?? "Unknown"}.{field.Name})"; + break; + } + } + } + } + + // only the first object[] in the chain is possible to be the static array + isPossibleStatic = false; + } + + objectOutput.WriteRow("->", obj, obj.Type, extraText); prevObj = link.Object; link = link.Next; @@ -305,7 +372,7 @@ private static string NameForHandle(ClrHandleKind handleKind) ClrHandleKind.SizedRef => "sized ref handle", ClrHandleKind.WeakWinRT => "weak WinRT handle", _ => handleKind.ToString() - }; ; + }; } private string GetFrameOutput(ClrStackFrame currFrame) diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/GCToNativeCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/GCToNativeCommand.cs index 1016dd685a..e5157d4d31 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/GCToNativeCommand.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/GCToNativeCommand.cs @@ -7,8 +7,10 @@ using System.Linq; using System.Text; using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.ExtensionCommands.Output; using Microsoft.Diagnostics.Runtime; using static Microsoft.Diagnostics.ExtensionCommands.NativeAddressHelper; +using static Microsoft.Diagnostics.ExtensionCommands.Output.ColumnKind; namespace Microsoft.Diagnostics.ExtensionCommands { @@ -99,6 +101,8 @@ public void PrintGCPointersToMemory(bool showAll, params string[] memoryTypes) foreach ((ClrSegment Segment, ulong Address, ulong Pointer, DescribedRegion MemoryRange) item in items) { + Console.CancellationToken.ThrowIfCancellationRequested(); + if (!segmentLists.TryGetValue(item.Segment, out List list)) { list = segmentLists[item.Segment] = new(); @@ -110,6 +114,8 @@ public void PrintGCPointersToMemory(bool showAll, params string[] memoryTypes) Console.WriteLine("Resolving object names..."); foreach (string type in memoryTypes) { + Console.CancellationToken.ThrowIfCancellationRequested(); + WriteHeader($" {type} Regions "); List addressesNotInObjects = new(); @@ -119,6 +125,8 @@ public void PrintGCPointersToMemory(bool showAll, params string[] memoryTypes) foreach (KeyValuePair> segEntry in segmentLists) { + Console.CancellationToken.ThrowIfCancellationRequested(); + ClrSegment seg = segEntry.Key; List pointers = segEntry.Value; pointers.Sort((x, y) => x.GCPointer.CompareTo(y.GCPointer)); @@ -133,6 +141,8 @@ public void PrintGCPointersToMemory(bool showAll, params string[] memoryTypes) while (index < pointers.Count && pointers[index].GCPointer < obj.Address) { + Console.CancellationToken.ThrowIfCancellationRequested(); + // If we "missed" the pointer then it's outside of an object range. addressesNotInObjects.Add(pointers[index].GCPointer); @@ -148,6 +158,8 @@ public void PrintGCPointersToMemory(bool showAll, params string[] memoryTypes) while (index < pointers.Count && obj.Address <= pointers[index].GCPointer && pointers[index].GCPointer < obj.Address + obj.Size) { + Console.CancellationToken.ThrowIfCancellationRequested(); + string typeName = obj.Type?.Name ?? $""; if (obj.IsFree) @@ -195,20 +207,20 @@ public void PrintGCPointersToMemory(bool showAll, params string[] memoryTypes) { Console.WriteLine($"All memory pointers:"); - IEnumerable<(ulong Pointer, ulong Size, ulong Object, string Type)> allPointers = unknownObjPointers.Select(unknown => (unknown.Pointer, 0ul, unknown.Object.Address, unknown.Object.Type?.Name ?? "")); - allPointers = allPointers.Concat(knownMemory.Values.Select(k => (k.Pointer, GetSize(sizeHints, k), k.Object.Address, k.Name))); + IEnumerable<(ulong Pointer, ulong Size, ClrObject Object, ClrType Type)> allPointers = unknownObjPointers.Select(unknown => (unknown.Pointer, 0ul, unknown.Object, unknown.Object.Type)); + allPointers = allPointers.Concat(knownMemory.Values.Select(k => (k.Pointer, GetSize(sizeHints, k), k.Object, k.Object.Type))); - TableOutput allOut = new(Console, (16, "x"), (16, "x"), (16, "x")) - { - Divider = " | " - }; + using BorderedTable allOut = new(Console, Pointer, ByteCount, DumpObj, TypeName); + + allOut.WriteHeader("Pointer", "Size", "Object", "Type"); - allOut.WriteRowWithSpacing('-', "Pointer", "Size", "Object", "Type"); - foreach ((ulong Pointer, ulong Size, ulong Object, string Type) entry in allPointers) + foreach ((ulong Pointer, ulong Size, ClrObject Object, ClrType Type) entry in allPointers) { + Console.CancellationToken.ThrowIfCancellationRequested(); + if (entry.Size == 0) { - allOut.WriteRow(entry.Pointer, "", entry.Object, entry.Type); + allOut.WriteRow(entry.Pointer, null, entry.Object, entry.Type); } else { @@ -225,38 +237,35 @@ public void PrintGCPointersToMemory(bool showAll, params string[] memoryTypes) // totals var knownMemorySummary = from known in knownMemory.Values - group known by known.Name into g - let Name = g.Key + group known by known.Object.Type into g + let Type = g.Key let Count = g.Count() let TotalSize = g.Sum(k => (long)GetSize(sizeHints, k)) - orderby TotalSize descending, Name ascending + orderby TotalSize descending, Type.Name ascending select new { - Name, + Type, Count, TotalSize, Pointer = g.Select(p => p.Pointer).FindMostCommonPointer() }; - int maxNameLen = Math.Min(80, knownMemory.Values.Max(r => r.Name.Length)); - - TableOutput summary = new(Console, (-maxNameLen, ""), (8, "n0"), (12, "n0"), (12, "n0"), (12, "x")) + Column typeNameColumn = TypeName.GetAppropriateWidth(knownMemory.Values.Select(r => r.Object.Type), 16); + using (BorderedTable summary = new(Console, typeNameColumn, Integer, HumanReadableSize, ByteCount, Pointer)) { - Divider = " | " - }; - - summary.WriteRowWithSpacing('-', "Type", "Count", "Size", "Size (bytes)", "RndPointer"); + summary.WriteHeader("Type", "Count", "Size", "Size (bytes)", "RndPointer"); - foreach (var item in knownMemorySummary) - { - summary.WriteRow(item.Name, item.Count, item.TotalSize.ConvertToHumanReadable(), item.TotalSize, item.Pointer); - } + foreach (var item in knownMemorySummary) + { + Console.CancellationToken.ThrowIfCancellationRequested(); - (int totalRegions, ulong totalBytes) = GetSizes(knownMemory, sizeHints); + summary.WriteRow(item.Type, item.Count, item.TotalSize, item.TotalSize, item.Pointer); + } - summary.WriteSpacer('-'); - summary.WriteRow("[TOTAL]", totalRegions, totalBytes.ConvertToHumanReadable(), totalBytes); + (int totalRegions, ulong totalBytes) = GetSizes(knownMemory, sizeHints); + summary.WriteFooter("[TOTAL]", totalRegions, totalBytes, totalBytes); + } - Console.WriteLine(""); + Console.WriteLine(); } @@ -277,17 +286,15 @@ orderby Count descending }; var unknownMem = unknownMemQuery.ToArray(); - int maxNameLen = Math.Min(80, unknownMem.Max(r => r.Name.Length)); - - TableOutput summary = new(Console, (-maxNameLen, ""), (8, "n0"), (12, "x")) - { - Divider = " | " - }; - summary.WriteRowWithSpacing('-', "Type", "Count", "RndPointer"); + Column typeNameColumn = TypeName.GetAppropriateWidth(unknownMem.Select(r => r.Name)); + using BorderedTable summary = new(Console, typeNameColumn, Integer, Pointer); + summary.WriteHeader("Type", "Count", "RndPointer"); foreach (var item in unknownMem) { + Console.CancellationToken.ThrowIfCancellationRequested(); + summary.WriteRow(item.Name, item.Count, item.Pointer); } } @@ -349,12 +356,14 @@ private void WriteHeader(string header) Console.WriteLine(header.PadRight(Width, '=')); } - private static string CollapseGenerics(string typeName) + private string CollapseGenerics(string typeName) { StringBuilder result = new(typeName.Length + 16); int nest = 0; for (int i = 0; i < typeName.Length; i++) { + Console.CancellationToken.ThrowIfCancellationRequested(); + if (typeName[i] == '<') { if (nest++ == 0) @@ -419,7 +428,6 @@ private sealed class KnownClrMemoryPointer private const string ExternalMemoryBlock = "System.Reflection.Internal.ExternalMemoryBlock"; private const string RuntimeParameterInfo = "System.Reflection.RuntimeParameterInfo"; - public string Name => Object.Type?.Name ?? ""; public ClrObject Object { get; } public ulong Pointer { get; } public ulong Size { get; } diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/GCWhereCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/GCWhereCommand.cs index fad8d35df1..96a61f3783 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/GCWhereCommand.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/GCWhereCommand.cs @@ -5,8 +5,9 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.ExtensionCommands.Output; using Microsoft.Diagnostics.Runtime; -using static Microsoft.Diagnostics.ExtensionCommands.TableOutput; +using static Microsoft.Diagnostics.ExtensionCommands.Output.ColumnKind; namespace Microsoft.Diagnostics.ExtensionCommands { @@ -40,13 +41,13 @@ public override void Invoke() return; } - (int, string) RangeFormat = (segments.Max(seg => RangeSizeForSegment(seg)), ""); - TableOutput output = new(Console, (16, "x"), (4, ""), (16, "x"), (10, ""), RangeFormat, RangeFormat, RangeFormat) - { - AlignLeft = true, - }; + Column objectRangeColumn = Range.WithDml(Dml.DumpHeap).GetAppropriateWidth(segments.Select(r => r.ObjectRange)); + Column committedColumn = Range.GetAppropriateWidth(segments.Select(r => r.CommittedMemory)); + Column reservedColumn = Range.GetAppropriateWidth(segments.Select(r => r.ReservedMemory)); + Table output = new(Console, Pointer, IntegerWithoutCommas.WithWidth(6).WithDml(Dml.DumpHeap), DumpHeap, Text.WithWidth(6), objectRangeColumn, committedColumn, reservedColumn); + output.SetAlignment(Align.Left); + output.WriteHeader("Address", "Heap", "Segment", "Generation", "Allocated", "Committed", "Reserved"); - output.WriteRow("Address", "Heap", "Segment", "Generation", "Allocated", "Committed", "Reserved"); foreach (ClrSegment segment in segments) { string generation; @@ -68,23 +69,16 @@ public override void Invoke() }; } - object addressColumn = segment.ObjectRange.Contains(address) ? new DmlListNearObj(address) : address; - output.WriteRow(addressColumn, segment.SubHeap.Index, segment.Address, generation, new DmlDumpHeap(FormatRange(segment.ObjectRange), segment.ObjectRange), FormatRange(segment.CommittedMemory), FormatRange(segment.ReservedMemory)); - } - } - - private static string FormatRange(MemoryRange range) => $"{range.Start:x}-{range.End:x}"; + if (segment.ObjectRange.Contains(address)) + { + output.Columns[0] = output.Columns[0].WithDml(Dml.ListNearObj); + } + else + { + output.Columns[0] = output.Columns[0].WithDml(null); + } - private static int RangeSizeForSegment(ClrSegment segment) - { - // segment.ObjectRange should always be less length than CommittedMemory - if (segment.CommittedMemory.Length > segment.ReservedMemory.Length) - { - return FormatRange(segment.CommittedMemory).Length; - } - else - { - return FormatRange(segment.ReservedMemory).Length; + output.WriteRow(address, segment.SubHeap, segment, generation, segment.ObjectRange, segment.CommittedMemory, segment.ReservedMemory); } } diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/ListNearObjCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/ListNearObjCommand.cs index f0bc90562e..1b4411925f 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/ListNearObjCommand.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/ListNearObjCommand.cs @@ -5,8 +5,9 @@ using System.Diagnostics; using System.Linq; using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.ExtensionCommands.Output; using Microsoft.Diagnostics.Runtime; -using static Microsoft.Diagnostics.ExtensionCommands.TableOutput; +using static Microsoft.Diagnostics.ExtensionCommands.Output.ColumnKind; namespace Microsoft.Diagnostics.ExtensionCommands { @@ -71,7 +72,9 @@ public override void Invoke() MemoryRange[] segAllocContexts = heap.EnumerateAllocationContexts().Where(context => segment.ObjectRange.Contains(context.Start)).ToArray(); int pointerColumnWidth = segAllocContexts.Length > 0 ? Math.Max(segAllocContexts.Max(r => FormatRange(r).Length), 16) : 16; - TableOutput output = new(Console, (-"Expected:".Length, ""), (pointerColumnWidth, "x16"), (20, ""), (0, "")); + Column kindColumn = Text.WithWidth("Expected:".Length).WithAlignment(Align.Left); + + Table output = new(Console, kindColumn, DumpObj.WithWidth(pointerColumnWidth), Text.WithWidth(32), TypeName); // Get current object, but objAddress may not point to an object. ClrObject curr = heap.GetObject(objAddress); @@ -99,7 +102,7 @@ public override void Invoke() if (prev.IsValid) { - expectedNextObject = Align(prev + prev.Size, segment); + expectedNextObject = AlignObj(prev + prev.Size, segment); } else { @@ -193,7 +196,7 @@ public override void Invoke() localConsistency = VerifyAndPrintObject(output, "Current:", heap, segment, curr) && localConsistency; // If curr is valid, we need to print and skip the allocation context - expectedNextObject = Align(curr + curr.Size, segment); + expectedNextObject = AlignObj(curr + curr.Size, segment); MemoryRange allocContextPlusGap = PrintGapIfExists(output, segment, segAllocContexts, new(curr, expectedNextObject)); if (allocContextPlusGap.End != 0) { @@ -278,12 +281,13 @@ private void CheckEndOfSegment(ClrSegment segment, ulong expectedNextObject, ulo } } - private MemoryRange PrintGapIfExists(TableOutput output, ClrSegment segment, MemoryRange[] segAllocContexts, MemoryRange objectDistance) + private MemoryRange PrintGapIfExists(Table output, ClrSegment segment, MemoryRange[] segAllocContexts, MemoryRange objectDistance) { // Print information about allocation context gaps between objects MemoryRange range = segAllocContexts.FirstOrDefault(ctx => objectDistance.Overlaps(ctx) || ctx.Contains(objectDistance.End)); if (range.Start != 0) { + output.Columns[1] = output.Columns[1].WithDml(null); output.WriteRow("Gap:", FormatRange(range), FormatSize(range.Length), "GC Allocation Context (expected gap in the heap)"); } @@ -296,12 +300,12 @@ private MemoryRange PrintGapIfExists(TableOutput output, ClrSegment segment, Mem } uint minObjectSize = (uint)MemoryService.PointerSize * 3; - return new(range.Start, range.End + Align(minObjectSize, segment)); + return new(range.Start, range.End + AlignObj(minObjectSize, segment)); } private static string FormatRange(MemoryRange range) => $"{range.Start:x}-{range.End:x}"; - private ulong Align(ulong size, ClrSegment seg) + private ulong AlignObj(ulong size, ClrSegment seg) { ulong AlignConst; ulong AlignLargeConst = 7; @@ -323,25 +327,28 @@ private ulong Align(ulong size, ClrSegment seg) return (size + AlignConst) & ~AlignConst; } - private bool VerifyAndPrintObject(TableOutput output, string which, ClrHeap heap, ClrSegment segment, ClrObject obj) + private bool VerifyAndPrintObject(Table output, string which, ClrHeap heap, ClrSegment segment, ClrObject obj) { bool isObjectValid = !heap.IsObjectCorrupted(obj, out ObjectCorruption corruption) && obj.IsValid; - // Here, isCorrupted may still be true, but it might not interfere with getting the type of the object. - // Since we know the information, we will print that out. - string typeName = obj.Type?.Name ?? GetErrorTypeName(obj); - // ClrObject.Size is not available if IsValid returns false string size = FormatSize(obj.IsValid ? obj.Size : 0); if (corruption is null) { - output.WriteRow(which, new DmlDumpObj(obj), size, typeName); + output.Columns[1] = output.Columns[1].WithDml(Dml.DumpObj); + output.WriteRow(which, obj, size, obj.Type); } else { - output.WriteRow(which, new DmlListNearObj(obj), size, typeName); + output.Columns[1] = output.Columns[1].WithDml(Dml.ListNearObj); + output.WriteRow(which, obj, size, obj.Type); + Console.Write($"Error Detected: {VerifyHeapCommand.GetObjectCorruptionMessage(MemoryService, heap, corruption)} "); - Console.WriteDmlExec("[verify heap]", $"!verifyheap -s {segment.Address:X}"); + if (Console.SupportsDml) + { + Console.WriteDmlExec("[verify heap]", $"!verifyheap -segment {segment.Address:X}"); + } + Console.WriteLine(); } @@ -349,17 +356,5 @@ private bool VerifyAndPrintObject(TableOutput output, string which, ClrHeap heap } private static string FormatSize(ulong size) => size > 0 ? $"{size:n0} (0x{size:x})" : ""; - - private string GetErrorTypeName(ClrObject obj) - { - if (!MemoryService.ReadPointer(obj.Address, out _)) - { - return $"[error reading mt at: {obj.Address:x}]"; - } - else - { - return $"Unknown"; - } - } } } diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/LiveObjectService.cs b/src/Microsoft.Diagnostics.ExtensionCommands/LiveObjectService.cs index ed39e0de0b..a77b997eba 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/LiveObjectService.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/LiveObjectService.cs @@ -35,6 +35,11 @@ public bool IsLive(ulong obj) return _liveObjs.Contains(obj); } + public void Initialize() + { + _liveObjs ??= CreateObjectSet(); + } + private HashSet CreateObjectSet() { ClrHeap heap = Runtime.Heap; diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/MAddressCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/MAddressCommand.cs index 670719f193..fa221c96ef 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/MAddressCommand.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/MAddressCommand.cs @@ -5,7 +5,9 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.ExtensionCommands.Output; using static Microsoft.Diagnostics.ExtensionCommands.NativeAddressHelper; +using static Microsoft.Diagnostics.ExtensionCommands.Output.ColumnKind; namespace Microsoft.Diagnostics.ExtensionCommands { @@ -17,11 +19,13 @@ public sealed class MAddressCommand : CommandBase private const string ReserveFlag = "-reserve"; private const string ReserveHeuristicFlag = "-reserveHeuristic"; private const string ForceHandleTableFlag = "-forceHandleTable"; + private const string ListFlag = "-list"; + private const string BySizeFlag = "-orderBySize"; [Option(Name = SummaryFlag, Aliases = new string[] { "-stat", }, Help = "Only print summary table.")] public bool Summary { get; set; } - [Option(Name = ImagesFlag, Aliases = new string[] { "-i" }, Help = "Prints a summary table of image memory usage.")] + [Option(Name = ImagesFlag, Help = "Prints a summary table of image memory usage.")] public bool ShowImageTable { get; set; } [Option(Name = ReserveFlag, Help = "Include MEM_RESERVE regions in the output.")] @@ -33,6 +37,12 @@ public sealed class MAddressCommand : CommandBase [Option(Name = ForceHandleTableFlag, Help = "We only tag the HandleTable if we can do so efficiently on newer runtimes. This option ensures we always tag HandleTable memory, even if it will take a long time.")] public bool IncludeHandleTableIfSlow { get; set; } + [Option(Name = BySizeFlag, Help = "List the raw addresses by size, not by base address.")] + public bool BySize { get; set; } + + [Option(Name = ListFlag, Help = "A separated list of memory regions to list allocations for.")] + public string List { get; set; } + [ServiceImport] public NativeAddressHelper AddressHelper { get; set; } @@ -51,35 +61,31 @@ public override void Invoke() DescribedRegion[] ranges = memoryRanges.ToArray(); - int nameSizeMax = ranges.Max(r => r.Name.Length); - // Tag reserved memory based on what's adjacent. if (TagReserveMemoryHeuristically) { CollapseReserveRegions(ranges); } - if (!Summary) + if (!Summary && List is null) { - int kindSize = ranges.Max(r => r.Type.ToString().Length); - int stateSize = ranges.Max(r => r.State.ToString().Length); - int protectSize = ranges.Max(r => r.Protection.ToString().Length); - - TableOutput output = new(Console, (nameSizeMax, ""), (12, "x"), (12, "x"), (12, ""), (kindSize, ""), (stateSize, ""), (protectSize, "")) - { - AlignLeft = true, - Divider = " | " - }; - - output.WriteRowWithSpacing('-', "Memory Kind", "StartAddr", "EndAddr-1", "Size", "Type", "State", "Protect", "Image"); - foreach (DescribedRegion mem in ranges) + Column nameColumn = Text.GetAppropriateWidth(ranges.Select(r => r.Name)); + Column kindColumn = Column.ForEnum(); + Column stateColumn = Column.ForEnum(); + + // These are flags, so we need a column wide enough for that output instead of ForEnum + Column protectionColumn = Text.GetAppropriateWidth(ranges.Select(r => r.Protection)); + Column imageColumn = Image.GetAppropriateWidth(ranges.Select(r => r.Image)); + using BorderedTable output = new(Console, nameColumn, Pointer, Pointer, HumanReadableSize, kindColumn, stateColumn, protectionColumn, imageColumn); + + output.WriteHeader("Memory Kind", "StartAddr", "EndAddr-1", "Size", "Type", "State", "Protect", "Image"); + IOrderedEnumerable ordered = BySize ? ranges.OrderByDescending(r => r.Size).ThenBy(r => r.Start) : ranges.OrderBy(r => r.Start); + foreach (DescribedRegion mem in ordered) { Console.CancellationToken.ThrowIfCancellationRequested(); - output.WriteRow(mem.Name, mem.Start, mem.End, mem.Size.ConvertToHumanReadable(), mem.Type, mem.State, mem.Protection, mem.Image); + output.WriteRow(mem.Name, mem.Start, mem.End, mem.Size, mem.Type, mem.State, mem.Protection, mem.Image); } - - output.WriteSpacer('-'); } if (ShowImageTable) @@ -94,34 +100,70 @@ orderby Size descending Size }; - int moduleLen = Math.Max(80, ranges.Max(r => r.Image?.Length ?? 0)); - - TableOutput output = new(Console, (moduleLen, ""), (8, "n0"), (12, ""), (24, "n0")) + using (BorderedTable output = new(Console, Image.GetAppropriateWidth(ranges.Select(r => r.Image), max: 80), Integer, HumanReadableSize, ByteCount)) { - Divider = " | " - }; + Console.CancellationToken.ThrowIfCancellationRequested(); - output.WriteRowWithSpacing('-', "Image", "Regions", "Size", "Size (bytes)"); + output.WriteHeader("Image", "Count", "Size", "Size (bytes)"); - int count = 0; - long size = 0; - foreach (var item in imageGroups) - { - Console.CancellationToken.ThrowIfCancellationRequested(); + int count = 0; + long size = 0; + foreach (var item in imageGroups) + { + Console.CancellationToken.ThrowIfCancellationRequested(); - output.WriteRow(item.Image, item.Count, item.Size.ConvertToHumanReadable(), item.Size); - count += item.Count; - size += item.Size; + output.WriteRow(item.Image, item.Count, item.Size, item.Size); + count += item.Count; + size += item.Size; + } + + output.WriteFooter("[TOTAL]", count, size, size); } - output.WriteSpacer('-'); - output.WriteRow("[TOTAL]", count, size.ConvertToHumanReadable(), size); - WriteLine(""); + Console.WriteLine(); } - // Print summary table unconditionally + if (List is not null) + { + // Print a list of the specified memory kind, ordered by size descending. + + string[] requested = List.Split(new char[] { ',', ' ' }, StringSplitOptions.RemoveEmptyEntries); + foreach (string kind in requested) + { + if (!ranges.Any(r => r.Name.Equals(kind, StringComparison.OrdinalIgnoreCase))) + { + Console.WriteLineError($"No memory regions match '{kind}'."); + } + else + { + Console.WriteLine($"{kind} Memory Regions:"); + + Table output = new(Console, Pointer, ByteCount, HumanReadableSize, Column.ForEnum(), Column.ForEnum(), Column.ForEnum().WithWidth(-1)); + output.WriteHeader("Base Address", "Size (bytes)", "Size", "Mem State", "Mem Type", "Mem Protect"); + + ulong totalSize = 0; + int count = 0; + + IEnumerable matching = ranges.Where(r => r.Name.Equals(kind, StringComparison.OrdinalIgnoreCase)).OrderByDescending(s => s.Size); + foreach (DescribedRegion region in matching) + { + output.WriteRow(region.Start, region.Size, region.Size, region.State, region.Type, region.Protection); + + count++; + totalSize += region.Size; + } + + Console.WriteLine($"{totalSize:n0} bytes ({totalSize.ConvertToHumanReadable()}) in {count:n0} memory regions"); + Console.WriteLine(); + } + } + } + + if (List is null || Summary) { + // Show the summary table in almost every case, unless the user specified -list without -summary. + var grouped = from mem in ranges let name = mem.Name group mem by name into g @@ -134,12 +176,9 @@ orderby Size descending Size }; - TableOutput output = new(Console, (-nameSizeMax, ""), (8, "n0"), (12, ""), (24, "n0")) - { - Divider = " | " - }; - - output.WriteRowWithSpacing('-', "Region Type", "Count", "Size", "Size (bytes)"); + Column nameColumn = Text.GetAppropriateWidth(ranges.Select(r => r.Name)); + using BorderedTable output = new(Console, nameColumn, Integer, HumanReadableSize, ByteCount); + output.WriteHeader("Memory Type", "Count", "Size", "Size (bytes)"); int count = 0; long size = 0; @@ -147,17 +186,16 @@ orderby Size descending { Console.CancellationToken.ThrowIfCancellationRequested(); - output.WriteRow(item.Name, item.Count, item.Size.ConvertToHumanReadable(), item.Size); + output.WriteRow(item.Name, item.Count, item.Size, item.Size); + count += item.Count; size += item.Size; } - output.WriteSpacer('-'); - output.WriteRow("[TOTAL]", count, size.ConvertToHumanReadable(), size); + output.WriteFooter("[TOTAL]", count, size, size); } } - [HelpInvoke] public void HelpInvoke() { @@ -170,7 +208,7 @@ with information about CLR's heaps. Flags: {SummaryFlag} - Show only a summary table of memory regions and not the list of every address region. + Show only a summary table of memory regions and not the list of every memory region. {ImagesFlag} Summarizes the memory ranges consumed by images in the process. @@ -192,6 +230,14 @@ Include reserved memory (MEM_RESERVE) in the output. This is usually only that reserve region HeapReserve. Note that this is a heuristic and NOT intended to be completely accurate. This can be useful to try to figure out what is creating large amount of MEM_RESERVE regions. + + {ListFlag} + A separated list of memory region types (as maddress defines them) to print the base + addresses and sizes of. This list may be separated by , or ""in quotes"". + + {BySizeFlag} + Order the list of memory blocks by size (descending) when printing the list + of all memory blocks instead of by address. "); } } diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/NativeAddressHelper.cs b/src/Microsoft.Diagnostics.ExtensionCommands/NativeAddressHelper.cs index 9d6f5f02e7..a8e46ed7c3 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/NativeAddressHelper.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/NativeAddressHelper.cs @@ -13,8 +13,19 @@ namespace Microsoft.Diagnostics.ExtensionCommands { [ServiceExport(Scope = ServiceScope.Target)] - public sealed class NativeAddressHelper + public sealed class NativeAddressHelper : IDisposable { + private readonly IDisposable _onFlushEvent; + private ((bool, bool, bool, bool) Key, DescribedRegion[] Result) _previous; + + public NativeAddressHelper(ITarget target) + { + Target = target; + _onFlushEvent = target.OnFlushEvent.Register(() => _previous = default); + } + + public void Dispose() => _onFlushEvent.Dispose(); + [ServiceImport] public ITarget Target { get; set; } @@ -58,8 +69,27 @@ public sealed class NativeAddressHelper /// If !address fails we will throw InvalidOperationException. This is usually /// because symbols for ntdll couldn't be found. /// An enumerable of memory ranges. - internal IEnumerable EnumerateAddressSpace(bool tagClrMemoryRanges, bool includeReserveMemory, bool tagReserveMemoryHeuristically, bool includeHandleTableIfSlow) + public IEnumerable EnumerateAddressSpace(bool tagClrMemoryRanges, bool includeReserveMemory, bool tagReserveMemoryHeuristically, bool includeHandleTableIfSlow) + { + (bool, bool, bool, bool) key = (tagClrMemoryRanges, includeReserveMemory, tagReserveMemoryHeuristically, includeHandleTableIfSlow); + + if (_previous.Result is not null && _previous.Key == key) + { + return _previous.Result; + } + + DescribedRegion[] result = EnumerateAddressSpaceWorker(tagClrMemoryRanges, includeReserveMemory, tagReserveMemoryHeuristically, includeHandleTableIfSlow); + _previous = (key, result); + + // Use AsReadOnly to ensure no modifications to the cached value + return Array.AsReadOnly(result); + } + + private DescribedRegion[] EnumerateAddressSpaceWorker(bool tagClrMemoryRanges, bool includeReserveMemory, bool tagReserveMemoryHeuristically, bool includeHandleTableIfSlow) { + Console.WriteLineWarning("Enumerating and tagging the entire address space and caching the result..."); + Console.WriteLineWarning("Subsequent runs of this command should be faster."); + bool printedTruncatedWarning = false; IEnumerable addressResult = from region in MemoryRegionService.EnumerateRegions() @@ -80,31 +110,53 @@ internal IEnumerable EnumerateAddressSpace(bool tagClrMemoryRan RootCacheService rootCache = runtime.Services.GetService(); if (clrRuntime is not null) { - foreach ((ulong Address, ulong? Size, ClrMemoryKind Kind) mem in EnumerateClrMemoryAddresses(clrRuntime, rootCache, includeHandleTableIfSlow)) + foreach ((ulong Address, ulong Size, ClrMemoryKind Kind) mem in EnumerateClrMemoryAddresses(clrRuntime, rootCache, includeHandleTableIfSlow)) { - DescribedRegion[] found = rangeList.Where(r => r.Start <= mem.Address && mem.Address < r.End).ToArray(); + // The GCBookkeeping range is a large region of memory that the GC reserved. We'll simply mark every + // region within it as bookkeeping. + if (mem.Kind == ClrMemoryKind.GCBookkeeping) + { + MemoryRange bookkeepingRange = MemoryRange.CreateFromLength(mem.Address, mem.Size); + foreach (DescribedRegion region in rangeList) + { + if (bookkeepingRange.Contains(region.Start)) + { + if (region.State == MemoryRegionState.MEM_RESERVE) + { + region.ClrMemoryKind = ClrMemoryKind.GCBookkeepingReserve; + } + else + { + region.ClrMemoryKind = ClrMemoryKind.GCBookkeeping; + } + } + } + + continue; + } + DescribedRegion[] found = rangeList.Where(r => r.Start <= mem.Address && mem.Address < r.End).ToArray(); if (found.Length == 0 && mem.Kind != ClrMemoryKind.GCHeapReserve) { Trace.WriteLine($"Warning: Could not find a memory range for {mem.Address:x} - {mem.Kind}."); if (!printedTruncatedWarning) { - Console.WriteLine($"Warning: Could not find a memory range for {mem.Address:x} - {mem.Kind}."); - Console.WriteLine($"This crash dump may not be a full dump!"); - Console.WriteLine(""); + Console.WriteLineWarning($"Warning: Could not find a memory range for {mem.Address:x} - {mem.Kind}."); + Console.WriteLineWarning($"This crash dump may not be a full dump!"); + Console.WriteLineWarning(""); printedTruncatedWarning = true; } // Add the memory range if we know its size. - if (mem.Size is ulong size && size > 0) + if (mem.Size > 0) { IModule module = ModuleService.GetModuleFromAddress(mem.Address); rangeList.Add(new DescribedRegion() { Start = mem.Address, - End = mem.Address + size, + End = mem.Address + mem.Size, ClrMemoryKind = mem.Kind, State = mem.Kind == ClrMemoryKind.GCHeapReserve ? MemoryRegionState.MEM_RESERVE : MemoryRegionState.MEM_COMMIT, Module = module, @@ -122,7 +174,7 @@ internal IEnumerable EnumerateAddressSpace(bool tagClrMemoryRan foreach (DescribedRegion region in found) { - if (!mem.Size.HasValue || mem.Size.Value == 0) + if (mem.Size == 0) { // If we don't know the length of memory, just mark the Region with this tag. SetRegionKindWithWarning(mem, region); @@ -146,7 +198,7 @@ internal IEnumerable EnumerateAddressSpace(bool tagClrMemoryRan DescribedRegion middleRegion = new(region) { Start = mem.Address, - End = mem.Address + mem.Size.Value, + End = mem.Address + mem.Size, ClrMemoryKind = mem.Kind, Usage = MemoryRegionUsage.CLR, }; @@ -173,7 +225,7 @@ internal IEnumerable EnumerateAddressSpace(bool tagClrMemoryRan // Region is now the starting region of this set. region.End = middleRegion.Start; } - else if (region.Size < mem.Size.Value) + else if (region.Size < mem.Size) { SetRegionKindWithWarning(mem, region); @@ -192,15 +244,15 @@ internal IEnumerable EnumerateAddressSpace(bool tagClrMemoryRan // If we found no matching regions, expand the current region to be the right length. if (!foundNext) { - region.End = mem.Address + mem.Size.Value; + region.End = mem.Address + mem.Size; } } - else if (region.Size > mem.Size.Value) + else if (region.Size > mem.Size) { // The CLR memory segment is at the beginning of this region. DescribedRegion newRange = new(region) { - End = mem.Address + mem.Size.Value, + End = mem.Address + mem.Size, ClrMemoryKind = mem.Kind }; @@ -210,8 +262,11 @@ internal IEnumerable EnumerateAddressSpace(bool tagClrMemoryRan region.ClrMemoryKind = mem.Kind; } } + else + { + SetRegionKindWithWarning(mem, region); + } } - } } } @@ -244,14 +299,26 @@ internal IEnumerable EnumerateAddressSpace(bool tagClrMemoryRan /// /// Enumerates pointers to various CLR heaps in memory. /// - private static IEnumerable<(ulong Address, ulong? Size, ClrMemoryKind Kind)> EnumerateClrMemoryAddresses(ClrRuntime runtime, RootCacheService rootCache, bool includeHandleTableIfSlow) + private static IEnumerable<(ulong Address, ulong Size, ClrMemoryKind Kind)> EnumerateClrMemoryAddresses(ClrRuntime runtime, RootCacheService rootCache, bool includeHandleTableIfSlow) { foreach (ClrNativeHeapInfo nativeHeap in runtime.EnumerateClrNativeHeaps()) { - yield return (nativeHeap.Address, nativeHeap.Size, nativeHeap.Kind == NativeHeapKind.Unknown ? ClrMemoryKind.None : (ClrMemoryKind)nativeHeap.Kind); + Debug.Assert((int)NativeHeapKind.GCBookkeeping == (int)ClrMemoryKind.GCBookkeeping); + + ClrMemoryKind kind = nativeHeap.Kind switch + { + NativeHeapKind.Unknown => ClrMemoryKind.Unknown, + > NativeHeapKind.Unknown and <= NativeHeapKind.GCBookkeeping => (ClrMemoryKind)nativeHeap.Kind, // enums match for these ranges + >= NativeHeapKind.GCFreeRegion and <= NativeHeapKind.GCFreeUohSegment => ClrMemoryKind.GCHeapToBeFreed, + _ => ClrMemoryKind.Unknown + }; + + yield return (nativeHeap.MemoryRange.Start, nativeHeap.MemoryRange.Length, kind); } - if (includeHandleTableIfSlow) + // .Net 8 and beyond has accurate HandleTable memory info. + bool haveAccurateHandleInfo = runtime.ClrInfo.Flavor == ClrFlavor.Core && runtime.ClrInfo.Version.Major >= 8; + if (includeHandleTableIfSlow && !haveAccurateHandleInfo) { ulong prevHandle = 0; ulong granularity = 0x100; @@ -265,29 +332,27 @@ internal IEnumerable EnumerateAddressSpace(bool tagClrMemoryRan if (handle.Address < prevHandle || handle.Address >= (prevHandle | (granularity - 1))) { - yield return (handle.Address, null, ClrMemoryKind.HandleTable); + yield return (handle.Address, 0, ClrMemoryKind.HandleTable); prevHandle = handle.Address; } } } - // We don't really have the true bounds of the committed or reserved segments. - // Return null for the size so that we will mark the entire region with this type. foreach (ClrSegment seg in runtime.Heap.Segments) { if (seg.CommittedMemory.Length > 0) { - yield return (seg.CommittedMemory.Start, null, ClrMemoryKind.GCHeap); + yield return (seg.CommittedMemory.Start, seg.CommittedMemory.Length, ClrMemoryKind.GCHeap); } if (seg.ReservedMemory.Length > 0) { - yield return (seg.ReservedMemory.Start, null, ClrMemoryKind.GCHeapReserve); + yield return (seg.ReservedMemory.Start, seg.ReservedMemory.Length, ClrMemoryKind.GCHeapReserve); } } } - private static void SetRegionKindWithWarning((ulong Address, ulong? Size, ClrMemoryKind Kind) mem, DescribedRegion region) + private static void SetRegionKindWithWarning((ulong Address, ulong Size, ClrMemoryKind Kind) mem, DescribedRegion region) { if (region.ClrMemoryKind != mem.Kind) { @@ -297,12 +362,7 @@ private static void SetRegionKindWithWarning((ulong Address, ulong? Size, ClrMem if (region.ClrMemoryKind is not ClrMemoryKind.None and not ClrMemoryKind.HighFrequencyHeap) { - if (mem.Size is not ulong size) - { - size = 0; - } - - Trace.WriteLine($"Warning: Overwriting range [{region.Start:x},{region.End:x}] {region.ClrMemoryKind} -> [{mem.Address:x},{mem.Address + size:x}] {mem.Kind}."); + Trace.WriteLine($"Warning: Overwriting range [{region.Start:x},{region.End:x}] {region.ClrMemoryKind} -> [{mem.Address:x},{mem.Address + mem.Size:x}] {mem.Kind}."); } region.ClrMemoryKind = mem.Kind; @@ -469,15 +529,22 @@ public enum ClrMemoryKind StubHeap, HighFrequencyHeap, LowFrequencyHeap, + ExecutableHeap, + FixupPrecodeHeap, + NewStubPrecodeHeap, + ThunkHeap, + HandleTable, + GCBookkeeping, // Skip ahead so new ClrMD NativeHeapKind values don't break the enum. Unknown = 100, GCHeap, + GCHeapToBeFreed, GCHeapReserve, - HandleTable, + GCBookkeepingReserve, } - internal sealed class DescribedRegion : IMemoryRegion + public sealed class DescribedRegion : IMemoryRegion { public DescribedRegion() { diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/Output/Align.cs b/src/Microsoft.Diagnostics.ExtensionCommands/Output/Align.cs new file mode 100644 index 0000000000..bcd1a06974 --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/Output/Align.cs @@ -0,0 +1,12 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Diagnostics.ExtensionCommands.Output +{ + internal enum Align + { + Left, + Right, + Center + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/Output/BorderedTable.cs b/src/Microsoft.Diagnostics.ExtensionCommands/Output/BorderedTable.cs new file mode 100644 index 0000000000..f3a9015e8d --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/Output/BorderedTable.cs @@ -0,0 +1,125 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Text; +using Microsoft.Diagnostics.DebugServices; + +namespace Microsoft.Diagnostics.ExtensionCommands.Output +{ + internal sealed class BorderedTable : Table, IDisposable + { + private bool _wroteAtLeastOneSpacer; + + public BorderedTable(IConsoleService console, params Column[] columns) + : base(console, columns) + { + _spacing = $" | "; + } + + public void Dispose() + { + WriteSpacer(); + } + + public override void WriteHeader(params string[] values) + { + IncreaseColumnWidth(values); + + WriteSpacer(); + WriteHeaderFooter(values, writeSides: true, writeNewline: true); + WriteSpacer(); + } + + public override void WriteFooter(params object[] values) + { + WriteSpacer(); + WriteHeaderFooter(values, writeSides: true, writeNewline: true); + } + + public override void WriteRow(params object[] values) + { + // Ensure the top of the table is written even if there's no header/footer. + if (!_wroteAtLeastOneSpacer) + { + WriteSpacer(); + } + + StringBuilder rowBuilder = _stringBuilderPool.Rent(); + rowBuilder.Append(Indent); + rowBuilder.Append(_spacing); + + WriteRowWorker(values, rowBuilder, _spacing, writeLine: false); + rowBuilder.Append(_spacing); + + FinishColumns(values.Length, rowBuilder); + + Console.WriteLine(rowBuilder.ToString()); + _stringBuilderPool.Return(rowBuilder); + } + + protected override void WriteHeaderFooter(object[] values, bool writeSides, bool writeNewline) + { + base.WriteHeaderFooter(values, writeSides, writeNewline: false); + + StringBuilder rowBuilder = _stringBuilderPool.Rent(); + FinishColumns(values.Length, rowBuilder); + + if (writeNewline) + { + rowBuilder.AppendLine(); + } + + Console.Write(rowBuilder.ToString()); + _stringBuilderPool.Return(rowBuilder); + } + + private void FinishColumns(int start, StringBuilder rowBuilder) + { + for (int i = start; i < Columns.Length; i++) + { + if (Columns[i].Width < 0) + { + break; + } + + rowBuilder.Append(' ', Columns[i].Width); + rowBuilder.Append(_spacing); + } + } + + private void WriteSpacer() + { + WriteBorder(" +-", '-', "-+ "); + _wroteAtLeastOneSpacer = true; + } + + private void WriteBorder(string left, char center, string right) + { + StringBuilder rowBuilder = _stringBuilderPool.Rent(); + rowBuilder.Append(Indent); + + rowBuilder.Append(left); + + for (int i = 0; i < Columns.Length; i++) + { + if (i != 0) + { + rowBuilder.Append(center, _spacing.Length); + } + + if (Columns[i].Width < 0) + { + break; + } + + rowBuilder.Append(center, Columns[i].Width); + } + + rowBuilder.Append(right); + Console.WriteLine(rowBuilder.ToString()); + + _stringBuilderPool.Return(rowBuilder); + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/Output/Column.cs b/src/Microsoft.Diagnostics.ExtensionCommands/Output/Column.cs new file mode 100644 index 0000000000..6c37a534e4 --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/Output/Column.cs @@ -0,0 +1,80 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.Diagnostics.ExtensionCommands.Output +{ + internal readonly struct Column + { + private static readonly StringBuilderPool s_stringBuilderPool = new(); + private static readonly Column s_enum = new(Align.Left, -1, new(), null); + + public readonly int Width; + public readonly Format Format; + public readonly Align Alignment; + public readonly DmlFormat Dml; + + public Column(Align alignment, int width, Format format, DmlFormat dml = null) + { + Alignment = alignment; + Width = width; + Format = format ?? throw new ArgumentNullException(nameof(format)); + Dml = dml; + } + + public readonly Column WithWidth(int width) => new(Alignment, width, Format, Dml); + internal readonly Column WithDml(DmlFormat dml) => new(Alignment, Width, Format, dml); + internal readonly Column WithAlignment(Align align) => new(align, Width, Format, Dml); + + public readonly Column GetAppropriateWidth(IEnumerable values, int min = -1, int max = -1) + { + int len = 0; + + StringBuilder sb = s_stringBuilderPool.Rent(); + + foreach (T value in values) + { + sb.Clear(); + Format.FormatValue(sb, value, -1, false); + len = Math.Max(len, sb.Length); + } + + s_stringBuilderPool.Return(sb); + + if (len < min) + { + len = min; + } + + if (max > 0 && len > max) + { + len = max; + } + + return WithWidth(len); + } + + internal static Column ForEnum() + where TEnum : struct + { + int len = 0; + foreach (TEnum t in Enum.GetValues(typeof(TEnum))) + { + len = Math.Max(len, t.ToString().Length); + } + + return s_enum.WithWidth(len); + } + + public override string ToString() + { + string format = Format?.GetType().Name ?? "null"; + string dml = Dml?.GetType().Name ?? "null"; + + return $"align:{Alignment} width:{Width} format:{format} dml:{dml}"; + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/Output/ColumnKind.cs b/src/Microsoft.Diagnostics.ExtensionCommands/Output/ColumnKind.cs new file mode 100644 index 0000000000..aeaae671c7 --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/Output/ColumnKind.cs @@ -0,0 +1,112 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Diagnostics.ExtensionCommands.Output +{ + internal static class ColumnKind + { + private static Column? s_pointer; + private static Column? s_text; + private static Column? s_hexOffset; + private static Column? s_hexValue; + private static Column? s_dumpObj; + private static Column? s_integer; + private static Column? s_dumpHeapMT; + private static Column? s_listNearObj; + private static Column? s_dumpDomain; + private static Column? s_thread; + private static Column? s_integerWithoutComma; + private static Column? s_humanReadable; + private static Column? s_range; + + // NOTE/BUGBUG: This assumes IntPtr.Size matches the target process, which it should not do + private static int PointerLength => IntPtr.Size * 2; + + /// + /// A pointer, displayed as hex. + /// + public static Column Pointer => s_pointer ??= new(Align.Right, PointerLength, Formats.Pointer); + + /// + /// Raw text which will not be truncated by default. + /// + public static Column Text => s_text ??= new(Align.Left, -1, Formats.Text); + + /// + /// A hex value, prefixed with 0x. + /// + public static Column HexValue => s_hexValue ??= new(Align.Right, PointerLength + 2, Formats.HexValue); + + /// + /// An offset (potentially negative), prefixed with 0x. For example: '0x20' or '-0x20'. + /// + public static Column HexOffset => s_hexOffset ??= new(Align.Right, 10, Formats.HexOffset); + + /// + /// An integer, with commas. i.e. i.ToString("n0") + /// + public static Column Integer => s_integer ??= new(Align.Right, 14, Formats.Integer); + + /// + /// An integer, without commas. + /// + public static Column IntegerWithoutCommas => s_integerWithoutComma ??= new(Align.Right, 10, Formats.IntegerWithoutCommas); + + /// + /// A count of bytes (size). + /// + public static Column ByteCount => Integer; + + /// + /// A human readable size count. e.g. "1.23mb" + /// + public static Column HumanReadableSize => s_humanReadable ??= new(Align.Right, 12, Formats.HumanReadableSize); + + /// + /// An object pointer, which we would like to link to !do if Dml is enabled. + /// + public static Column DumpObj => s_dumpObj ??= new(Align.Right, PointerLength, Formats.Pointer, Dml.DumpObj); + + /// + /// A link to any number of ClrMD objects (ClrSubHeap, ClrSegment, a MethodTable or ClrType, etc) which will + /// print an appropriate !dumpheap filter for, if dml is enabled. + /// + public static Column DumpHeap => s_dumpHeapMT ??= new(Align.Right, PointerLength, Formats.Pointer, Dml.DumpHeap); + + /// + /// A link to !dumpdomain for the given domain, if dml is enabled. This also puts the domain's name in the + /// hover text for the link. + /// + public static Column DumpDomain => s_dumpDomain ??= new(Align.Right, PointerLength, Formats.Pointer, Dml.DumpDomain); + + /// + /// The ClrThread address with a link to the OSThreadID to change threads (if dml is enabled). + /// + public static Column Thread => s_thread ??= new(Align.Right, PointerLength, Formats.Pointer, Dml.Thread); + + /// + /// A link to !listnearobj for the given ClrObject or address, if dml is enabled. + /// + public static Column ListNearObj => s_listNearObj ??= new(Align.Right, PointerLength, Formats.Pointer, Dml.ListNearObj); + + /// + /// The name of a given type. Note that types are always truncated by removing the beginning of the type's + /// name instead of truncating based on alignment. This ensures the most important part of the name (the + /// actual type name) is preserved instead of the namespace. + /// + public static Column TypeName => s_text ??= new(Align.Left, -1, Formats.TypeName); + + /// + /// A path to an image on disk. Note that images are always truncted by removing the beginning of the image's + /// path instead of the end, preserving the filename. + /// + public static Column Image => s_text ??= new(Align.Left, -1, Formats.Image); + + /// + /// A MemoryRange printed as "[start-end]". + /// + public static Column Range => s_range ??= new(Align.Left, PointerLength * 2 + 1, Formats.Range); + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/Output/Dml.cs b/src/Microsoft.Diagnostics.ExtensionCommands/Output/Dml.cs new file mode 100644 index 0000000000..e309f40ae0 --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/Output/Dml.cs @@ -0,0 +1,306 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics; +using System.Text; +using Microsoft.Diagnostics.Runtime; + +namespace Microsoft.Diagnostics.ExtensionCommands.Output +{ + internal static class Dml + { + private static DmlDumpObject s_dumpObj; + private static DmlDumpHeap s_dumpHeap; + private static DmlBold s_bold; + private static DmlListNearObj s_listNearObj; + private static DmlDumpDomain s_dumpDomain; + private static DmlThread s_thread; + + /// + /// Runs !dumpobj on the given pointer or ClrObject. If a ClrObject is invalid, + /// this will instead link to !verifyobj. + /// + public static DmlFormat DumpObj => s_dumpObj ??= new(); + + /// + /// Marks the output in bold. + /// + public static DmlFormat Bold => s_bold ??= new(); + + /// + /// Dumps the heap. If given a ClrSegment, ClrSubHeap, or MemoryRange it will + /// just dump that particular section of the heap. + /// + public static DmlFormat DumpHeap => s_dumpHeap ??= new(); + + /// + /// Runs ListNearObj on the given address or ClrObject. + /// + public static DmlFormat ListNearObj => s_listNearObj ??= new(); + + /// + /// Runs !dumpdomain on the given doman, additionally it will put the domain + /// name as the hover text. + /// + public static DmlFormat DumpDomain => s_dumpDomain ??= new(); + + /// + /// Changes the debugger to the given thread. + /// + public static DmlFormat Thread => s_thread ??= new(); + + private sealed class DmlBold : DmlFormat + { + public override void FormatValue(StringBuilder sb, string outputText, object value) + { + sb.Append(""); + sb.Append(DmlEscape(outputText)); + sb.Append(""); + } + } + + private abstract class DmlExec : DmlFormat + { + public override void FormatValue(StringBuilder sb, string outputText, object value) + { + string command = GetCommand(outputText, value); + if (string.IsNullOrWhiteSpace(command)) + { + sb.Append(DmlEscape(outputText)); + return; + } + + sb.Append("'); + sb.Append(DmlEscape(outputText)); + sb.Append(""); + } + + protected abstract string GetCommand(string outputText, object value); + protected virtual string GetAltText(string outputText, object value) => null; + + protected static bool IsNullOrZeroValue(object obj, out string value) + { + if (obj is null) + { + value = null; + return true; + } + else if (TryGetPointerValue(obj, out ulong ul) && ul == 0) + { + value = "0"; + return true; + } + + value = null; + return false; + } + + protected static bool TryGetPointerValue(object value, out ulong ulVal) + { + if (value is ulong ul) + { + ulVal = ul; + return true; + } + else if (value is nint ni) + { + unchecked + { + ulVal = (ulong)ni; + } + return true; + } + else if (value is nuint nuint) + { + ulVal = nuint; + return true; + } + + ulVal = 0; + return false; + } + } + + private sealed class DmlThread : DmlExec + { + protected override string GetCommand(string outputText, object value) + { + if (value is uint id) + { + return $"~~[{id:x}]s"; + } + + if (value is ClrThread thread) + { + return $"~~[{thread.OSThreadId:x}]s"; + } + + return null; + } + } + + private class DmlDumpObject : DmlExec + { + protected override string GetCommand(string outputText, object value) + { + bool isValid = true; + if (value is ClrObject obj) + { + isValid = obj.IsValid; + } + + value = Format.Unwrap(value); + if (IsNullOrZeroValue(value, out string result)) + { + return result; + } + + return isValid ? $"!dumpobj /d {value:x}" : $"!verifyobj {value:x}"; + } + + protected override string GetAltText(string outputText, object value) + { + if (value is ClrObject obj) + { + if (obj.IsValid) + { + return obj.Type?.Name; + } + + return "Invalid Object"; + } + + return null; + } + } + + private sealed class DmlListNearObj : DmlDumpObject + { + protected override string GetCommand(string outputText, object value) + { + value = Format.Unwrap(value); + if (IsNullOrZeroValue(value, out string result)) + { + return result; + } + + return $"!listnearobj {value:x}"; + } + } + + private sealed class DmlDumpHeap : DmlExec + { + protected override string GetCommand(string outputText, object value) + { + if (value is null) + { + return null; + } + + if (TryGetMethodTableOrTypeHandle(value, out ulong mtOrTh)) + { + // !dumpheap will only work on a method table + if ((mtOrTh & 2) == 2) + { + // Can't use typehandles + return null; + } + else if ((mtOrTh & 1) == 1) + { + // Clear mark bit + value = mtOrTh & ~1ul; + } + + if (mtOrTh == 0) + { + return null; + } + + return $"!dumpheap -mt {value:x}"; + } + + if (value is ClrSegment seg) + { + return $"!dumpheap -segment {seg.Address:x}"; + } + + if (value is MemoryRange range) + { + return $"!dumpheap {range.Start:x} {range.End:x}"; + } + + if (value is ClrSubHeap subHeap) + { + return $"!dumpheap -heap {subHeap.Index}"; + } + + Debug.Fail($"Unknown cannot use type {value.GetType().FullName} with DumpObj"); + return null; + } + + private static bool TryGetMethodTableOrTypeHandle(object value, out ulong mtOrTh) + { + if (TryGetPointerValue(value, out mtOrTh)) + { + return true; + } + + if (value is ClrType type) + { + mtOrTh = type.MethodTable; + return true; + } + + mtOrTh = 0; + return false; + } + + protected override string GetAltText(string outputText, object value) + { + if (value is ClrType type) + { + return type.Name; + } + + return null; + } + } + + private sealed class DmlDumpDomain : DmlExec + { + protected override string GetCommand(string outputText, object value) + { + value = Format.Unwrap(value); + if (IsNullOrZeroValue(value, out string result)) + { + return result; + } + + return $"!dumpdomain /d {value:x}"; + } + + protected override string GetAltText(string outputText, object value) + { + if (value is ClrAppDomain domain) + { + return domain.Name; + } + + return null; + } + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/Output/DmlFormat.cs b/src/Microsoft.Diagnostics.ExtensionCommands/Output/DmlFormat.cs new file mode 100644 index 0000000000..07bb1c2c0d --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/Output/DmlFormat.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text; +using System.Xml.Linq; + +namespace Microsoft.Diagnostics.ExtensionCommands.Output +{ + internal abstract class DmlFormat + { + // intentionally not shared with Format + private static readonly StringBuilderPool s_stringBuilderPool = new(); + + public virtual string FormatValue(string outputText, object value) + { + StringBuilder sb = s_stringBuilderPool.Rent(); + + FormatValue(sb, outputText, value); + string result = sb.ToString(); + s_stringBuilderPool.Return(sb); + return result; + } + + public abstract void FormatValue(StringBuilder sb, string outputText, object value); + + protected static string DmlEscape(string text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return text; + } + + return new XText(text).ToString(); + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/Output/Format.cs b/src/Microsoft.Diagnostics.ExtensionCommands/Output/Format.cs new file mode 100644 index 0000000000..1ed4a4e075 --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/Output/Format.cs @@ -0,0 +1,130 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Text; +using Microsoft.Diagnostics.Runtime; + +namespace Microsoft.Diagnostics.ExtensionCommands.Output +{ + internal class Format + { + private static StringBuilderPool s_stringBuilderPool = new(); + + /// + /// Returns true if a format of this type should never be truncated. If true, + /// DEBUG builds of SOS Assert.Fail if attempting to truncate the value of the + /// column. In release builds, we will simply not truncate the value, resulting + /// in a jagged looking table, but usable output. + /// + public bool CanTruncate { get; protected set; } + + public Format() { } + public Format(bool canTruncate) => CanTruncate = canTruncate; + + // Unwraps an object to get at what should be formatted. + internal static object Unwrap(object value) + { + return value switch + { + ClrObject obj => obj.Address, + ClrAppDomain domain => domain.Address, + ClrType type => type.MethodTable, + ClrSegment seg => seg.Address, + ClrThread thread => thread.Address, + ClrSubHeap subHeap => subHeap.Index, + _ => value + }; + } + + public virtual string FormatValue(object value, int maxLength, bool truncateBegin) + { + StringBuilder sb = s_stringBuilderPool.Rent(); + + FormatValue(sb, value, maxLength, truncateBegin); + string result = sb.ToString(); + + s_stringBuilderPool.Return(sb); + return TruncateString(result, maxLength, truncateBegin); + } + + public virtual int FormatValue(StringBuilder sb, object value, int maxLength, bool truncateBegin) + { + int currLength = sb.Length; + sb.Append(value); + TruncateStringBuilder(sb, maxLength, sb.Length - currLength, truncateBegin); + + return sb.Length - currLength; + } + + protected string TruncateString(string result, int maxLength, bool truncateBegin) + { + if (maxLength >= 0 && result.Length > maxLength) + { + if (CanTruncate) + { + if (maxLength <= 3) + { + result = new string('.', maxLength); + } + else if (truncateBegin) + { + result = "..." + result.Substring(result.Length - (maxLength - 3)); + } + else + { + result = result.Substring(0, maxLength - 3) + "..."; + } + } + else + { + Debug.Fail("Tried to truncate a column we should never truncate."); + } + } + + Debug.Assert(maxLength < 0 || result.Length <= maxLength); + return result; + } + + protected void TruncateStringBuilder(StringBuilder result, int maxLength, int lengthWritten, bool truncateBegin) + { + Debug.Assert(lengthWritten >= 0); + + if (maxLength >= 0 && lengthWritten > maxLength) + { + if (CanTruncate) + { + if (truncateBegin) + { + int start = result.Length - lengthWritten; + int wrote; + for (wrote = 0; wrote < 3 && wrote < maxLength; wrote++) + { + result[start + wrote] = '.'; + } + + int gap = lengthWritten - maxLength; + for (; wrote < maxLength; wrote++) + { + result[start + wrote] = result[start + wrote + gap]; + } + + result.Length = start + maxLength; + } + else + { + result.Length = result.Length - lengthWritten + maxLength; + for (int i = 0; i < maxLength && i < 3; i++) + { + result[result.Length - i - 1] = '.'; + } + } + } + else + { + Debug.Fail("Tried to truncate a column we should never truncate."); + } + } + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/Output/Formats.cs b/src/Microsoft.Diagnostics.ExtensionCommands/Output/Formats.cs new file mode 100644 index 0000000000..1fdc488e69 --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/Output/Formats.cs @@ -0,0 +1,255 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Microsoft.Diagnostics.Runtime; + +namespace Microsoft.Diagnostics.ExtensionCommands.Output +{ + internal static class Formats + { + private static HexValueFormat s_hexOffsetFormat; + private static HexValueFormat s_hexValueFormat; + private static Format s_text; + private static IntegerFormat s_integerFormat; + private static TypeOrImageFormat s_typeNameFormat; + private static TypeOrImageFormat s_imageFormat; + private static IntegerFormat s_integerWithoutCommaFormat; + private static HumanReadableFormat s_humanReadableFormat; + private static RangeFormat s_range; + + static Formats() + { + int pointerSize = IntPtr.Size; + Pointer = new IntegerFormat(pointerSize == 4 ? "x8" : "x12"); + } + + public static Format Pointer { get; } + + public static Format HexOffset => s_hexOffsetFormat ??= new(printPrefix: true, signed: true); + public static Format HexValue => s_hexValueFormat ??= new(printPrefix: true, signed: false); + public static Format Integer => s_integerFormat ??= new("n0"); + public static Format IntegerWithoutCommas => s_integerWithoutCommaFormat ??= new(""); + public static Format Text => s_text ??= new(true); + public static Format TypeName => s_typeNameFormat ??= new(type: true); + public static Format Image => s_imageFormat ??= new(type: false); + public static Format HumanReadableSize => s_humanReadableFormat ??= new(); + public static Format Range => s_range ??= new(); + + private sealed class IntegerFormat : Format + { + private readonly string _format; + + public IntegerFormat(string format) + { + _format = "{0:" + format + "}"; + } + + public override int FormatValue(StringBuilder result, object value, int maxLength, bool truncateBegin) + { + value = Unwrap(value); + + int startLength = result.Length; + switch (value) + { + case null: + break; + + case nuint nui: + result.AppendFormat(_format, (ulong)nui); + break; + + case nint ni: + unchecked + { + result.AppendFormat(_format, (ulong)ni); + } + break; + + default: + result.AppendFormat(_format, value); + break; + } + + TruncateStringBuilder(result, maxLength, result.Length - startLength, truncateBegin); + return result.Length - startLength; + } + } + + /// + /// Unlike plain text, this Format always truncates the beginning of the type name or image path, + /// as the most important part is at the end. + /// + private sealed class TypeOrImageFormat : Format + { + private const string UnknownTypeName = "Unknown"; + private readonly bool _type; + + public TypeOrImageFormat(bool type) + : base(canTruncate: true) + { + _type = type; + } + + public override int FormatValue(StringBuilder sb, object value, int maxLength, bool truncateBegin) + { + int startLength = sb.Length; + + if (!_type) + { + sb.Append(value); + } + else + { + if (value is null) + { + sb.Append(UnknownTypeName); + } + else if (value is ClrType type) + { + string typeName = type.Name; + if (!string.IsNullOrWhiteSpace(typeName)) + { + sb.Append(typeName); + } + else + { + string module = type.Module?.Name; + if (!string.IsNullOrWhiteSpace(module)) + { + try + { + module = System.IO.Path.GetFileNameWithoutExtension(module); + sb.Append(module); + sb.Append('!'); + } + catch (ArgumentException) + { + } + } + + sb.Append(UnknownTypeName); + if (type.MethodTable != 0) + { + sb.Append($" (MethodTable: "); + sb.AppendFormat("{0:x12}", type.MethodTable); + sb.Append(')'); + } + } + } + else + { + sb.Append(value); + } + } + + TruncateStringBuilder(sb, maxLength, sb.Length - startLength, truncateBegin: true); + return sb.Length - startLength; + } + } + + private sealed class HumanReadableFormat : Format + { + public override int FormatValue(StringBuilder sb, object value, int maxLength, bool truncateBegin) + { + string humanReadable = value switch + { + null => null, + int i => ((long)i).ConvertToHumanReadable(), + uint ui => ((ulong)ui).ConvertToHumanReadable(), + long l => l.ConvertToHumanReadable(), + ulong ul => ul.ConvertToHumanReadable(), + float f => ((double)f).ConvertToHumanReadable(), + double d => d.ConvertToHumanReadable(), + nuint nu => ((ulong)nu).ConvertToHumanReadable(), + nint ni => ((long)ni).ConvertToHumanReadable(), + string s => s, + _ => throw new NotSupportedException($"Cannot convert '{value.GetType().FullName}' to a human readable size.") + }; + + if (!string.IsNullOrWhiteSpace(humanReadable)) + { + return base.FormatValue(sb, humanReadable, maxLength, truncateBegin); + } + + return 0; + } + } + + private sealed class HexValueFormat : Format + { + public bool PrintPrefix { get; } + public bool Signed { get; } + + public HexValueFormat(bool printPrefix, bool signed) + { + PrintPrefix = printPrefix; + Signed = signed; + } + + private string GetStringValue(long offset) + { + if (Signed) + { + if (PrintPrefix) + { + return offset < 0 ? $"-0x{Math.Abs(offset):x2}" : $"0x{offset:x2}"; + } + else + { + return offset < 0 ? $"-{Math.Abs(offset):x2}" : $"{offset:x2}"; + } + } + + return PrintPrefix ? $"0x{offset:x2}" : offset.ToString("x2"); + } + + private string GetHexOffsetString(object value) + { + return value switch + { + null => "", + string s => s, + nint ni => GetStringValue(ni), + nuint nui => PrintPrefix ? $"0x{nui:x2}" : "{nui:x2}", + ulong ul => PrintPrefix ? $"0x{ul:x2}" : ul.ToString("x2"), + long l => GetStringValue(l), + int i => GetStringValue(i), + uint u => PrintPrefix ? $"0x{u:x2}" : u.ToString("x2"), + IEnumerable bytes => (PrintPrefix ? "0x" : "") + string.Join("", bytes.Select(b => b.ToString("x2"))), + _ => throw new InvalidOperationException($"Cannot convert value of type {value.GetType().FullName} to a HexOffset") + }; + } + + public override int FormatValue(StringBuilder sb, object value, int maxLength, bool truncateBegin) + { + int startLength = sb.Length; + sb.Append(GetHexOffsetString(value)); + TruncateStringBuilder(sb, maxLength, sb.Length - startLength, truncateBegin); + + return sb.Length - startLength; + } + } + + private sealed class RangeFormat : Format + { + public override int FormatValue(StringBuilder sb, object value, int maxLength, bool truncateBegin) + { + int startLength = sb.Length; + if (value is MemoryRange range) + { + sb.AppendFormat("{0:x}", range.Start); + sb.Append('-'); + sb.AppendFormat("{0:x}", range.End); + + return sb.Length - startLength; + } + + return base.FormatValue(sb, value, maxLength, truncateBegin); + } + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/Output/StringBuilderPool.cs b/src/Microsoft.Diagnostics.ExtensionCommands/Output/StringBuilderPool.cs new file mode 100644 index 0000000000..902ab05116 --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/Output/StringBuilderPool.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text; + +namespace Microsoft.Diagnostics.ExtensionCommands.Output +{ + internal sealed class StringBuilderPool + { + private StringBuilder _stringBuilder; + private readonly int _initialCapacity; + + public StringBuilderPool(int initialCapacity = 64) + { + _initialCapacity = initialCapacity > 0 ? initialCapacity : 0; + } + + // This code all assumes SOS runs single threaded. We would want to change this + // code to use Interlocked.Exchange if that ever changes. + public StringBuilder Rent() + { + StringBuilder sb = _stringBuilder; + _stringBuilder = null; + + if (sb is null) + { + sb = new StringBuilder(_initialCapacity); + } + else + { + sb.Clear(); + } + + return sb; + } + + public void Return(StringBuilder sb) + { + if (sb.Capacity < 1024) + { + _stringBuilder = sb; + } + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/Output/Table.cs b/src/Microsoft.Diagnostics.ExtensionCommands/Output/Table.cs new file mode 100644 index 0000000000..71bcddc9dc --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/Output/Table.cs @@ -0,0 +1,240 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics; +using System.Linq; +using System.Text; +using Microsoft.Diagnostics.DebugServices; + +namespace Microsoft.Diagnostics.ExtensionCommands.Output +{ + internal class Table + { + protected readonly StringBuilderPool _stringBuilderPool = new(); + protected string _spacing = " "; + protected static readonly Column s_headerColumn = new(Align.Center, -1, Formats.Text, Dml.Bold); + + public string Indent { get; set; } = ""; + + public IConsoleService Console { get; } + + public int TotalWidth => 1 * (Columns.Length - 1) + Columns.Sum(c => Math.Abs(c.Width)); + + public Column[] Columns { get; set; } + + public Table(IConsoleService console, params Column[] columns) + { + Columns = columns.ToArray(); + Console = console; + } + + public void SetAlignment(Align align) + { + for (int i = 0; i < Columns.Length; i++) + { + Columns[i] = Columns[i].WithAlignment(align); + } + } + + public virtual void WriteHeader(params string[] values) + { + IncreaseColumnWidth(values); + WriteHeaderFooter(values); + } + + public virtual void WriteFooter(params object[] values) + { + WriteHeaderFooter(values); + } + + protected void IncreaseColumnWidth(string[] values) + { + // Increase column width if too small + for (int i = 0; i < Columns.Length && i < values.Length; i++) + { + if (Columns.Length >= 0 && values[i].Length > Columns.Length) + { + if (Columns[i].Width != -1 && Columns[i].Width < values[i].Length) + { + Columns[i] = Columns[i].WithWidth(values[i].Length); + } + } + } + } + + public virtual void WriteRow(params object[] values) + { + StringBuilder rowBuilder = _stringBuilderPool.Rent(); + rowBuilder.Append(Indent); + + WriteRowWorker(values, rowBuilder, _spacing); + + _stringBuilderPool.Return(rowBuilder); + } + + protected void WriteRowWorker(object[] values, StringBuilder rowBuilder, string spacing, bool writeLine = true) + { + bool isRowBuilderDml = false; + + for (int i = 0; i < values.Length; i++) + { + if (i != 0) + { + rowBuilder.Append(spacing); + } + + Column column = i < Columns.Length ? Columns[i] : ColumnKind.Text; + + bool isColumnDml = Console.SupportsDml && column.Dml is not null; + if (isRowBuilderDml != isColumnDml) + { + WriteAndClearRowBuilder(rowBuilder, isRowBuilderDml); + isRowBuilderDml = isColumnDml; + } + + Append(column, rowBuilder, values[i]); + } + + if (writeLine) + { + rowBuilder.AppendLine(); + } + + WriteAndClearRowBuilder(rowBuilder, isRowBuilderDml); + } + + private void WriteAndClearRowBuilder(StringBuilder rowBuilder, bool dml) + { + if (rowBuilder.Length != 0) + { + if (dml) + { + Console.WriteDml(rowBuilder.ToString()); + } + else + { + Console.Write(rowBuilder.ToString()); + } + + rowBuilder.Clear(); + } + } + + private void Append(Column column, StringBuilder sb, object value) + { + DmlFormat dml = null; + if (Console.SupportsDml) + { + dml = column.Dml; + } + + // Efficient case + if (dml is null && column.Alignment == Align.Left) + { + int written = column.Format.FormatValue(sb, value, column.Width, column.Alignment == Align.Left); + Debug.Assert(written >= 0); + if (written < column.Width) + { + sb.Append(' ', column.Width - written); + } + + return; + } + + string toWrite = column.Format.FormatValue(value, column.Width, column.Alignment == Align.Left); + int displayLength = toWrite.Length; + if (dml is not null) + { + toWrite = dml.FormatValue(toWrite, value); + } + + if (column.Width < 0) + { + sb.Append(toWrite); + } + else + { + if (column.Alignment == Align.Left) + { + sb.Append(toWrite); + if (displayLength < column.Width) + { + sb.Append(' ', column.Width - displayLength); + } + + return; + } + else if (column.Alignment == Align.Right) + { + sb.Append(' ', column.Width - displayLength); + sb.Append(toWrite); + } + else + { + Debug.Assert(column.Alignment == Align.Center); + + int remainder = column.Width - displayLength; + int right = remainder >> 1; + int left = right + (remainder % 2); + + sb.Append(' ', left); + sb.Append(toWrite); + sb.Append(' ', right); + } + } + } + + protected virtual void WriteHeaderFooter(object[] values, bool writeSides = false, bool writeNewline = true) + { + StringBuilder rowBuilder = _stringBuilderPool.Rent(); + rowBuilder.Append(Indent); + + if (writeSides) + { + rowBuilder.Append(_spacing); + } + + for (int i = 0; i < values.Length; i++) + { + if (i != 0) + { + rowBuilder.Append(_spacing); + } + + Column curr = i < Columns.Length ? Columns[i] : s_headerColumn; + if (Console.SupportsDml) + { + curr = curr.WithDml(Dml.Bold); + } + else + { + curr = curr.WithDml(null); + } + + Append(curr, rowBuilder, values[i]); + } + + if (writeSides) + { + rowBuilder.Append(_spacing); + } + + if (writeNewline) + { + rowBuilder.AppendLine(); + } + + if (Console.SupportsDml) + { + Console.WriteDml(rowBuilder.ToString()); + } + else + { + Console.Write(rowBuilder.ToString()); + } + + _stringBuilderPool.Return(rowBuilder); + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/PathToCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/PathToCommand.cs index e4c2254fa6..e63940d3be 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/PathToCommand.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/PathToCommand.cs @@ -58,7 +58,7 @@ public override void Invoke() GCRoot.ChainLink path = gcroot.FindPathFrom(sourceObj); if (path is not null) { - GCRootCommand.PrintPath(Console, RootCache, heap, path); + GCRootCommand.PrintPath(Console, RootCache, null, heap, path); } else { diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/SimulateGCHeapCorruption.cs b/src/Microsoft.Diagnostics.ExtensionCommands/SimulateGCHeapCorruption.cs index 2158d8ead6..2105875b9d 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/SimulateGCHeapCorruption.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/SimulateGCHeapCorruption.cs @@ -5,9 +5,10 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.ExtensionCommands.Output; using Microsoft.Diagnostics.Runtime; -using static Microsoft.Diagnostics.ExtensionCommands.TableOutput; +using static Microsoft.Diagnostics.ExtensionCommands.Output.ColumnKind; namespace Microsoft.Diagnostics.ExtensionCommands { @@ -59,12 +60,12 @@ private void List() } else { - TableOutput output = new(Console, (12, "x12"), (12, "x12"), (16, "x"), (16, "x"), (0, "")); - output.WriteRow("Object", "ModifiedAddr", "Old Value", "New Value", "Expected Failure"); + Table output = new(Console, DumpObj, Pointer, HexValue, HexValue, Text); + output.WriteHeader("Object", "ModifiedAddr", "Old Value", "New Value", "Expected Failure"); foreach (Change change in _changes) { - output.WriteRow(new DmlDumpObj(change.Object), change.AddressModified, change.OriginalValue.Reverse(), change.NewValue.Reverse(), change.ExpectedFailure); + output.WriteRow(change.Object, change.AddressModified, change.OriginalValue.Reverse(), change.NewValue.Reverse(), change.ExpectedFailure); } } @@ -120,7 +121,7 @@ private void Corrupt() ClrObject[] withRefs = FindObjectsWithReferences().Take(3).ToArray(); if (withRefs.Length >= 1) { - (ulong Object, ulong FirstReference) entry = GetFirstReference(withRefs[0]); + (ClrObject Object, ulong FirstReference) entry = GetFirstReference(withRefs[0]); WriteValue(ObjectCorruptionKind.InvalidObjectReference, entry.Object, entry.FirstReference, 0xcccccccc); } if (withRefs.Length >= 2) @@ -128,13 +129,13 @@ private void Corrupt() ulong free = Runtime.Heap.EnumerateObjects().FirstOrDefault(f => f.IsFree); if (free != 0) { - (ulong Object, ulong FirstReference) entry = GetFirstReference(withRefs[1]); + (ClrObject Object, ulong FirstReference) entry = GetFirstReference(withRefs[1]); WriteValue(ObjectCorruptionKind.FreeObjectReference, entry.Object, entry.FirstReference, free); } } if (withRefs.Length >= 3) { - (ulong Object, ulong FirstReference) entry = GetFirstReference(withRefs[2]); + (ClrObject Object, ulong FirstReference) entry = GetFirstReference(withRefs[2]); WriteValue(ObjectCorruptionKind.ObjectReferenceNotPointerAligned, entry.Object, entry.FirstReference, (byte)1); } @@ -152,7 +153,7 @@ private void Corrupt() List(); } - private static (ulong Object, ulong FirstReference) GetFirstReference(ClrObject obj) + private static (ClrObject Object, ulong FirstReference) GetFirstReference(ClrObject obj) { return (obj, obj.EnumerateReferenceAddresses().First()); } @@ -217,7 +218,7 @@ private IEnumerable FindArrayObjects() } } - private unsafe void WriteValue(ObjectCorruptionKind kind, ulong obj, ulong address, T value) + private unsafe void WriteValue(ObjectCorruptionKind kind, ClrObject obj, ulong address, T value) where T : unmanaged { byte[] old = new byte[sizeof(T)]; @@ -246,7 +247,7 @@ private unsafe void WriteValue(ObjectCorruptionKind kind, ulong obj, ulong ad private sealed class Change { - public ulong Object { get; set; } + public ClrObject Object { get; set; } public ulong AddressModified { get; set; } public byte[] OriginalValue { get; set; } public byte[] NewValue { get; set; } diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/SizeStatsCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/SizeStatsCommand.cs index 5208edaa48..b8f67a7a5f 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/SizeStatsCommand.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/SizeStatsCommand.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.ExtensionCommands.Output; using Microsoft.Diagnostics.Runtime; namespace Microsoft.Diagnostics.ExtensionCommands @@ -78,23 +79,32 @@ private void SizeStats(Generation requestedGen, bool isFree) Console.WriteLine($"Size Statistics for {requestedGen.ToString().ToLowerInvariant()} {freeStr}objects"); Console.WriteLine(); - TableOutput output = new(Console, (16, "n0"), (16, "n0"), (16, "n0"), (16, "n0")); - output.WriteRow("Size", "Count", "Cumulative Size", "Cumulative Count"); - - IEnumerable<(ulong Size, ulong Count)> sorted = from i in stats orderby i.Key ascending select (i.Key, i.Value); ulong cumulativeSize = 0; ulong cumulativeCount = 0; + Table output = null; foreach ((ulong size, ulong count) in sorted) { Console.CancellationToken.ThrowIfCancellationRequested(); + if (output is null) + { + output = new(Console, ColumnKind.ByteCount, ColumnKind.Integer, ColumnKind.Integer, ColumnKind.Integer); + output.WriteHeader("Size", "Count", "Cumulative Size", "Cumulative Count"); + } + + output.WriteRow(size, count, cumulativeSize, cumulativeCount); + cumulativeSize += size * count; cumulativeCount += count; - output.WriteRow(size, count, cumulativeSize, cumulativeCount); + } + + if (output is null) + { + Console.WriteLine("(none)"); } Console.WriteLine(); diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/StaticVariableService.cs b/src/Microsoft.Diagnostics.ExtensionCommands/StaticVariableService.cs new file mode 100644 index 0000000000..a9e03d4ecc --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/StaticVariableService.cs @@ -0,0 +1,99 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.Runtime; + +namespace Microsoft.Diagnostics.ExtensionCommands +{ + [ServiceExport(Scope = ServiceScope.Runtime)] + public class StaticVariableService + { + private Dictionary _fields; + private IEnumerator<(ulong Address, ClrStaticField Static)> _enumerator; + + [ServiceImport] + public ClrRuntime Runtime { get; set; } + + /// + /// Returns the static field at the given address. + /// + /// The address of the static field. Note that this is not a pointer to + /// an object, but rather a pointer to where the CLR runtime tracks the static variable's + /// location. In all versions of the runtime, address will live in the middle of a pinned + /// object[]. + /// The field corresponding to the given address. Non-null if return + /// is true. + /// True if the address corresponded to a static variable, false otherwise. + public bool TryGetStaticByAddress(ulong address, out ClrStaticField field) + { + if (_fields is null) + { + _fields = new(); + _enumerator = EnumerateStatics().GetEnumerator(); + } + + if (_fields.TryGetValue(address, out field)) + { + return true; + } + + // pay for play lookup + if (_enumerator is not null) + { + do + { + _fields[_enumerator.Current.Address] = _enumerator.Current.Static; + if (_enumerator.Current.Address == address) + { + field = _enumerator.Current.Static; + return true; + } + } while (_enumerator.MoveNext()); + + _enumerator = null; + } + + return false; + } + + public IEnumerable<(ulong Address, ClrStaticField Static)> EnumerateStatics() + { + ClrAppDomain shared = Runtime.SharedDomain; + + foreach (ClrModule module in Runtime.EnumerateModules()) + { + foreach ((ulong mt, _) in module.EnumerateTypeDefToMethodTableMap()) + { + ClrType type = Runtime.GetTypeByMethodTable(mt); + if (type is null) + { + continue; + } + + foreach (ClrStaticField stat in type.StaticFields) + { + foreach (ClrAppDomain domain in Runtime.AppDomains) + { + ulong address = stat.GetAddress(domain); + if (address != 0) + { + yield return (address, stat); + } + } + + if (shared is not null) + { + ulong address = stat.GetAddress(shared); + if (address != 0) + { + yield return (address, stat); + } + } + } + } + } + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/TableOutput.cs b/src/Microsoft.Diagnostics.ExtensionCommands/TableOutput.cs deleted file mode 100644 index c6b91c65ad..0000000000 --- a/src/Microsoft.Diagnostics.ExtensionCommands/TableOutput.cs +++ /dev/null @@ -1,294 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using Microsoft.Diagnostics.DebugServices; -using Microsoft.Diagnostics.Runtime; - -namespace Microsoft.Diagnostics.ExtensionCommands -{ - internal sealed class TableOutput - { - private readonly StringBuilder _rowBuilder = new(260); - private readonly char _spacing = ' '; - - public string Divider { get; set; } = " "; - - public string Indent { get; set; } = ""; - - public bool AlignLeft { get; set; } - - public int ColumnCount => _formats.Length; - - public IConsoleService Console { get; } - - public int TotalWidth => 1 * (_formats.Length - 1) + _formats.Sum(c => Math.Abs(c.width)); - - private readonly (int width, string format)[] _formats; - - public TableOutput(IConsoleService console, params (int width, string format)[] columns) - { - _formats = columns.ToArray(); - Console = console; - } - - public void WriteSpacer(char spacer) - { - Console.WriteLine(new string(spacer, Divider.Length * (_formats.Length - 1) + _formats.Sum(c => Math.Abs(c.width)))); - } - - public void WriteRow(params object[] columns) - { - _rowBuilder.Clear(); - _rowBuilder.Append(Indent); - - for (int i = 0; i < columns.Length; i++) - { - if (i != 0) - { - _rowBuilder.Append(_spacing); - } - - (int width, string format) = i < _formats.Length ? _formats[i] : default; - FormatColumn(_spacing, columns[i], _rowBuilder, width, format); - } - - Console.WriteLine(_rowBuilder.ToString()); - } - - public void WriteRowWithSpacing(char spacing, params object[] columns) - { - _rowBuilder.Clear(); - _rowBuilder.Append(Indent); - - for (int i = 0; i < columns.Length; i++) - { - if (i != 0) - { - _rowBuilder.Append(spacing, Divider.Length); - } - - (int width, string format) = i < _formats.Length ? _formats[i] : default; - - FormatColumn(spacing, columns[i], _rowBuilder, width, format); - } - - Console.WriteLine(_rowBuilder.ToString()); - } - - private void FormatColumn(char spacing, object value, StringBuilder sb, int width, string format) - { - string action = null; - string text; - if (value is DmlExec dml) - { - value = dml.Text; - if (Console.SupportsDml) - { - action = dml.Action; - } - } - - if (string.IsNullOrWhiteSpace(format)) - { - text = value?.ToString(); - } - else - { - text = Format(value, format); - } - - AddValue(spacing, sb, width, text ?? "", action); - } - - private void AddValue(char spacing, StringBuilder sb, int width, string value, string action) - { - bool leftAlign = AlignLeft ? width > 0 : width < 0; - width = Math.Abs(width); - - if (width == 0) - { - if (string.IsNullOrWhiteSpace(action)) - { - sb.Append(value); - } - else - { - WriteAndClear(sb); - Console.WriteDmlExec(value, action); - } - } - else if (value.Length > width) - { - if (!string.IsNullOrWhiteSpace(action)) - { - WriteAndClear(sb); - } - - if (width <= 3) - { - sb.Append(value, 0, width); - } - else if (leftAlign) - { - value = value.Substring(0, width - 3); - sb.Append(value); - sb.Append("..."); - } - else - { - value = value.Substring(value.Length - (width - 3)); - sb.Append("..."); - sb.Append(value); - } - - if (!string.IsNullOrWhiteSpace(action)) - { - WriteDmlExecAndClear(sb, action); - } - } - else if (leftAlign) - { - if (!string.IsNullOrWhiteSpace(action)) - { - WriteAndClear(sb); - Console.WriteDmlExec(value, action); - } - else - { - sb.Append(value); - } - - int remaining = width - value.Length; - if (remaining > 0) - { - sb.Append(spacing, remaining); - } - } - else - { - int remaining = width - value.Length; - if (remaining > 0) - { - sb.Append(spacing, remaining); - } - - if (!string.IsNullOrWhiteSpace(action)) - { - WriteAndClear(sb); - Console.WriteDmlExec(value, action); - } - else - { - sb.Append(value); - } - } - } - - private void WriteDmlExecAndClear(StringBuilder sb, string action) - { - Console.WriteDmlExec(sb.ToString(), action); - sb.Clear(); - } - - private void WriteAndClear(StringBuilder sb) - { - Console.Write(sb.ToString()); - sb.Clear(); - } - - private static string Format(object obj, string format) - { - if (obj is null) - { - return null; - } - - if (obj is Enum) - { - return obj.ToString(); - } - - return obj switch - { - nint ni => ni.ToString(format), - ulong ul => ul.ToString(format), - long l => l.ToString(format), - uint ui => ui.ToString(format), - int i => i.ToString(format), - StringBuilder sb => sb.ToString(), - IEnumerable bytes => string.Join("", bytes.Select(b => b.ToString("x2"))), - string s => s, - _ => throw new NotImplementedException(obj.GetType().ToString()), - }; - } - - public class DmlExec - { - public object Text { get; } - public string Action { get; } - - public DmlExec(object text, string action) - { - Text = text; - Action = action; - } - } - - public sealed class DmlDumpObj : DmlExec - { - public DmlDumpObj(ulong address) - : base(address, address != 0 ? $"!dumpobj /d {address:x}" : "") - { - } - } - - public sealed class DmlListNearObj : DmlExec - { - public DmlListNearObj(ulong address) - : base(address, address != 0 ? $"!sos listnearobj {address:x}" : "") - { - } - } - - public sealed class DmlVerifyObj : DmlExec - { - public DmlVerifyObj(ulong address) - : base(address, address != 0 ? $"!verifyobj /d {address:x}" : "") - { - } - } - - public sealed class DmlDumpHeap : DmlExec - { - public DmlDumpHeap(string text, MemoryRange range) - : base(text, $"!dumpheap {range.Start:x} {range.End:x}") - { - } - - public DmlDumpHeap(ulong methodTable) - : base (methodTable, methodTable != 0 ? $"!dumpheap -mt {methodTable:x}" : "") - { - } - } - - public sealed class DmlVerifyHeap : DmlExec - { - public DmlVerifyHeap(string text, ClrSegment what) - : base(text, $"!verifyheap -segment {what.Address}") - { - } - } - - public sealed class DmlDumpHeapSegment : DmlExec - { - public DmlDumpHeapSegment(ClrSegment seg) - : base(seg?.Address ?? 0, seg != null ? $"!dumpheap -segment {seg.Address:x}" : "") - { - } - } - } -} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/ThreadPoolCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/ThreadPoolCommand.cs new file mode 100644 index 0000000000..85f7c4d973 --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/ThreadPoolCommand.cs @@ -0,0 +1,248 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.ExtensionCommands.Output; +using Microsoft.Diagnostics.Runtime; +using static Microsoft.Diagnostics.ExtensionCommands.Output.ColumnKind; + +namespace Microsoft.Diagnostics.ExtensionCommands +{ + [Command(Name = "threadpool", Help = "Displays info about the runtime thread pool.")] + public sealed class ThreadPoolCommand : CommandBase + { + [ServiceImport] + public ClrRuntime Runtime { get; set; } + + [Option(Name = "-ti", Help = "Print the hill climbing log.", Aliases = new string[] { "-hc" })] + public bool PrintHillClimbingLog { get; set; } + + [Option(Name = "-wi", Help = "Print all work items that are queued.")] + public bool PrintWorkItems { get; set; } + + public override void Invoke() + { + // Runtime.ThreadPool shouldn't be null unless there was a problem with the dump. + ClrThreadPool threadPool = Runtime.ThreadPool; + if (threadPool is null) + { + Console.WriteLineError("Failed to obtain ThreadPool data."); + } + else + { + Table output = new(Console, Text.WithWidth(17), Text); + output.WriteRow("CPU utilization:", $"{threadPool.CpuUtilization}%"); + output.WriteRow("Workers Total:", threadPool.ActiveWorkerThreads + threadPool.IdleWorkerThreads + threadPool.RetiredWorkerThreads); + output.WriteRow("Workers Running:", threadPool.ActiveWorkerThreads); + output.WriteRow("Workers Idle:", threadPool.IdleWorkerThreads); + output.WriteRow("Worker Min Limit:", threadPool.MinThreads); + output.WriteRow("Worker Max Limit:", threadPool.MaxThreads); + Console.WriteLine(); + + ClrType threadPoolType = Runtime.BaseClassLibrary.GetTypeByName("System.Threading.ThreadPool"); + ClrStaticField usePortableIOField = threadPoolType?.GetStaticFieldByName("UsePortableThreadPoolForIO"); + + // Desktop CLR work items. + if (PrintWorkItems) + { + LegacyThreadPoolWorkRequest[] requests = threadPool.EnumerateLegacyWorkRequests().ToArray(); + if (requests.Length > 0) + { + Console.WriteLine($"Work Request in Queue: {requests.Length:n0}"); + foreach (LegacyThreadPoolWorkRequest request in requests) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + if (request.IsAsyncTimerCallback) + { + Console.WriteLine($" AsyncTimerCallbackCompletion TimerInfo@{request.Context:x}"); + } + else + { + Console.WriteLine($" Unknown Function: {request.Function:x} Context: {request.Context:x}"); + } + } + } + } + + // We will assume that if UsePortableThreadPoolForIO field is deleted from ThreadPool then we are always + // using C# version. + bool usingPortableCompletionPorts = threadPool.Portable && (usePortableIOField is null || usePortableIOField.Read(usePortableIOField.Type.Module.AppDomain)); + if (!usingPortableCompletionPorts) + { + output.Columns[0] = output.Columns[0].WithWidth(19); + output.WriteRow("Completion Total:", threadPool.TotalCompletionPorts); + output.WriteRow("Completion Free:", threadPool.FreeCompletionPorts); + output.WriteRow("Completion MaxFree:", threadPool.MaxFreeCompletionPorts); + + output.Columns[0] = output.Columns[0].WithWidth(25); + output.WriteRow("Completion Current Limit:", threadPool.CompletionPortCurrentLimit); + output.WriteRow("Completion Min Limit:", threadPool.MinCompletionPorts); + output.WriteRow("Completion Max Limit:", threadPool.MaxCompletionPorts); + Console.WriteLine(); + } + + if (PrintHillClimbingLog) + { + HillClimbingLogEntry[] hcl = threadPool.EnumerateHillClimbingLog().ToArray(); + if (hcl.Length > 0) + { + output = new(Console, Text.WithWidth(10).WithAlignment(Align.Right), Column.ForEnum(), Integer, Integer, Text.WithAlignment(Align.Right)); + + Console.WriteLine("Hill Climbing Log:"); + output.WriteHeader("Time", "Transition", "#New Threads", "#Samples", "Throughput"); + + int end = hcl.Last().TickCount; + foreach (HillClimbingLogEntry entry in hcl) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + output.WriteRow($"{(entry.TickCount - end)/1000.0:0.00}", entry.StateOrTransition, entry.NewThreadCount, entry.SampleCount, $"{entry.Throughput:0.00}"); + } + + Console.WriteLine(); + } + } + } + + // We can print managed work items even if we failed to request the ThreadPool. + if (PrintWorkItems && (threadPool is null || threadPool.Portable)) + { + DumpWorkItems(); + } + } + + private void DumpWorkItems() + { + Table output = null; + + ClrType workQueueType = Runtime.BaseClassLibrary.GetTypeByName("System.Threading.ThreadPoolWorkQueue"); + ClrType workStealingQueueType = Runtime.BaseClassLibrary.GetTypeByName("System.Threading.ThreadPoolWorkQueue+WorkStealingQueue"); + + foreach (ClrObject obj in Runtime.Heap.EnumerateObjects()) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + if (obj.Type == workQueueType) + { + if (obj.TryReadObjectField("highPriorityWorkItems", out ClrObject workItems)) + { + foreach (ClrObject entry in EnumerateConcurrentQueue(workItems)) + { + WriteEntry(ref output, entry, isHighPri: true); + } + } + + if (obj.TryReadObjectField("workItems", out workItems)) + { + foreach (ClrObject entry in EnumerateConcurrentQueue(workItems)) + { + WriteEntry(ref output, entry, isHighPri: false); + } + } + + if (obj.Type.Fields.Any(r => r.Name == "_assignableWorkItems")) + { + if (obj.TryReadObjectField("_assignableWorkItems", out workItems)) + { + foreach (ClrObject entry in EnumerateConcurrentQueue(workItems)) + { + WriteEntry(ref output, entry, isHighPri: false); + } + } + } + } + else if (obj.Type == workStealingQueueType) + { + if (obj.TryReadObjectField("m_array", out ClrObject m_array) && m_array.IsValid && !m_array.IsNull) + { + ClrArray arrayView = m_array.AsArray(); + int len = Math.Min(8192, arrayView.Length); // ensure a sensible max in case we have heap corruption + + nuint[] buffer = arrayView.ReadValues(0, len); + if (buffer != null) + { + for (int i = 0; i < len; i++) + { + if (buffer[i] != 0) + { + ClrObject entry = Runtime.Heap.GetObject(buffer[i]); + if (entry.IsValid && !entry.IsNull) + { + WriteEntry(ref output, entry, isHighPri: false); + } + } + } + } + } + } + } + } + + private void WriteEntry(ref Table output, ClrObject entry, bool isHighPri) + { + if (output is null) + { + output = new(Console, Text.WithWidth(17), DumpObj, TypeName); + output.SetAlignment(Align.Left); + output.WriteHeader("Queue", "Object", "Type"); + } + + output.WriteRow(isHighPri ? "[Global high-pri]" : "[Global]", entry, entry.Type); + if (entry.IsDelegate) + { + ClrDelegate del = entry.AsDelegate(); + ClrDelegateTarget target = del.GetDelegateTarget(); + if (target is not null) + { + Console.WriteLine($" => {target.TargetObject.Address:x} {target.Method.Name}"); + } + } + } + + private IEnumerable EnumerateConcurrentQueue(ClrObject concurrentQueue) + { + if (!concurrentQueue.IsValid || concurrentQueue.IsNull) + { + yield break; + } + + if (concurrentQueue.TryReadObjectField("_head", out ClrObject curr)) + { + while (curr.IsValid && !curr.IsNull) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + if (curr.TryReadObjectField("_slots", out ClrObject slots) && slots.IsValid && slots.IsArray) + { + ClrArray slotsArray = slots.AsArray(); + for (int i = 0; i < slotsArray.Length; i++) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + ClrObject item = slotsArray.GetStructValue(i).ReadObjectField("Item"); + if (item.IsValid && !item.IsNull) + { + yield return item; + } + } + } + + if (!curr.TryReadObjectField("_nextSegment", out ClrObject next)) + { + if (curr.Type is not null && curr.Type.GetFieldByName("_nextSegment") == null) + { + Console.WriteLineError($"Error: Type '{curr.Type?.Name}' does not contain a '_nextSegment' field."); + } + + break; + } + + curr = next; + } + } + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/TraverseHeapCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/TraverseHeapCommand.cs new file mode 100644 index 0000000000..40d2820502 --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/TraverseHeapCommand.cs @@ -0,0 +1,164 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Xml; +using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.Runtime; + +namespace Microsoft.Diagnostics.ExtensionCommands +{ + [Command(Name = "traverseheap", Help = "Writes out heap information to a file in a format understood by the CLR Profiler.")] + public class TraverseHeapCommand : CommandBase + { + [ServiceImport] + public ClrRuntime Runtime { get; set; } + + [ServiceImport] + public RootCacheService RootCache { get; set; } + + [Option(Name = "-xml")] + public bool Xml { get; set; } + + [Argument(Name = "filename")] + public string Filename { get; set; } + + public override void Invoke() + { + if (string.IsNullOrWhiteSpace(Filename)) + { + throw new ArgumentException($"Output filename cannot be empty.", nameof(Filename)); + } + + // create file early in case it throws + using StreamWriter output = File.CreateText(Filename); + using (XmlWriter xml = Xml ? XmlWriter.Create(output, new XmlWriterSettings() { Encoding = new UTF8Encoding(true), Indent = true, OmitXmlDeclaration = true }) : null) + { + using StreamWriter text = Xml ? null : output; + + // must be called first to initialize types + (MemoryStream rootObjectStream, Dictionary types) = WriteRootsAndObjects(); + + xml?.WriteStartElement("gcheap"); + xml?.WriteStartElement("types"); + + foreach (KeyValuePair kv in types.OrderBy(kv => kv.Value)) + { + string name = kv.Key?.Name ?? $"error-reading-type-name:{kv.Key.MethodTable:x}"; + int typeId = kv.Value; + + xml?.WriteStartElement("type"); + xml?.WriteAttributeString("id", typeId.ToString()); + xml?.WriteAttributeString("name", name); + xml?.WriteEndElement(); + + text?.WriteLine($"t {typeId} 0 {name}"); + } + xml?.WriteEndElement(); + + xml?.Flush(); + text?.Flush(); + + output.WriteLine(); + output.Flush(); + + rootObjectStream.Position = 0; + rootObjectStream.CopyTo(output.BaseStream); + + xml?.WriteEndElement(); + } + } + + private (MemoryStream Stream, Dictionary Types) WriteRootsAndObjects() + { + Dictionary types = new(); + MemoryStream rootObjectStream = new(); + + using StreamWriter text = Xml ? null : new StreamWriter(rootObjectStream, Encoding.Default, 4096, leaveOpen: true); + using XmlWriter xml = Xml ? XmlWriter.Create(rootObjectStream, new XmlWriterSettings() + { + Encoding = new UTF8Encoding(false), + CloseOutput = false, + Indent = true, + OmitXmlDeclaration = true, + ConformanceLevel = ConformanceLevel.Fragment + }) : null; + + int currObj = 1; + int currType = 1; + + xml?.WriteStartElement("roots"); + text?.Write("r"); + foreach (ClrRoot root in RootCache.EnumerateRoots()) + { + string kind = root switch + { + ClrStackRoot => "stack", + ClrHandle => "handle", + _ => "finalizer" + }; + + xml?.WriteStartElement("root"); + xml?.WriteAttributeString("kind", kind); + xml?.WriteAttributeString("address", FormatHex(root.Address)); + xml?.WriteEndElement(); + + text?.Write(" "); + text?.Write(FormatHex(root.Address)); + } + xml?.WriteEndElement(); + text?.WriteLine(); + + xml?.WriteStartElement("objects"); + foreach (ClrObject obj in Runtime.Heap.EnumerateObjects()) + { + if (!obj.IsValid) + { + continue; + } + + ulong size = obj.Size; + int objId = currObj++; + if (!types.TryGetValue(obj.Type, out int typeId)) + { + typeId = types[obj.Type] = currType++; + } + + xml?.WriteStartElement("object"); + xml?.WriteAttributeString("address", FormatHex(obj.Address)); + xml?.WriteAttributeString("typeid", typeId.ToString()); + xml?.WriteAttributeString("size", size.ToString()); + + text?.WriteLine($"n {objId} 1 {typeId} {size}"); + text?.WriteLine($"! 1 {FormatHex(obj.Address)} {objId}"); + + text?.Write($"o {FormatHex(obj.Address)} {typeId} {size} "); // trailing space intentional + + if (obj.ContainsPointers) + { + foreach (ClrObject objRef in obj.EnumerateReferences(considerDependantHandles: true)) + { + xml?.WriteStartElement("member"); + xml?.WriteAttributeString("address", FormatHex(objRef.Address)); + xml?.WriteEndElement(); + + text?.Write($" "); + text?.Write(FormatHex(objRef.Address)); + } + } + + text?.WriteLine(); + xml?.WriteEndElement(); + } + xml?.WriteEndElement(); + + return (rootObjectStream, types); + } + + private static string FormatHex(ulong address) => $"0x{address:x16}"; + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/VerifyHeapCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/VerifyHeapCommand.cs index f204d69124..e5229cbf32 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/VerifyHeapCommand.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/VerifyHeapCommand.cs @@ -1,12 +1,11 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; using System.Collections.Generic; using System.Diagnostics; -using System.Linq; using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.ExtensionCommands.Output; using Microsoft.Diagnostics.Runtime; -using static Microsoft.Diagnostics.ExtensionCommands.TableOutput; +using static Microsoft.Diagnostics.ExtensionCommands.Output.ColumnKind; namespace Microsoft.Diagnostics.ExtensionCommands { @@ -69,7 +68,7 @@ private void VerifyHeap(IEnumerable objects, bool verifySyncTable) objects = EnumerateWithCount(objects); int errors = 0; - TableOutput output = null; + Table output = null; ClrHeap heap = Runtime.Heap; // Verify heap @@ -128,7 +127,7 @@ private void VerifyHeap(IEnumerable objects, bool verifySyncTable) } } - private void WriteError(ref TableOutput output, ClrHeap heap, ObjectCorruption corruption) + private void WriteError(ref Table output, ClrHeap heap, ObjectCorruption corruption) { string message = GetObjectCorruptionMessage(MemoryService, heap, corruption); WriteRow(ref output, heap, corruption, message); @@ -145,14 +144,14 @@ internal static string GetObjectCorruptionMessage(IMemoryService memory, ClrHeap ObjectCorruptionKind.ObjectNotPointerAligned => $"Object {obj.Address:x} is not pointer aligned", // Object failures - ObjectCorruptionKind.ObjectTooLarge => $"Object {obj.Address:x} is too large, size={obj.Size:x}, segmentEnd: {ValueWithError(heap.GetSegmentByAddress(obj)?.End)}", + ObjectCorruptionKind.ObjectTooLarge => $"Object {obj.Address:x} is too large, size={obj.Size:x}, segmentEnd: {heap.GetSegmentByAddress(obj)?.End.ToString("x") ?? "???"}", ObjectCorruptionKind.InvalidMethodTable => $"Object {obj.Address:x} has an invalid method table {ReadPointerWithError(memory, obj):x}", ObjectCorruptionKind.InvalidThinlock => $"Object {obj.Address:x} has an invalid thin lock", ObjectCorruptionKind.SyncBlockMismatch => GetSyncBlockFailureMessage(corruption), ObjectCorruptionKind.SyncBlockZero => GetSyncBlockFailureMessage(corruption), // Object reference failures - ObjectCorruptionKind.ObjectReferenceNotPointerAligned => $"Object {obj.Address:x} has an unaligned member at {corruption.Offset:x}: is not pointer aligned", + ObjectCorruptionKind.ObjectReferenceNotPointerAligned => $"Object {obj.Address:x} has an unaligned member at offset {corruption.Offset:x}: is not pointer aligned", ObjectCorruptionKind.InvalidObjectReference => $"Object {obj.Address:x} has a bad member at offset {corruption.Offset:x}: {ReadPointerWithError(memory, obj + (uint)corruption.Offset)}", ObjectCorruptionKind.FreeObjectReference => $"Object {obj.Address:x} contains free object at offset {corruption.Offset:x}: {ReadPointerWithError(memory, obj + (uint)corruption.Offset)}", @@ -167,42 +166,37 @@ internal static string GetObjectCorruptionMessage(IMemoryService memory, ClrHeap return message; } - private void WriteRow(ref TableOutput output, ClrHeap heap, ObjectCorruption corruption, string message) + private void WriteRow(ref Table output, ClrHeap heap, ObjectCorruption corruption, string message) { if (output is null) { if (heap.IsServer) { - output = new(Console, (-4, ""), (-12, "x12"), (-12, "x12"), (32, ""), (0, "")) - { - AlignLeft = true, - }; + output = new(Console, IntegerWithoutCommas.WithWidth(4), Pointer, ListNearObj, Column.ForEnum(), Text); + output.SetAlignment(Align.Left); - output.WriteRow("Heap", "Segment", "Object", "Failure", ""); + output.WriteHeader("Heap", "Segment", "Object", "Failure", "Reason"); } else { - output = new(Console, (-12, "x12"), (-12, "x12"), (22, ""), (0, "")) - { - AlignLeft = true, - }; + output = new(Console, Pointer, ListNearObj, Column.ForEnum(), Text); + output.SetAlignment(Align.Left); - output.WriteRow("Segment", "Object", "Failure", ""); + output.WriteHeader("Segment", "Object", "Failure", "Reason"); } } - ClrSegment segment = heap.GetSegmentByAddress(corruption.Object); - object[] columns = new object[output.ColumnCount]; + object[] columns = new object[output.Columns.Length]; int i = 0; if (heap.IsServer) { - columns[i++] = ValueWithError(segment?.SubHeap.Index, format: "", error: ""); + columns[i++] = (object)segment?.SubHeap.Index ?? "???"; } - columns[i++] = ValueWithError(segment?.Address, format: "x12", error: ""); - columns[i++] = new DmlExec(corruption.Object.Address, $"!ListNearObj {corruption.Object.Address:x}"); + columns[i++] = (object)segment ?? "???"; + columns[i++] = corruption.Object; columns[i++] = corruption.Kind; columns[i++] = message; @@ -244,26 +238,6 @@ private static string GetSyncBlockFailureMessage(ObjectCorruption corruption) return result; } - private static string ValueWithError(int? value, string format = "x", string error = "???") - { - if (value.HasValue) - { - return value.Value.ToString(format); - } - - return error; - } - - private static string ValueWithError(ulong? value, string format = "x", string error = "???") - { - if (value.HasValue) - { - return value.Value.ToString(format); - } - - return error; - } - private static string ReadPointerWithError(IMemoryService memory, ulong address) { if (memory.ReadPointer(address, out ulong value)) diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/VerifyObjectCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/VerifyObjectCommand.cs new file mode 100644 index 0000000000..f952aca361 --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/VerifyObjectCommand.cs @@ -0,0 +1,56 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.ExtensionCommands.Output; +using Microsoft.Diagnostics.Runtime; +using static Microsoft.Diagnostics.ExtensionCommands.Output.ColumnKind; + +namespace Microsoft.Diagnostics.ExtensionCommands +{ + [Command(Name = "verifyobj", Help = "Checks the given object for signs of corruption.")] + public sealed class VerifyObjectCommand : CommandBase + { + [ServiceImport] + public ClrRuntime Runtime { get; set; } + + [ServiceImport] + public IMemoryService Memory { get; set; } + + [Argument(Name = "ObjectAddress", Help = "The object to verify.")] + public string ObjectAddress { get; set; } + + public override void Invoke() + { + if (!TryParseAddress(ObjectAddress, out ulong objAddress)) + { + throw new ArgumentException($"Invalid object address: '{ObjectAddress}'", nameof(ObjectAddress)); + } + + bool isNotCorrupt = Runtime.Heap.FullyVerifyObject(objAddress, out IEnumerable corruptionEnum); + if (isNotCorrupt) + { + Console.WriteLine($"object 0x{objAddress:x} is a valid object"); + return; + } + + ObjectCorruption[] corruption = corruptionEnum.OrderBy(r => r.Offset).ToArray(); + + Column offsetColumn = HexOffset.WithAlignment(Align.Left); + offsetColumn = offsetColumn.GetAppropriateWidth(corruption.Select(r => r.Offset)); + + Table output = new(Console, offsetColumn, Column.ForEnum(), Text); + output.WriteHeader("Offset", "Issue", "Description"); + foreach (ObjectCorruption oc in corruption) + { + output.WriteRow(oc.Offset, oc.Kind, VerifyHeapCommand.GetObjectCorruptionMessage(Memory, Runtime.Heap, oc)); + } + + Console.WriteLine(); + Console.WriteLine($"{corruption.Length:n0} error{(corruption.Length == 1 ? "" : "s")} detected."); + } + } +} diff --git a/src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/CounterPayload.cs b/src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/CounterPayload.cs index 18eb5264e3..99e3030081 100644 --- a/src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/CounterPayload.cs +++ b/src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/CounterPayload.cs @@ -86,6 +86,17 @@ public GaugePayload(string providerName, string name, string displayName, string } } + internal class UpDownCounterPayload : CounterPayload + { + public UpDownCounterPayload(string providerName, string name, string displayName, string displayUnits, string metadata, double value, DateTime timestamp) : + base(providerName, name, metadata, value, timestamp, "Metric", EventType.UpDownCounter) + { + // In case these properties are not provided, set them to appropriate values. + string counterName = string.IsNullOrEmpty(displayName) ? name : displayName; + DisplayName = !string.IsNullOrEmpty(displayUnits) ? $"{counterName} ({displayUnits})" : counterName; + } + } + internal class CounterEndedPayload : CounterPayload { public CounterEndedPayload(string providerName, string name, DateTime timestamp) @@ -144,6 +155,7 @@ internal enum EventType : int Rate, Gauge, Histogram, + UpDownCounter, Error, CounterEnded } diff --git a/src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/TraceEventExtensions.cs b/src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/TraceEventExtensions.cs index eedc106d18..347a4e11f2 100644 --- a/src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/TraceEventExtensions.cs +++ b/src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/TraceEventExtensions.cs @@ -90,6 +90,10 @@ public static bool TryGetCounterPayload(this TraceEvent traceEvent, CounterFilte { HandleCounterRate(traceEvent, filter, sessionId, out payload); } + else if (traceEvent.EventName == "UpDownCounterRateValuePublished") + { + HandleUpDownCounterValue(traceEvent, filter, sessionId, out payload); + } else if (traceEvent.EventName == "TimeSeriesLimitReached") { HandleTimeSeriesLimitReached(traceEvent, sessionId, out payload); @@ -183,7 +187,47 @@ private static void HandleCounterRate(TraceEvent traceEvent, CounterFilter filte else { // for observable instruments we assume the lack of data is meaningful and remove it from the UI - // this happens when the ObservableCounter callback function throws an exception. + // this happens when the ObservableCounter callback function throws an exception + // or when the ObservableCounter doesn't include a measurement for a particular set of tag values. + payload = new CounterEndedPayload(meterName, instrumentName, traceEvent.TimeStamp); + } + } + + private static void HandleUpDownCounterValue(TraceEvent traceEvent, CounterFilter filter, string sessionId, out ICounterPayload payload) + { + payload = null; + + string payloadSessionId = (string)traceEvent.PayloadValue(0); + + if (payloadSessionId != sessionId || traceEvent.Version < 1) // Version 1 added the value field. + { + return; + } + + string meterName = (string)traceEvent.PayloadValue(1); + //string meterVersion = (string)obj.PayloadValue(2); + string instrumentName = (string)traceEvent.PayloadValue(3); + string unit = (string)traceEvent.PayloadValue(4); + string tags = (string)traceEvent.PayloadValue(5); + //string rateText = (string)traceEvent.PayloadValue(6); // Not currently using rate for UpDownCounters. + string valueText = (string)traceEvent.PayloadValue(7); + + if (!filter.IsIncluded(meterName, instrumentName)) + { + return; + } + + if (double.TryParse(valueText, NumberStyles.Number | NumberStyles.Float, CultureInfo.InvariantCulture, out double value)) + { + // UpDownCounter reports the value, not the rate - this is different than how Counter behaves. + payload = new UpDownCounterPayload(meterName, instrumentName, null, unit, tags, value, traceEvent.TimeStamp); + + } + else + { + // for observable instruments we assume the lack of data is meaningful and remove it from the UI + // this happens when the ObservableUpDownCounter callback function throws an exception + // or when the ObservableUpDownCounter doesn't include a measurement for a particular set of tag values. payload = new CounterEndedPayload(meterName, instrumentName, traceEvent.TimeStamp); } } diff --git a/src/Microsoft.Diagnostics.Monitoring.EventPipe/Microsoft.Diagnostics.Monitoring.EventPipe.csproj b/src/Microsoft.Diagnostics.Monitoring.EventPipe/Microsoft.Diagnostics.Monitoring.EventPipe.csproj index cf1089533a..aef58a9d4f 100644 --- a/src/Microsoft.Diagnostics.Monitoring.EventPipe/Microsoft.Diagnostics.Monitoring.EventPipe.csproj +++ b/src/Microsoft.Diagnostics.Monitoring.EventPipe/Microsoft.Diagnostics.Monitoring.EventPipe.csproj @@ -26,7 +26,7 @@ - + diff --git a/src/Microsoft.Diagnostics.NETCore.Client/DiagnosticsClient/DiagnosticsClient.cs b/src/Microsoft.Diagnostics.NETCore.Client/DiagnosticsClient/DiagnosticsClient.cs index 3168df5e15..8a98866293 100644 --- a/src/Microsoft.Diagnostics.NETCore.Client/DiagnosticsClient/DiagnosticsClient.cs +++ b/src/Microsoft.Diagnostics.NETCore.Client/DiagnosticsClient/DiagnosticsClient.cs @@ -4,6 +4,7 @@ using System; using System.Buffers.Binary; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; @@ -322,6 +323,15 @@ static IEnumerable GetAllPublishedProcesses(string[] files) continue; } + try + { + Process.GetProcessById(processId); + } + catch (ArgumentException) + { + continue; + } + yield return processId; } } diff --git a/src/Microsoft.Diagnostics.NETCore.Client/Microsoft.Diagnostics.NETCore.Client.csproj b/src/Microsoft.Diagnostics.NETCore.Client/Microsoft.Diagnostics.NETCore.Client.csproj index de97cad6fd..094b309355 100644 --- a/src/Microsoft.Diagnostics.NETCore.Client/Microsoft.Diagnostics.NETCore.Client.csproj +++ b/src/Microsoft.Diagnostics.NETCore.Client/Microsoft.Diagnostics.NETCore.Client.csproj @@ -22,8 +22,12 @@ - - + + + + + + diff --git a/src/SOS/SOS.Extensions/ConsoleServiceFromDebuggerServices.cs b/src/SOS/SOS.Extensions/ConsoleServiceFromDebuggerServices.cs index 9fceefd687..77bc7744e3 100644 --- a/src/SOS/SOS.Extensions/ConsoleServiceFromDebuggerServices.cs +++ b/src/SOS/SOS.Extensions/ConsoleServiceFromDebuggerServices.cs @@ -30,8 +30,15 @@ public ConsoleServiceFromDebuggerServices(DebuggerServices debuggerServices) public void WriteDmlExec(string text, string cmd) { - string dml = $"{DmlEscape(text)}"; - WriteDml(dml); + if (!SupportsDml || string.IsNullOrWhiteSpace(cmd)) + { + Write(text); + } + else + { + string dml = $"{DmlEscape(text)}"; + WriteDml(dml); + } } public bool SupportsDml => _supportsDml ??= _debuggerServices.SupportsDml; @@ -42,6 +49,6 @@ public void WriteDmlExec(string text, string cmd) #endregion - private static string DmlEscape(string text) => new XText(text).ToString(); + private static string DmlEscape(string text) => string.IsNullOrWhiteSpace(text) ? text : new XText(text).ToString(); } } diff --git a/src/SOS/SOS.Extensions/ModuleServiceFromDebuggerServices.cs b/src/SOS/SOS.Extensions/ModuleServiceFromDebuggerServices.cs index 39110a923a..ed88c37705 100644 --- a/src/SOS/SOS.Extensions/ModuleServiceFromDebuggerServices.cs +++ b/src/SOS/SOS.Extensions/ModuleServiceFromDebuggerServices.cs @@ -86,7 +86,7 @@ public ModuleFromDebuggerServices( { _moduleService = moduleService; ModuleIndex = moduleIndex; - FileName = imageName; + FileName = imageName ?? string.Empty; ImageBase = imageBase; ImageSize = imageSize; IndexFileSize = indexTimeStamp == InvalidTimeStamp ? null : indexFileSize; diff --git a/src/SOS/SOS.Hosting/Commands/SOSCommand.cs b/src/SOS/SOS.Hosting/Commands/SOSCommand.cs index 8a126eed95..1190fd29ab 100644 --- a/src/SOS/SOS.Hosting/Commands/SOSCommand.cs +++ b/src/SOS/SOS.Hosting/Commands/SOSCommand.cs @@ -25,15 +25,12 @@ namespace SOS.Hosting [Command(Name = "dumpmodule", DefaultOptions = "DumpModule", Help = "Displays information about a EE module structure at the specified address.")] [Command(Name = "dumpmt", DefaultOptions = "DumpMT", Help = "Displays information about a method table at the specified address.")] [Command(Name = "dumpobj", DefaultOptions = "DumpObj", Aliases = new string[] { "do" }, Help = "Displays info about an object at the specified address.")] - [Command(Name = "dumpruntimetypes", DefaultOptions = "DumpRuntimeTypes", Help = "Finds all System.RuntimeType objects in the GC heap and prints the type name and MethodTable they refer too.")] [Command(Name = "dumpsig", DefaultOptions = "DumpSig", Help = "Dumps the signature of a method or field specified by .")] [Command(Name = "dumpsigelem", DefaultOptions = "DumpSigElem", Help = "Dumps a single element of a signature object.")] - [Command(Name = "dumpstackobjects", DefaultOptions = "DumpStackObjects", Aliases = new string[] { "dso" }, Help = "Displays all managed objects found within the bounds of the current stack.")] [Command(Name = "dumpvc", DefaultOptions = "DumpVC", Help = "Displays info about the fields of a value class.")] [Command(Name = "eeversion", DefaultOptions = "EEVersion", Help = "Displays information about the runtime version.")] [Command(Name = "ehinfo", DefaultOptions = "EHInfo", Help = "Displays the exception handling blocks in a JIT-ed method.")] [Command(Name = "enummem", DefaultOptions = "enummem", Help = "ICLRDataEnumMemoryRegions.EnumMemoryRegions test command.")] - [Command(Name = "finalizequeue", DefaultOptions = "FinalizeQueue", Help = "Displays all objects registered for finalization.")] [Command(Name = "findappdomain", DefaultOptions = "FindAppDomain", Help = "Attempts to resolve the AppDomain of a GC object.")] [Command(Name = "gchandles", DefaultOptions = "GCHandles", Help = "Provides statistics about GCHandles in the process.")] [Command(Name = "gcinfo", DefaultOptions = "GCInfo", Help = "Displays JIT GC encoding for a method.")] @@ -48,10 +45,7 @@ namespace SOS.Hosting [Command(Name = "printexception", DefaultOptions = "PrintException", Aliases = new string[] { "pe" }, Help = "Displays and formats fields of any object derived from the Exception class at the specified address.")] [Command(Name = "soshelp", DefaultOptions = "Help", Help = "Displays help for a specific SOS command.")] [Command(Name = "syncblk", DefaultOptions = "SyncBlk", Help = "Displays the SyncBlock holder info.")] - [Command(Name = "threadpool", DefaultOptions = "ThreadPool", Help = "Lists basic information about the thread pool.")] [Command(Name = "threadstate", DefaultOptions = "ThreadState", Help = "Pretty prints the meaning of a threads state.")] - [Command(Name = "traverseheap", DefaultOptions = "TraverseHeap", Help = "Writes out heap information to a file in a format understood by the CLR Profiler.")] - [Command(Name = "verifyobj", DefaultOptions = "VerifyObj", Help = "Checks the object for signs of corruption.")] [Command(Name = "comstate", DefaultOptions = "COMState", Flags = CommandFlags.Windows, Help = "Lists the COM apartment model for each thread.")] [Command(Name = "dumprcw", DefaultOptions = "DumpRCW", Flags = CommandFlags.Windows, Help = "Displays information about a Runtime Callable Wrapper.")] [Command(Name = "dumpccw", DefaultOptions = "DumpCCW", Flags = CommandFlags.Windows, Help = "Displays information about a COM Callable Wrapper.")] diff --git a/src/SOS/SOS.Hosting/CorDebugDataTargetWrapper.cs b/src/SOS/SOS.Hosting/CorDebugDataTargetWrapper.cs index d229f23f82..5e69c47532 100644 --- a/src/SOS/SOS.Hosting/CorDebugDataTargetWrapper.cs +++ b/src/SOS/SOS.Hosting/CorDebugDataTargetWrapper.cs @@ -143,7 +143,7 @@ private int GetThreadContext( IntPtr self, uint threadId, uint contextFlags, - uint contextSize, + int contextSize, IntPtr context) { byte[] registerContext; @@ -157,7 +157,7 @@ private int GetThreadContext( } try { - Marshal.Copy(registerContext, 0, context, (int)contextSize); + Marshal.Copy(registerContext, 0, context, Math.Min(registerContext.Length, contextSize)); } catch (Exception ex) when (ex is ArgumentOutOfRangeException or ArgumentNullException) { @@ -246,7 +246,7 @@ private delegate int GetThreadContextDelegate( [In] IntPtr self, [In] uint threadId, [In] uint contextFlags, - [In] uint contextSize, + [In] int contextSize, [Out] IntPtr context); #endregion diff --git a/src/SOS/SOS.Hosting/DataTargetWrapper.cs b/src/SOS/SOS.Hosting/DataTargetWrapper.cs index 301fdaab50..303e88d825 100644 --- a/src/SOS/SOS.Hosting/DataTargetWrapper.cs +++ b/src/SOS/SOS.Hosting/DataTargetWrapper.cs @@ -231,7 +231,7 @@ private int GetThreadContext( } try { - Marshal.Copy(registerContext, 0, context, contextSize); + Marshal.Copy(registerContext, 0, context, Math.Min(registerContext.Length, contextSize)); } catch (Exception ex) when (ex is ArgumentOutOfRangeException or ArgumentNullException) { diff --git a/src/SOS/SOS.Hosting/DbgEng/DebugAdvanced.cs b/src/SOS/SOS.Hosting/DbgEng/DebugAdvanced.cs index 83958ee622..5ac7000357 100644 --- a/src/SOS/SOS.Hosting/DbgEng/DebugAdvanced.cs +++ b/src/SOS/SOS.Hosting/DbgEng/DebugAdvanced.cs @@ -24,13 +24,13 @@ internal DebugAdvanced(DebugClient client, SOSHost soshost) private delegate int GetThreadContextDelegate( [In] IntPtr self, [In] IntPtr context, - [In] uint contextSize); + [In] int contextSize); [UnmanagedFunctionPointer(CallingConvention.Winapi)] private delegate int SetThreadContextDelegate( [In] IntPtr self, [In] IntPtr context, - [In] uint contextSize); + [In] int contextSize); #endregion } diff --git a/src/SOS/SOS.Hosting/LLDBServices.cs b/src/SOS/SOS.Hosting/LLDBServices.cs index 30ea3dbb8d..5ac39d92a0 100644 --- a/src/SOS/SOS.Hosting/LLDBServices.cs +++ b/src/SOS/SOS.Hosting/LLDBServices.cs @@ -443,7 +443,7 @@ private delegate int GetThreadContextBySystemIdDelegate( IntPtr self, uint threadId, uint contextFlags, - uint contextSize, + int contextSize, IntPtr context); [UnmanagedFunctionPointer(CallingConvention.Winapi)] diff --git a/src/SOS/SOS.Hosting/SOSHost.cs b/src/SOS/SOS.Hosting/SOSHost.cs index 4dadf27fa4..03d011d0da 100644 --- a/src/SOS/SOS.Hosting/SOSHost.cs +++ b/src/SOS/SOS.Hosting/SOSHost.cs @@ -575,7 +575,7 @@ internal static unsafe int GetSymbolPath( internal int GetThreadContext( IntPtr self, IntPtr context, - uint contextSize) + int contextSize) { IThread thread = ContextService.GetCurrentThread(); if (thread is not null) @@ -589,7 +589,7 @@ internal int GetThreadContextBySystemId( IntPtr self, uint threadId, uint contextFlags, - uint contextSize, + int contextSize, IntPtr context) { byte[] registerContext; @@ -603,7 +603,7 @@ internal int GetThreadContextBySystemId( } try { - Marshal.Copy(registerContext, 0, context, (int)contextSize); + Marshal.Copy(registerContext, 0, context, Math.Min(registerContext.Length, contextSize)); } catch (Exception ex) when (ex is ArgumentOutOfRangeException or ArgumentNullException) { @@ -615,7 +615,7 @@ internal int GetThreadContextBySystemId( internal static int SetThreadContext( IntPtr self, IntPtr context, - uint contextSize) + int contextSize) { return DebugClient.NotImplemented; } diff --git a/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt b/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt index bd1568c039..dbe012fada 100644 --- a/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt +++ b/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt @@ -21,6 +21,9 @@ $(RootBinDir)/TestResults/$(TargetConfiguration)/sos.unittests_$(Timestamp) $(RootBinDir)/tmp/$(TargetConfiguration)\dumps + true + true + true false @@ -140,6 +143,8 @@