From 3a6925c21afccf417eb81578686961e241c6ef20 Mon Sep 17 00:00:00 2001 From: Tomas Date: Mon, 11 Nov 2019 20:00:43 +0100 Subject: [PATCH 01/14] WIP: remaining changes to enable CoreCLR PR pipeline in the runtime repo --- NuGet.config | 1 + eng/helixcorefxtests.proj | 1 + eng/pipelines/coreclr/templates/build-test-job.yml | 2 +- .../coreclr/templates/crossgen-comparison-job.yml | 10 +++------- eng/pipelines/coreclr/templates/format-job.yml | 10 +++++++--- eng/pipelines/coreclr/templates/perf-job.yml | 4 ++-- eng/pipelines/coreclr/templates/run-test-job.yml | 2 +- eng/pipelines/coreclr/templates/xplat-pipeline-job.yml | 4 ++-- src/coreclr/build-test.sh | 2 +- src/coreclr/src/pal/tools/gen-buildsys.sh | 2 +- src/coreclr/tests/src/runtest.proj | 2 +- 11 files changed, 21 insertions(+), 19 deletions(-) diff --git a/NuGet.config b/NuGet.config index 7071908f1abac..4ea7e326f205a 100644 --- a/NuGet.config +++ b/NuGet.config @@ -13,6 +13,7 @@ + diff --git a/eng/helixcorefxtests.proj b/eng/helixcorefxtests.proj index b335ab42a4297..7c2153d6a0d5e 100644 --- a/eng/helixcorefxtests.proj +++ b/eng/helixcorefxtests.proj @@ -85,6 +85,7 @@ TODO: ProjectDir, RootBinDir, TestWorkingDir, and TargetsWindows are global properties set in dir.props, remove the property assignment here when we port to arcade. --> $(MSBuildThisFileDirectory)..\ + $(ProjectDir)\src\coreclr $(ProjectDir)bin\ $(RootBinDir)tests\$(__BuildOS).$(__BuildArch).$(__BuildType)\ true diff --git a/eng/pipelines/coreclr/templates/build-test-job.yml b/eng/pipelines/coreclr/templates/build-test-job.yml index 880ee2935feb8..851673b48cd34 100644 --- a/eng/pipelines/coreclr/templates/build-test-job.yml +++ b/eng/pipelines/coreclr/templates/build-test-job.yml @@ -67,7 +67,7 @@ jobs: displayName: Install native dependencies - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: # Necessary to install correct cmake version - - script: $(coreClrRepoRootDir)eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force + - script: $(Build.SourcesDirectory)\eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force displayName: Install native dependencies diff --git a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml index b10490379eae5..c045128391f09 100644 --- a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml @@ -83,12 +83,8 @@ jobs: # Populate Core_Root - - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - script: ./build-test.sh $(buildConfig) $(archType) $(crossArg) generatelayoutonly - displayName: Populate Core_Root - - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - script: build-test.cmd $(buildConfig) $(archType) generateLayoutOnly - displayName: Populate Core_Root + - script: $(coreClrRepoRootDir)build-test$(scriptExt) $(buildConfig) $(archType) $(crossArg) generatelayoutonly + displayName: Populate Core_Root # Create baseline output on the host (x64) machine @@ -96,7 +92,7 @@ jobs: displayName: Create cross-platform crossgen baseline inputs: scriptSource: 'filePath' - scriptPath: $(Build.SourcesDirectory)/tests/scripts/crossgen_comparison.py + scriptPath: $(coreClrRepoRoot)/tests/scripts/crossgen_comparison.py ${{ if ne(parameters.osGroup, 'Windows_NT') }}: arguments: crossgen_framework diff --git a/eng/pipelines/coreclr/templates/format-job.yml b/eng/pipelines/coreclr/templates/format-job.yml index 30e90348dd5ad..55d0f35139994 100644 --- a/eng/pipelines/coreclr/templates/format-job.yml +++ b/eng/pipelines/coreclr/templates/format-job.yml @@ -10,7 +10,7 @@ parameters: ### Format job jobs: -- template: xplat-job.yml +- template: xplat-pipeline-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} @@ -32,8 +32,12 @@ jobs: displayName: Run tests/scripts/format.py inputs: scriptSource: 'filePath' - scriptPath: $(Build.SourcesDirectory)/tests/scripts/format.py - arguments: '-c $(Build.SourcesDirectory) -o $(osGroup) -a $(archType)' + ${{ if startsWith(variables['Build.DefinitionName'], 'runtime') }}: + scriptPath: $(Build.SourcesDirectory)/src/coreclr/tests/scripts/format.py + arguments: '-c $(Build.SourcesDirectory)/src/coreclr -o $(osGroup) -a $(archType)' + ${{ if not(startsWith(variables['Build.DefinitionName'], 'runtime')) }}: + scriptPath: $(Build.SourcesDirectory)/tests/scripts/format.py + arguments: '-c $(Build.SourcesDirectory) -o $(osGroup) -a $(archType)' - task: PublishBuildArtifacts@1 displayName: Publish format.patch inputs: diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index f0d598d718049..a6b0998105778 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -55,9 +55,9 @@ jobs: # Create Core_Root - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - script: ./build-test.sh ${{ parameters.buildConfig }} ${{ parameters.archType }} generatelayoutonly + - script: $(coreClrRepoRootDir)build-test$(scriptExt) ${{ parameters.buildConfig }} ${{ parameters.archType }} generatelayoutonly displayName: Create Core_Root - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: # TODO: add generatelayoutonly to build-test.cmd. - - script: build-test.cmd ${{ parameters.buildConfig }} ${{ parameters.archType }} skipmanaged skipnative + - script: $(coreClrRepoRootDir)build-test$(scriptExt) ${{ parameters.buildConfig }} ${{ parameters.archType }} skipmanaged skipnative displayName: Create Core_Root \ No newline at end of file diff --git a/eng/pipelines/coreclr/templates/run-test-job.yml b/eng/pipelines/coreclr/templates/run-test-job.yml index ddac9b17b6181..3b26e220a8a58 100644 --- a/eng/pipelines/coreclr/templates/run-test-job.yml +++ b/eng/pipelines/coreclr/templates/run-test-job.yml @@ -250,7 +250,7 @@ jobs: # Choose which tests to send to Helix: CoreFX or CoreCLR. ${{ if eq(parameters.corefxTests, true) }}: - helixProjectArguments: '$(coreClrRepoRoot)/eng/helixcorefxtests.proj' + helixProjectArguments: '$(Build.SourcesDirectory)/eng/helixcorefxtests.proj' ${{ if ne(parameters.corefxTests, true) }}: helixProjectArguments: '$(coreClrRepoRoot)/tests/src/helixpublishwitharcade.proj' diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index ffc806edf8714..ac96e979781ae 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -80,9 +80,9 @@ jobs: value: '/' - name: coreClrRepoRoot - value: '$(Build.SourcesDirectory)/src/coreclr + value: '$(Build.SourcesDirectory)/src/coreclr' - - ${{ if not(startsWith(variables['Build.DefinitionName'], 'runtime')) }}:' + - ${{ if not(startsWith(variables['Build.DefinitionName'], 'runtime')) }}: - name: coreClrRepoRoot value: '$(Build.SourcesDirectory)' diff --git a/src/coreclr/build-test.sh b/src/coreclr/build-test.sh index 9f178bfa84fad..ecf417a10f974 100755 --- a/src/coreclr/build-test.sh +++ b/src/coreclr/build-test.sh @@ -537,7 +537,7 @@ build_native_projects() extraCmakeArguments="$extraCmakeArguments -DCLR_CMAKE_ENABLE_CODE_COVERAGE=1" fi - nextCommand="\"$scriptDir/gen-buildsys.sh\" \"$__TestDir\" \"$intermediatesForBuild\" $platformArch $__BuildType $generator $extraCmakeArguments $__cmakeargs" + nextCommand="CONFIG_DIR=\"$__RepoRootDir/eng/common/cross\" \"$scriptDir/gen-buildsys.sh\" \"$__TestDir\" \"$intermediatesForBuild\" $platformArch $__BuildType $generator $extraCmakeArguments $__cmakeargs" echo "Invoking $nextCommand" eval $nextCommand diff --git a/src/coreclr/src/pal/tools/gen-buildsys.sh b/src/coreclr/src/pal/tools/gen-buildsys.sh index 41c49f032b4f9..f555119e33915 100755 --- a/src/coreclr/src/pal/tools/gen-buildsys.sh +++ b/src/coreclr/src/pal/tools/gen-buildsys.sh @@ -75,7 +75,7 @@ if [ "$CROSSCOMPILE" == "1" ]; then exit 1 fi export TARGET_BUILD_ARCH=$build_arch - cmake_extra_defines="$cmake_extra_defines -C ${__CoreClrDir}/tryrun.cmake" + cmake_extra_defines="$cmake_extra_defines -C ${__CoreClrDir}/src/pal/tools/tryrun.cmake" cmake_extra_defines="$cmake_extra_defines -DCMAKE_TOOLCHAIN_FILE=${__RepoRootDir}/eng/common/cross/toolchain.cmake" fi diff --git a/src/coreclr/tests/src/runtest.proj b/src/coreclr/tests/src/runtest.proj index c7b3483f293a6..420f865c9acda 100644 --- a/src/coreclr/tests/src/runtest.proj +++ b/src/coreclr/tests/src/runtest.proj @@ -26,7 +26,7 @@ $([System.IO.Path]::GetFullPath(%(Identity))) - + From 1da2ebe53d453dc429ed1bf7f537f74d69eb6a76 Mon Sep 17 00:00:00 2001 From: Tomas Date: Mon, 11 Nov 2019 22:39:08 +0100 Subject: [PATCH 02/14] Fix location of empty.csproj in Linux build.sh --- src/coreclr/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/build.sh b/src/coreclr/build.sh index d2d3dad1f5ee4..0446592217b29 100755 --- a/src/coreclr/build.sh +++ b/src/coreclr/build.sh @@ -235,7 +235,7 @@ build_native() __versionSourceFile="$intermediatesForBuild/version.c" if [ $__SkipGenerateVersion == 0 ]; then pwd - "$__RepoRootDir/eng/common/msbuild.sh" $__ArcadeScriptArgs $__ProjectRoot/eng/empty.csproj \ + "$__RepoRootDir/eng/common/msbuild.sh" $__ArcadeScriptArgs $__RepoRootDir/eng/empty.csproj \ /p:NativeVersionFile=$__versionSourceFile \ /t:GenerateNativeVersionFile /restore \ $__CommonMSBuildArgs $__UnprocessedBuildArgs @@ -1063,7 +1063,7 @@ __MsbuildDebugLogsDir="$__LogsDir/MsbuildDebugLogs" __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType" __PackagesBinDir="$__BinDir/.nuget" export __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType" -export __ArtifactsIntermediatesDir="$__ProjectDir/artifacts/obj" +export __ArtifactsIntermediatesDir="$__RepoRootDir/artifacts/obj" __isMSBuildOnNETCoreSupported=0 __CrossComponentBinDir="$__BinDir" From cf32e9e3ec64ea195b98519b8ac9792d1fd9e9dc Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 12 Nov 2019 01:03:20 +0100 Subject: [PATCH 03/14] Fix path to CoreFX.issues.rsp file; hopefully fix Linux builds on ARM --- eng/helixcorefxtests.proj | 2 +- src/coreclr/build.sh | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/eng/helixcorefxtests.proj b/eng/helixcorefxtests.proj index 7c2153d6a0d5e..584b54f041581 100644 --- a/eng/helixcorefxtests.proj +++ b/eng/helixcorefxtests.proj @@ -85,7 +85,7 @@ TODO: ProjectDir, RootBinDir, TestWorkingDir, and TargetsWindows are global properties set in dir.props, remove the property assignment here when we port to arcade. --> $(MSBuildThisFileDirectory)..\ - $(ProjectDir)\src\coreclr + $(ProjectDir)\src\coreclr $(ProjectDir)bin\ $(RootBinDir)tests\$(__BuildOS).$(__BuildArch).$(__BuildType)\ true diff --git a/src/coreclr/build.sh b/src/coreclr/build.sh index 0446592217b29..f042e9ccc2c4b 100755 --- a/src/coreclr/build.sh +++ b/src/coreclr/build.sh @@ -273,8 +273,9 @@ build_native() extraCmakeArguments="$extraCmakeArguments -DCLR_CMAKE_ENABLE_CODE_COVERAGE=1" fi - echo "Invoking \"$scriptDir/gen-buildsys.sh\" \"$__ProjectRoot\" \"$intermediatesForBuild\" $platformArch $__BuildType $generator $scan_build $extraCmakeArguments $__cmakeargs" - source "$scriptDir/gen-buildsys.sh" "$__ProjectRoot" "$intermediatesForBuild" $platformArch $__BuildType $generator $scan_build "$extraCmakeArguments" "$__cmakeargs" + nextCommand="CONFIG_DIR=\"$__RepoRootDir/eng/common/cross\" \"$scriptDir/gen-buildsys.sh\" \"$__ProjectRoot\" \"$intermediatesForBuild\" $platformArch $__BuildType $generator $scan_build $extraCmakeArguments $__cmakeargs" + echo "Invoking $nextCommand" + eval $nextCommand if [ $? != 0 ]; then echo "${__ErrMsgPrefix}Failed to generate $message build project!" From 74ce55b755192373e95fb45dd07a5cb1113ccfa7 Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 12 Nov 2019 08:17:15 +0100 Subject: [PATCH 04/14] Two more path fixes --- eng/helixcorefxtests.proj | 2 +- src/coreclr/src/pal/tools/gen-buildsys.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/helixcorefxtests.proj b/eng/helixcorefxtests.proj index 584b54f041581..2ee3ac4290fcc 100644 --- a/eng/helixcorefxtests.proj +++ b/eng/helixcorefxtests.proj @@ -85,7 +85,7 @@ TODO: ProjectDir, RootBinDir, TestWorkingDir, and TargetsWindows are global properties set in dir.props, remove the property assignment here when we port to arcade. --> $(MSBuildThisFileDirectory)..\ - $(ProjectDir)\src\coreclr + $(ProjectDir)\src\coreclr\ $(ProjectDir)bin\ $(RootBinDir)tests\$(__BuildOS).$(__BuildArch).$(__BuildType)\ true diff --git a/src/coreclr/src/pal/tools/gen-buildsys.sh b/src/coreclr/src/pal/tools/gen-buildsys.sh index f555119e33915..41c49f032b4f9 100755 --- a/src/coreclr/src/pal/tools/gen-buildsys.sh +++ b/src/coreclr/src/pal/tools/gen-buildsys.sh @@ -75,7 +75,7 @@ if [ "$CROSSCOMPILE" == "1" ]; then exit 1 fi export TARGET_BUILD_ARCH=$build_arch - cmake_extra_defines="$cmake_extra_defines -C ${__CoreClrDir}/src/pal/tools/tryrun.cmake" + cmake_extra_defines="$cmake_extra_defines -C ${__CoreClrDir}/tryrun.cmake" cmake_extra_defines="$cmake_extra_defines -DCMAKE_TOOLCHAIN_FILE=${__RepoRootDir}/eng/common/cross/toolchain.cmake" fi From c2d075ae5174028718146904ad695e8e4a6a6396 Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 12 Nov 2019 09:36:08 +0100 Subject: [PATCH 05/14] Various pipeline fixes --- eng/helixcorefxtests.proj | 2 +- .../coreclr/templates/crossgen-comparison-job.yml | 8 ++++---- eng/pipelines/coreclr/templates/format-job.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/helixcorefxtests.proj b/eng/helixcorefxtests.proj index 2ee3ac4290fcc..f25f1cc0e19e9 100644 --- a/eng/helixcorefxtests.proj +++ b/eng/helixcorefxtests.proj @@ -150,7 +150,7 @@ - <_ProjectsToBuild Include="..\tests\testenvironment.proj"> + <_ProjectsToBuild Include="$(ProjectDir)tests\testenvironment.proj"> Scenario=$(Scenario);TestEnvFileName=$(TestEnvFilePath);TargetsWindows=$(TargetsWindows) diff --git a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml index c045128391f09..bf192cb3cef1a 100644 --- a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml @@ -40,14 +40,14 @@ jobs: value: $(osGroup).$(hostArchType)_$(archType).$(buildConfigUpper) - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - name: binDirectory - value: $(Build.SourcesDirectory)/bin + value: $(coreClrRepoRoot)/bin - name: productDirectory - value: $(Build.SourcesDirectory)/bin/Product + value: $(binDirectory)/Product - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - name: binDirectory - value: $(Build.SourcesDirectory)\bin + value: $(coreClrRepoRoot)\bin - name: productDirectory - value: $(Build.SourcesDirectory)\bin\Product + value: $(binDirectory\Product # Test job depends on the corresponding build job dependsOn: ${{ format('build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} diff --git a/eng/pipelines/coreclr/templates/format-job.yml b/eng/pipelines/coreclr/templates/format-job.yml index 55d0f35139994..f40733ffd3225 100644 --- a/eng/pipelines/coreclr/templates/format-job.yml +++ b/eng/pipelines/coreclr/templates/format-job.yml @@ -27,7 +27,7 @@ jobs: - task: DotNetCoreInstaller@0 inputs: packageType: 'sdk' - version: '2.1.402' + version: '3.1.100' - task: PythonScript@0 displayName: Run tests/scripts/format.py inputs: From fb555f5bbd69575e2b554145b7fd0c26aa8f4426 Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 12 Nov 2019 10:55:00 +0100 Subject: [PATCH 06/14] Crossgen comparison & format job fixes --- eng/pipelines/coreclr/templates/crossgen-comparison-job.yml | 2 +- eng/pipelines/coreclr/templates/format-job.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml index bf192cb3cef1a..a44820f01828e 100644 --- a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml @@ -122,7 +122,7 @@ jobs: Creator: $(Creator) WorkItemTimeout: 1:00 # 1 hour WorkItemDirectory: '$(binDirectory)' - CorrelationPayloadDirectory: '$(Build.SourcesDirectory)/tests/scripts' + CorrelationPayloadDirectory: '$(coreClrRepoRoot)/tests/scripts' ${{ if ne(parameters.osName, 'Windows_NT') }}: WorkItemCommand: chmod +x $HELIX_WORKITEM_PAYLOAD/Product/$(targetFlavor)/crossgen; diff --git a/eng/pipelines/coreclr/templates/format-job.yml b/eng/pipelines/coreclr/templates/format-job.yml index f40733ffd3225..fc09facd7e7c3 100644 --- a/eng/pipelines/coreclr/templates/format-job.yml +++ b/eng/pipelines/coreclr/templates/format-job.yml @@ -27,7 +27,7 @@ jobs: - task: DotNetCoreInstaller@0 inputs: packageType: 'sdk' - version: '3.1.100' + version: '3.1.100-preview3-010431' - task: PythonScript@0 displayName: Run tests/scripts/format.py inputs: From a414ceaf1136451b57bd96cf54da0b6833923cbc Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 12 Nov 2019 11:12:56 +0100 Subject: [PATCH 07/14] More fixes for the formatting job --- eng/pipelines/coreclr/templates/format-job.yml | 12 ++++-------- src/coreclr/tests/scripts/format.py | 5 ++++- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/coreclr/templates/format-job.yml b/eng/pipelines/coreclr/templates/format-job.yml index fc09facd7e7c3..d7bd4cbbe4de1 100644 --- a/eng/pipelines/coreclr/templates/format-job.yml +++ b/eng/pipelines/coreclr/templates/format-job.yml @@ -27,21 +27,17 @@ jobs: - task: DotNetCoreInstaller@0 inputs: packageType: 'sdk' - version: '3.1.100-preview3-010431' + version: '2.1.402' - task: PythonScript@0 displayName: Run tests/scripts/format.py inputs: scriptSource: 'filePath' - ${{ if startsWith(variables['Build.DefinitionName'], 'runtime') }}: - scriptPath: $(Build.SourcesDirectory)/src/coreclr/tests/scripts/format.py - arguments: '-c $(Build.SourcesDirectory)/src/coreclr -o $(osGroup) -a $(archType)' - ${{ if not(startsWith(variables['Build.DefinitionName'], 'runtime')) }}: - scriptPath: $(Build.SourcesDirectory)/tests/scripts/format.py - arguments: '-c $(Build.SourcesDirectory) -o $(osGroup) -a $(archType)' + scriptPath: $(coreClrRepoRoot)/tests/scripts/format.py + arguments: '-c $(coreClrRepoRoot) -o $(osGroup) -a $(archType)' - task: PublishBuildArtifacts@1 displayName: Publish format.patch inputs: - PathtoPublish: '$(Build.SourcesDirectory)/format.patch' + PathtoPublish: '$(coreClrRepoRoot)/format.patch' ArtifactName: format.$(osGroup).$(archType).patch continueOnError: true condition: failed() diff --git a/src/coreclr/tests/scripts/format.py b/src/coreclr/tests/scripts/format.py index a20368a5905d6..08910d7dc8de8 100644 --- a/src/coreclr/tests/scripts/format.py +++ b/src/coreclr/tests/scripts/format.py @@ -157,9 +157,12 @@ def main(argv): os.chdir(current_dir) + patchFilePath = os.path.join(coreclr, "format.patch") + if returncode != 0: # Create a patch file - patchFile = open("format.patch", "w") + print("Creating patch file " + patchFilePath) + patchFile = open(patchFilePath, "w") proc = subprocess.Popen(["git", "diff", "--patch", "-U20"], env=my_env, stdout=patchFile) output,error = proc.communicate() From 14fb98deecfbf8805941067027f2cc4db4426db5 Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 12 Nov 2019 11:27:43 +0100 Subject: [PATCH 08/14] Introduce repo root variable in format.py --- src/coreclr/tests/scripts/format.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/coreclr/tests/scripts/format.py b/src/coreclr/tests/scripts/format.py index 08910d7dc8de8..39aaedfaefb70 100644 --- a/src/coreclr/tests/scripts/format.py +++ b/src/coreclr/tests/scripts/format.py @@ -64,6 +64,12 @@ def main(argv): return -1 coreclr = args.coreclr + repoRoot = os.path.join(coreclr, '..', '..') + + # Remove after repo consolidation + if not os.path.isfile(os.path.join(repoRoot, '.dotnet-runtime-placeholder')): + repoRoot = coreclr + platform = args.os arch = args.arch @@ -73,7 +79,7 @@ def main(argv): bootstrapFilename = "" - jitUtilsPath = os.path.join(coreclr, "jitutils") + jitUtilsPath = os.path.join(repoRoot, "jitutils") if os.path.isdir(jitUtilsPath): print("Deleting " + jitUtilsPath) @@ -86,7 +92,7 @@ def main(argv): bootstrapUrl = "https://raw.githubusercontent.com/dotnet/jitutils/master/" + bootstrapFilename - bootstrapPath = os.path.join(coreclr, bootstrapFilename) + bootstrapPath = os.path.join(repoRoot, bootstrapFilename) urlretrieve(bootstrapUrl, bootstrapPath) if not os.path.isfile(bootstrapPath): From 087e8fc62cb62d24b7bbd2f4110bdb0b5c32de03 Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 12 Nov 2019 11:51:32 +0100 Subject: [PATCH 09/14] Try to fix Windows Formatting job by fixing slashes in path --- src/coreclr/tests/scripts/format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/tests/scripts/format.py b/src/coreclr/tests/scripts/format.py index 39aaedfaefb70..b89f3e07d25ec 100644 --- a/src/coreclr/tests/scripts/format.py +++ b/src/coreclr/tests/scripts/format.py @@ -63,7 +63,7 @@ def main(argv): print('Bad path to coreclr') return -1 - coreclr = args.coreclr + coreclr = args.coreclr.replace('/', os.sep) repoRoot = os.path.join(coreclr, '..', '..') # Remove after repo consolidation From fe7217d57ff6adda236700fa5e28addf2305ff3c Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 12 Nov 2019 15:09:14 +0100 Subject: [PATCH 10/14] Shorten intermediate paths for two interop test native components --- .../ReversePInvoke/PassingByOut/CMakeLists.txt | 9 +++++---- .../ReversePInvoke/PassingByRef/CMakeLists.txt | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/coreclr/tests/src/Interop/PInvoke/SizeParamIndex/ReversePInvoke/PassingByOut/CMakeLists.txt b/src/coreclr/tests/src/Interop/PInvoke/SizeParamIndex/ReversePInvoke/PassingByOut/CMakeLists.txt index f9639aa95aa7d..151d42f06c3d2 100644 --- a/src/coreclr/tests/src/Interop/PInvoke/SizeParamIndex/ReversePInvoke/PassingByOut/CMakeLists.txt +++ b/src/coreclr/tests/src/Interop/PInvoke/SizeParamIndex/ReversePInvoke/PassingByOut/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required (VERSION 2.6) -project (ReversePInvokePassingByOutNative) +project (RPIP_ByOutNative) include ("${CLR_INTEROP_TEST_ROOT}/Interop.cmake") include_directories("..") set(SOURCES @@ -7,7 +7,8 @@ set(SOURCES ) # Additional files to reference: # add the executable -add_library (ReversePInvokePassingByOutNative SHARED ${SOURCES}) -target_link_libraries(ReversePInvokePassingByOutNative ${LINK_LIBRARIES_ADDITIONAL}) +add_library (RPIP_ByOutNative SHARED ${SOURCES}) +set_property (TARGET RPIP_ByOutNative PROPERTY OUTPUT_NAME ReversePInvokePassingByOutNative) +target_link_libraries(RPIP_ByOutNative ${LINK_LIBRARIES_ADDITIONAL}) # add the install targets -install (TARGETS ReversePInvokePassingByOutNative DESTINATION bin) +install (TARGETS RPIP_ByOutNative DESTINATION bin) diff --git a/src/coreclr/tests/src/Interop/PInvoke/SizeParamIndex/ReversePInvoke/PassingByRef/CMakeLists.txt b/src/coreclr/tests/src/Interop/PInvoke/SizeParamIndex/ReversePInvoke/PassingByRef/CMakeLists.txt index 180d43c9676c7..3176cfc826e96 100644 --- a/src/coreclr/tests/src/Interop/PInvoke/SizeParamIndex/ReversePInvoke/PassingByRef/CMakeLists.txt +++ b/src/coreclr/tests/src/Interop/PInvoke/SizeParamIndex/ReversePInvoke/PassingByRef/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required (VERSION 2.6) -project (ReversePInvokePassingByRefNative) +project (RPIP_ByRefNative) include ("${CLR_INTEROP_TEST_ROOT}/Interop.cmake") include_directories("..") set(SOURCES @@ -7,7 +7,8 @@ set(SOURCES ) # Additional files to reference: # add the executable -add_library (ReversePInvokePassingByRefNative SHARED ${SOURCES}) -target_link_libraries(ReversePInvokePassingByRefNative ${LINK_LIBRARIES_ADDITIONAL}) +add_library (RPIP_ByRefNative SHARED ${SOURCES}) +set_property (TARGET RPIP_ByRefNative PROPERTY OUTPUT_NAME ReversePInvokePassingByRefNative) +target_link_libraries(RPIP_ByRefNative ${LINK_LIBRARIES_ADDITIONAL}) # add the install targets -install (TARGETS ReversePInvokePassingByRefNative DESTINATION bin) +install (TARGETS RPIP_ByRefNative DESTINATION bin) From a84d432c4a77439b00108e7ab9afe9e7d359af84 Mon Sep 17 00:00:00 2001 From: jashook Date: Tue, 12 Nov 2019 10:58:16 -0800 Subject: [PATCH 11/14] Initial attempt to fix format jobs --- src/coreclr/tests/scripts/format.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/coreclr/tests/scripts/format.py b/src/coreclr/tests/scripts/format.py index b89f3e07d25ec..56d40c5d5e00c 100644 --- a/src/coreclr/tests/scripts/format.py +++ b/src/coreclr/tests/scripts/format.py @@ -64,11 +64,6 @@ def main(argv): return -1 coreclr = args.coreclr.replace('/', os.sep) - repoRoot = os.path.join(coreclr, '..', '..') - - # Remove after repo consolidation - if not os.path.isfile(os.path.join(repoRoot, '.dotnet-runtime-placeholder')): - repoRoot = coreclr platform = args.os arch = args.arch @@ -79,7 +74,7 @@ def main(argv): bootstrapFilename = "" - jitUtilsPath = os.path.join(repoRoot, "jitutils") + jitUtilsPath = os.path.join(coreclr, "jitutils") if os.path.isdir(jitUtilsPath): print("Deleting " + jitUtilsPath) @@ -92,7 +87,10 @@ def main(argv): bootstrapUrl = "https://raw.githubusercontent.com/dotnet/jitutils/master/" + bootstrapFilename - bootstrapPath = os.path.join(repoRoot, bootstrapFilename) + bootstrapPath = os.path.join(os.path.join(coreclr, "bin", "jitutils"), bootstrapFilename) + if not os.path.isdir(os.path.dirname(bootstrapPath)): + os.makedirs(os.path.dirname(bootstrapPath)) + urlretrieve(bootstrapUrl, bootstrapPath) if not os.path.isfile(bootstrapPath): @@ -119,7 +117,7 @@ def main(argv): # Run jit-format returncode = 0 - jitutilsBin = os.path.join(coreclr, "jitutils", "bin") + jitutilsBin = os.path.join(os.path.dirname(bootstrapPath), "jitutils", "bin") my_env["PATH"] = jitutilsBin + os.pathsep + my_env["PATH"] current_dir = os.getcwd() From f423e18b33beadd7ce135bcad4c34d267de17814 Mon Sep 17 00:00:00 2001 From: jashook Date: Wed, 13 Nov 2019 00:29:08 +0000 Subject: [PATCH 12/14] Change the cwd to the bootstrap location --- src/coreclr/tests/scripts/format.py | 37 +++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/src/coreclr/tests/scripts/format.py b/src/coreclr/tests/scripts/format.py index 56d40c5d5e00c..2946dcf47e7ef 100644 --- a/src/coreclr/tests/scripts/format.py +++ b/src/coreclr/tests/scripts/format.py @@ -20,6 +20,18 @@ import subprocess import shutil +class ChangeDir: + def __init__(self, dir): + self.dir = dir + self.cwd = None + + def __enter__(self): + self.cwd = os.getcwd() + os.chdir(self.dir) + + def __exit__(self, exc_type, exc_val, exc_tb): + os.chdir(self.cwd) + # Version specific imports if sys.version_info.major < 3: @@ -88,8 +100,12 @@ def main(argv): bootstrapUrl = "https://raw.githubusercontent.com/dotnet/jitutils/master/" + bootstrapFilename bootstrapPath = os.path.join(os.path.join(coreclr, "bin", "jitutils"), bootstrapFilename) - if not os.path.isdir(os.path.dirname(bootstrapPath)): - os.makedirs(os.path.dirname(bootstrapPath)) + + if os.path.isdir(os.path.dirname(bootstrapPath)): + shutil.rmtree(os.path.dirname(bootstrapPath)) + + assert not os.path.isdir(os.path.dirname(bootstrapPath)) + os.makedirs(os.path.dirname(bootstrapPath)) urlretrieve(bootstrapUrl, bootstrapPath) @@ -105,14 +121,15 @@ def main(argv): print(bootstrapPath) - # Run bootstrap - if platform == 'Linux' or platform == 'OSX': - print("Running bootstrap") - proc = subprocess.Popen(['bash', bootstrapPath], env=my_env) - output,error = proc.communicate() - elif platform == 'Windows_NT': - proc = subprocess.Popen([bootstrapPath], env=my_env) - output,error = proc.communicate() + with ChangeDir(os.path.dirname(bootstrapPath)): + # Run bootstrap + if platform == 'Linux' or platform == 'OSX': + print("Running bootstrap") + proc = subprocess.Popen(['bash', bootstrapPath], env=my_env) + output,error = proc.communicate() + elif platform == 'Windows_NT': + proc = subprocess.Popen([bootstrapPath], env=my_env) + output,error = proc.communicate() # Run jit-format From 0518f65cf89a126d37924aee477ea7259d74809f Mon Sep 17 00:00:00 2001 From: jashook Date: Wed, 13 Nov 2019 03:37:03 -0800 Subject: [PATCH 13/14] Use tempdir for jitutils --- src/coreclr/tests/scripts/format.py | 200 +++++++++++++++------------- 1 file changed, 108 insertions(+), 92 deletions(-) diff --git a/src/coreclr/tests/scripts/format.py b/src/coreclr/tests/scripts/format.py index 2946dcf47e7ef..85a59dfc1726c 100644 --- a/src/coreclr/tests/scripts/format.py +++ b/src/coreclr/tests/scripts/format.py @@ -16,6 +16,7 @@ import os import sys import tarfile +import tempfile import zipfile import subprocess import shutil @@ -23,14 +24,33 @@ class ChangeDir: def __init__(self, dir): self.dir = dir + self.cwd = os.getcwd() + + def __enter__(self): + try: + self.cwd = os.getcwd() + except: + self.cwd = os.getcwd() + + os.chdir(self.dir) + + def __exit__(self, exc_type, exc_val, exc_tb): + os.chdir(self.cwd) + +class TempDir: + def __init__(self, path=None): + self.dir = tempfile.mkdtemp() if path is None else path self.cwd = None def __enter__(self): self.cwd = os.getcwd() os.chdir(self.dir) + return self.dir + def __exit__(self, exc_type, exc_val, exc_tb): os.chdir(self.cwd) + shutil.rmtree(self.dir) # Version specific imports @@ -99,29 +119,25 @@ def main(argv): bootstrapUrl = "https://raw.githubusercontent.com/dotnet/jitutils/master/" + bootstrapFilename - bootstrapPath = os.path.join(os.path.join(coreclr, "bin", "jitutils"), bootstrapFilename) + with TempDir() as temp_location: + bootstrapPath = os.path.join(temp_location, bootstrapFilename) - if os.path.isdir(os.path.dirname(bootstrapPath)): - shutil.rmtree(os.path.dirname(bootstrapPath)) + assert len(os.listdir(os.path.dirname(bootstrapPath))) == 0 - assert not os.path.isdir(os.path.dirname(bootstrapPath)) - os.makedirs(os.path.dirname(bootstrapPath)) + urlretrieve(bootstrapUrl, bootstrapPath) - urlretrieve(bootstrapUrl, bootstrapPath) + if not os.path.isfile(bootstrapPath): + print("Did not download bootstrap!") + return -1 - if not os.path.isfile(bootstrapPath): - print("Did not download bootstrap!") - return -1 - - # On *nix platforms, we need to make the bootstrap file executable + # On *nix platforms, we need to make the bootstrap file executable - if platform == 'Linux' or platform == 'OSX': - print("Making bootstrap executable") - os.chmod(bootstrapPath, 0o751) + if platform == 'Linux' or platform == 'OSX': + print("Making bootstrap executable") + os.chmod(bootstrapPath, 0o751) - print(bootstrapPath) + print(bootstrapPath) - with ChangeDir(os.path.dirname(bootstrapPath)): # Run bootstrap if platform == 'Linux' or platform == 'OSX': print("Running bootstrap") @@ -131,88 +147,88 @@ def main(argv): proc = subprocess.Popen([bootstrapPath], env=my_env) output,error = proc.communicate() - # Run jit-format - - returncode = 0 - jitutilsBin = os.path.join(os.path.dirname(bootstrapPath), "jitutils", "bin") - my_env["PATH"] = jitutilsBin + os.pathsep + my_env["PATH"] - current_dir = os.getcwd() - - if not os.path.isdir(jitutilsBin): - print("Jitutils not built!") - return -1 + # Run jit-format - jitformat = jitutilsBin + returncode = 0 + jitutilsBin = os.path.join(os.path.dirname(bootstrapPath), "jitutils", "bin") + my_env["PATH"] = jitutilsBin + os.pathsep + my_env["PATH"] + current_dir = os.getcwd() - if platform == 'Linux' or platform == 'OSX': - jitformat = os.path.join(jitformat, "jit-format") - elif platform == 'Windows_NT': - jitformat = os.path.join(jitformat,"jit-format.bat") - errorMessage = "" + if not os.path.isdir(jitutilsBin): + print("Jitutils not built!") + return -1 - builds = ["Checked", "Debug", "Release"] - projects = ["dll", "standalone", "crossgen"] + jitformat = jitutilsBin - for build in builds: - for project in projects: - proc = subprocess.Popen([jitformat, "-a", arch, "-b", build, "-o", platform, "-c", coreclr, "--verbose", "--projects", project], env=my_env) + if platform == 'Linux' or platform == 'OSX': + jitformat = os.path.join(jitformat, "jit-format") + elif platform == 'Windows_NT': + jitformat = os.path.join(jitformat,"jit-format.bat") + errorMessage = "" + + builds = ["Checked", "Debug", "Release"] + projects = ["dll", "standalone", "crossgen"] + + for build in builds: + for project in projects: + proc = subprocess.Popen([jitformat, "-a", arch, "-b", build, "-o", platform, "-c", coreclr, "--verbose", "--projects", project], env=my_env) + output,error = proc.communicate() + errorcode = proc.returncode + + if errorcode != 0: + errorMessage += "\tjit-format -a " + arch + " -b " + build + " -o " + platform + errorMessage += " -c --verbose --fix --projects " + project +"\n" + returncode = errorcode + + # Fix mode doesn't return an error, so we have to run the build, then run with + # --fix to generate the patch. This means that it is likely only the first run + # of jit-format will return a formatting failure. + if errorcode == -2: + # If errorcode was -2, no need to run clang-tidy again + proc = subprocess.Popen([jitformat, "--fix", "--untidy", "-a", arch, "-b", build, "-o", platform, "-c", coreclr, "--verbose", "--projects", project], env=my_env) + output,error = proc.communicate() + else: + # Otherwise, must run both + proc = subprocess.Popen([jitformat, "--fix", "-a", arch, "-b", build, "-o", platform, "-c", coreclr, "--verbose", "--projects", project], env=my_env) + output,error = proc.communicate() + + os.chdir(current_dir) + + patchFilePath = os.path.join(coreclr, "format.patch") + + if returncode != 0: + # Create a patch file + print("Creating patch file " + patchFilePath) + patchFile = open(patchFilePath, "w") + proc = subprocess.Popen(["git", "diff", "--patch", "-U20"], env=my_env, stdout=patchFile) output,error = proc.communicate() - errorcode = proc.returncode - - if errorcode != 0: - errorMessage += "\tjit-format -a " + arch + " -b " + build + " -o " + platform - errorMessage += " -c --verbose --fix --projects " + project +"\n" - returncode = errorcode - - # Fix mode doesn't return an error, so we have to run the build, then run with - # --fix to generate the patch. This means that it is likely only the first run - # of jit-format will return a formatting failure. - if errorcode == -2: - # If errorcode was -2, no need to run clang-tidy again - proc = subprocess.Popen([jitformat, "--fix", "--untidy", "-a", arch, "-b", build, "-o", platform, "-c", coreclr, "--verbose", "--projects", project], env=my_env) - output,error = proc.communicate() - else: - # Otherwise, must run both - proc = subprocess.Popen([jitformat, "--fix", "-a", arch, "-b", build, "-o", platform, "-c", coreclr, "--verbose", "--projects", project], env=my_env) - output,error = proc.communicate() - - os.chdir(current_dir) - - patchFilePath = os.path.join(coreclr, "format.patch") - - if returncode != 0: - # Create a patch file - print("Creating patch file " + patchFilePath) - patchFile = open(patchFilePath, "w") - proc = subprocess.Popen(["git", "diff", "--patch", "-U20"], env=my_env, stdout=patchFile) - output,error = proc.communicate() - - if os.path.isdir(jitUtilsPath): - print("Deleting " + jitUtilsPath) - shutil.rmtree(jitUtilsPath, onerror=del_rw) - if os.path.isfile(bootstrapPath): - print("Deleting " + bootstrapPath) - os.remove(bootstrapPath) - - if returncode != 0: - print("There were errors in formatting. Please run jit-format locally with: \n") - print(errorMessage) - print("\nOr download and apply generated patch:") - print("1. From the GitHub 'Checks' page on the Pull Request, with the failing Formatting") - print(" job selected (e.g., 'Formatting Linux x64'), click the 'View more details on") - print(" Azure Pipelines' link.") - print("3. Select the 'Summary' tab.") - print("4. Open the 'Build artifacts published' entry.") - print("5. Find the link to the OS/architecture appropriate format patch file.") - print("6. Click on the link to download it.") - print("7. Unzip the patch file.") - print("8. git apply format.patch") - - if (returncode != 0) and (os.environ.get("TF_BUILD") == "True"): - print("##vso[task.logissue type=error](NETCORE_ENGINEERING_TELEMETRY=Build) Format job found errors, please apply the format patch.") - - return returncode + if os.path.isdir(jitUtilsPath): + print("Deleting " + jitUtilsPath) + shutil.rmtree(jitUtilsPath, onerror=del_rw) + + if os.path.isfile(bootstrapPath): + print("Deleting " + bootstrapPath) + os.remove(bootstrapPath) + + if returncode != 0: + print("There were errors in formatting. Please run jit-format locally with: \n") + print(errorMessage) + print("\nOr download and apply generated patch:") + print("1. From the GitHub 'Checks' page on the Pull Request, with the failing Formatting") + print(" job selected (e.g., 'Formatting Linux x64'), click the 'View more details on") + print(" Azure Pipelines' link.") + print("3. Select the 'Summary' tab.") + print("4. Open the 'Build artifacts published' entry.") + print("5. Find the link to the OS/architecture appropriate format patch file.") + print("6. Click on the link to download it.") + print("7. Unzip the patch file.") + print("8. git apply format.patch") + + if (returncode != 0) and (os.environ.get("TF_BUILD") == "True"): + print("##vso[task.logissue type=error](NETCORE_ENGINEERING_TELEMETRY=Build) Format job found errors, please apply the format patch.") + + return returncode if __name__ == '__main__': return_code = main(sys.argv[1:]) From 2448f1147bcb6979fada28190e94e9329a38d8e2 Mon Sep 17 00:00:00 2001 From: jashook Date: Wed, 13 Nov 2019 04:10:45 -0800 Subject: [PATCH 14/14] Remove funny try logic and rmtree --- src/coreclr/tests/scripts/format.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/coreclr/tests/scripts/format.py b/src/coreclr/tests/scripts/format.py index 85a59dfc1726c..bf34116831196 100644 --- a/src/coreclr/tests/scripts/format.py +++ b/src/coreclr/tests/scripts/format.py @@ -24,14 +24,10 @@ class ChangeDir: def __init__(self, dir): self.dir = dir - self.cwd = os.getcwd() + self.cwd = None def __enter__(self): - try: - self.cwd = os.getcwd() - except: - self.cwd = os.getcwd() - + self.cwd = os.getcwd() os.chdir(self.dir) def __exit__(self, exc_type, exc_val, exc_tb): @@ -50,7 +46,6 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): os.chdir(self.cwd) - shutil.rmtree(self.dir) # Version specific imports