Skip to content

Commit 2061de3

Browse files
chris-eiblseehwan80
chris-eibl
authored and
seehwan80
committed
pythongh-130090: Support PGO for clang-cl on Windows (pythonGH-129907)
1 parent db4acd4 commit 2061de3

File tree

7 files changed

+99
-5
lines changed

7 files changed

+99
-5
lines changed

‎Misc/ACKS‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -504,6 +504,7 @@ Grant Edwards
504504
Vlad Efanov
505505
Zvi Effron
506506
John Ehresman
507+
Chris Eibl
507508
Tal Einat
508509
Eric Eisner
509510
Andrew Eland

‎Misc/NEWS.d/next/Build/2025-02-13-19-21-41.gh-issue-130090.3ngJaV.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Building with ``PlatformToolset=ClangCL`` on Windows now supports PGO
2+
(profile guided optimization). Patch by Chris Eibl with invaluable support from Steve Dover.

‎PCbuild/_freeze_module.vcxproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@
9292
<AdditionalIncludeDirectories>$(IntDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
9393
<Optimization>Disabled</Optimization>
9494
<WholeProgramOptimization>false</WholeProgramOptimization>
95+
<AdditionalOptions Condition="$(PlatformToolset) == 'ClangCL'">%(AdditionalOptions) -fno-lto</AdditionalOptions>
9596
</ClCompile>
9697
<Link>
9798
<SubSystem>Console</SubSystem>

‎PCbuild/pyproject-clangcl.props‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
<PropertyGroup Label="Globals">
4+
<__PyprojectClangCl_Props_Imported>true</__PyprojectClangCl_Props_Imported>
5+
</PropertyGroup>
6+
7+
<PropertyGroup>
8+
<!-- CLANG_PROFILE_PATH is configurable for "remote PGO builds"
9+
For convenience, we also accept paths without trailing slashes.
10+
-->
11+
<CLANG_PROFILE_PATH Condition="'$(CLANG_PROFILE_PATH)' == ''">$(OutDir)</CLANG_PROFILE_PATH>
12+
<_CLANG_PROFILE_PATH>$(CLANG_PROFILE_PATH)</_CLANG_PROFILE_PATH>
13+
<_CLANG_PROFILE_PATH Condition="!HasTrailingSlash($(_CLANG_PROFILE_PATH))">$(_CLANG_PROFILE_PATH)\</_CLANG_PROFILE_PATH>
14+
</PropertyGroup>
15+
16+
<ItemGroup>
17+
<_profrawFiles Include="$(OutDir)instrumented\$(TargetName)_*.profraw" />
18+
</ItemGroup>
19+
20+
<Target Name="EnsureClangProfileData" BeforeTargets="PrepareForBuild"
21+
Condition="'$(SupportPGO)' and $(Configuration) == 'PGUpdate'">
22+
<Error Text="PGO run did not succeed (no $(TargetName)_*.profraw files) and there is no data to merge"
23+
Condition="$(RequireProfileData) == 'true' and @(_profrawFiles) == ''" />
24+
</Target>
25+
26+
<Target Name="MergeClangProfileData" BeforeTargets="PrepareForBuild"
27+
Condition="'$(SupportPGO)' and $(Configuration) == 'PGUpdate'"
28+
Inputs="@(_profrawFiles)"
29+
Outputs="$(OutDir)instrumented\profdata.profdata">
30+
<Exec
31+
Command='"$(LLVMInstallDir)\bin\llvm-profdata.exe" merge -output="$(OutDir)instrumented\profdata.profdata" "$(OutDir)instrumented\*_*.profraw"' />
32+
</Target>
33+
34+
<Target Name="CleanClangProfileData" BeforeTargets="Clean">
35+
<Delete Files="@(_profrawFiles)" TreatErrorsAsWarnings="true" />
36+
<Delete Files="$(OutDir)instrumented\profdata.profdata" TreatErrorsAsWarnings="true" />
37+
</Target>
38+
39+
<ItemDefinitionGroup>
40+
<ClCompile>
41+
<AdditionalOptions>-Wno-deprecated-non-prototype -Wno-unused-label -Wno-pointer-sign -Wno-incompatible-pointer-types-discards-qualifiers -Wno-unused-function %(AdditionalOptions)</AdditionalOptions>
42+
<AdditionalOptions Condition="'$(Platform)' == 'Win32'">-m32 %(AdditionalOptions)</AdditionalOptions>
43+
<AdditionalOptions Condition="'$(Platform)' == 'x64'">-m64 %(AdditionalOptions)</AdditionalOptions>
44+
<AdditionalOptions Condition="$(Configuration) != 'Debug'">-flto %(AdditionalOptions)</AdditionalOptions>
45+
<AdditionalOptions Condition="$(SupportPGO) and $(Configuration) == 'PGInstrument'">-fprofile-instr-generate=$(_CLANG_PROFILE_PATH)$(TargetName)_%m.profraw %(AdditionalOptions)</AdditionalOptions>
46+
<AdditionalOptions Condition="$(SupportPGO) and $(Configuration) == 'PGUpdate'">-fprofile-instr-use=$(OutDir)instrumented\profdata.profdata -Wno-profile-instr-unprofiled %(AdditionalOptions)</AdditionalOptions>
47+
</ClCompile>
48+
</ItemDefinitionGroup>
49+
50+
</Project>

‎PCbuild/pyproject.props‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@
2424
<LinkIncremental Condition="$(Configuration) != 'Debug'">false</LinkIncremental>
2525
</PropertyGroup>
2626

27+
<!-- We need the above overridden OutDir, so this must be imported after PropertyGroup -->
28+
<Import Project="pyproject-clangcl.props" Condition="$(PlatformToolset) == 'ClangCL' and $(__PyprojectClangCl_Props_Imported) != 'true'" />
29+
2730
<PropertyGroup Condition="$(TargetExt) != ''">
2831
<TargetNameExt>$(TargetName)$(TargetExt)</TargetNameExt>
2932
<_TargetNameSep>$(TargetNameExt.LastIndexOf(`.`))</_TargetNameSep>
@@ -69,8 +72,6 @@
6972
<ControlFlowGuard Condition="$(EnableControlFlowGuard) != ''">$(EnableControlFlowGuard)</ControlFlowGuard>
7073
<MultiProcessorCompilation>true</MultiProcessorCompilation>
7174
<AdditionalOptions>/utf-8 %(AdditionalOptions)</AdditionalOptions>
72-
<AdditionalOptions Condition="$(PlatformToolset) == 'ClangCL'">-Wno-deprecated-non-prototype -Wno-unused-label -Wno-pointer-sign -Wno-incompatible-pointer-types-discards-qualifiers -Wno-unused-function %(AdditionalOptions)</AdditionalOptions>
73-
<AdditionalOptions Condition="$(Configuration) != 'Debug' and $(PlatformToolset) == 'ClangCL'">-flto %(AdditionalOptions)</AdditionalOptions>
7475
<AdditionalOptions Condition="$(MSVCHasBrokenARM64Clamping) == 'true' and $(Platform) == 'ARM64'">-d2pattern-opt-disable:-932189325 %(AdditionalOptions)</AdditionalOptions>
7576
<AdditionalOptions Condition="$(MSVCHasBrokenARM64SignExtension) == 'true' and $(Platform) == 'ARM64'">-d2ssa-patterns-all- %(AdditionalOptions)</AdditionalOptions>
7677
<AdditionalOptions Condition="$(GenerateSourceDependencies) == 'true'">/sourceDependencies "$(IntDir.Trim(`\`))" %(AdditionalOptions)</AdditionalOptions>
@@ -186,15 +187,15 @@ public override bool Execute() {
186187
Targets="CleanAll" />
187188
</Target>
188189

189-
<Target Name="CopyPGCFiles" BeforeTargets="PrepareForBuild" Condition="$(Configuration) == 'PGUpdate'">
190+
<Target Name="CopyPGCFiles" BeforeTargets="PrepareForBuild" Condition="$(Configuration) == 'PGUpdate' and $(PlatformToolset) != 'ClangCL'">
190191
<ItemGroup>
191192
<_PGCFiles Include="$(OutDir)instrumented\$(TargetName)!*.pgc" />
192193
<_PGDFile Include="$(OutDir)instrumented\$(TargetName).pgd" />
193194
<_CopyFiles Include="@(_PGCFiles);@(_PGDFile)" Condition="Exists(%(FullPath))" />
194195
</ItemGroup>
195196
<Delete Files="@(_CopyFiles->'$(OutDir)%(Filename)%(Extension)')" />
196197
<Error Text="PGO run did not succeed (no $(TargetName)!*.pgc files) and there is no data to merge"
197-
Condition="$(RequirePGCFiles) == 'true' and @(_PGCFiles) == ''" />
198+
Condition="$(RequireProfileData) == 'true' and @(_PGCFiles) == ''" />
198199
<Copy SourceFiles="@(_CopyFiles)"
199200
DestinationFolder="$(OutDir)"
200201
UseHardLinksIfPossible="true"

‎PCbuild/pythoncore.vcxproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@
8181
</ImportGroup>
8282
<PropertyGroup>
8383
<KillPython>true</KillPython>
84-
<RequirePGCFiles>true</RequirePGCFiles>
84+
<RequireProfileData>true</RequireProfileData>
8585
<IncludeExternals Condition="$(IncludeExternals) == '' and Exists('$(zlibDir)\zlib.h')">true</IncludeExternals>
8686
<IncludeExternals Condition="$(IncludeExternals) == ''">false</IncludeExternals>
8787
</PropertyGroup>

‎PCbuild/readme.txt‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,45 @@ Release
5252
settings, though without PGO.
5353

5454

55+
Building Python using Clang/LLVM
56+
--------------------------------
57+
58+
See https://learn.microsoft.com/cpp/build/clang-support-msbuild
59+
for how to install and use clang-cl bundled with Microsoft Visual Studio.
60+
You can use the IDE to switch to clang-cl for local development,
61+
but because this alters the *.vcxproj files, the recommended way is
62+
to use build.bat:
63+
64+
build.bat "/p:PlatformToolset=ClangCL"
65+
66+
All other build.bat options continue to work as with MSVC, so this
67+
will create a 64bit release binary.
68+
69+
You can also use a specific version of clang-cl downloaded from
70+
https://github.com/llvm/llvm-project/releases, e.g.
71+
clang+llvm-18.1.8-x86_64-pc-windows-msvc.tar.xz.
72+
Given you have extracted that to <my-clang-dir>, you can use it like so
73+
build.bat --pgo "/p:PlatformToolset=ClangCL" "/p:LLVMInstallDir=<my-clang-dir> "/p:LLVMToolsVersion=18"
74+
75+
Setting LLVMToolsVersion to the major version is enough, although you
76+
can be specific and use 18.1.8 in the above example, too.
77+
78+
Use the --pgo option to build with PGO (Profile Guided Optimization).
79+
80+
However, if you want to run the PGO task
81+
on a different host than the build host, you must pass
82+
"/p:CLANG_PROFILE_PATH=<relative-path-to-instrumented-dir-on-remote-host>"
83+
in the PGInstrument step to make sure the profile data is generated
84+
into the instrumented directory when running the PGO task.
85+
E.g., if you place the instrumented binaries into the folder
86+
"workdir/instrumented" and then run the PGO task using "workdir"
87+
as the current working directory, the usage is
88+
"/p:CLANG_PROFILE_PATH=instrumented"
89+
90+
Like in the MSVC case, after fetching (or manually copying) the instrumented
91+
folder back into your build tree, you can continue with the PGUpdate
92+
step with no further parameters.
93+
5594
Building Python using the build.bat script
5695
----------------------------------------------
5796

0 commit comments

Comments
 (0)