From a1c74fa7eef73b15e7bd9a905c5250c69d7d4379 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 19 Nov 2024 22:53:13 +0100 Subject: [PATCH 01/10] SPMI: Switch back to unpinned 9.0 SDK for micro benchmarks --- src/coreclr/scripts/superpmi_collect_setup.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 84fca091e6ddaa..e97704ed5824db 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -404,11 +404,8 @@ def setup_benchmark(workitem_directory, arch): print("Missing " + dotnet_install_script) return - # Sometimes the dotnet version installed by the script is latest and expect certain versions of SDK that - # have not published yet. As a result, we hit errors of "dotnet restore". As a workaround, hard code the - # working version until we move to ".NET 9" in the script. run_command( - get_python_name() + [dotnet_install_script, "install", "--dotnet-versions", "9.0.100-rc.2.24474.11", "--architecture", arch, "--install-dir", + get_python_name() + [dotnet_install_script, "install", "--channels", "9.0", "--architecture", arch, "--install-dir", dotnet_directory, "--verbose"]) From 56384da3dc0f56c0f06bd8f89c7fdce0bffd0211 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 20 Nov 2024 20:11:41 +0100 Subject: [PATCH 02/10] Restore with target framework --- src/coreclr/scripts/superpmi_benchmarks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/coreclr/scripts/superpmi_benchmarks.py b/src/coreclr/scripts/superpmi_benchmarks.py index 117976ee00b021..a1573928182ff1 100644 --- a/src/coreclr/scripts/superpmi_benchmarks.py +++ b/src/coreclr/scripts/superpmi_benchmarks.py @@ -187,12 +187,13 @@ def build_and_run(coreclr_args, output_mch_name): env_copy["NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY"] = "9,2000" # If `dotnet restore` fails, retry. + tfm = "net9.0" num_tries = 3 for try_num in range(num_tries): # On the last try, exit on fail exit_on_fail = try_num + 1 == num_tries (_, _, return_code) = run_command( - [dotnet_exe, "restore", project_file, "--packages", artifacts_packages_directory], + [dotnet_exe, "restore", project_file, "--packages", artifacts_packages_directory, "-p:TargetFramework=" + tfm], _exit_on_fail=exit_on_fail, _env=env_copy) if return_code == 0: # It succeeded! @@ -203,7 +204,7 @@ def build_and_run(coreclr_args, output_mch_name): run_command( [dotnet_exe, "build", project_file, "--configuration", "Release", - "--framework", "net9.0", "--no-restore", "/p:NuGetPackageRoot=" + artifacts_packages_directory, + "--framework", tfm, "--no-restore", "/p:NuGetPackageRoot=" + artifacts_packages_directory, "-o", artifacts_directory], _exit_on_fail=True) # This is specifically for PowerShell.Benchmarks. From 1786b0e4de3083b2bbe10c949f76c28793fc8f63 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 21 Nov 2024 12:33:03 +0100 Subject: [PATCH 03/10] Also restore netstandard2.0 --- src/coreclr/scripts/superpmi_benchmarks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/scripts/superpmi_benchmarks.py b/src/coreclr/scripts/superpmi_benchmarks.py index a1573928182ff1..47eeb0deae9ce2 100644 --- a/src/coreclr/scripts/superpmi_benchmarks.py +++ b/src/coreclr/scripts/superpmi_benchmarks.py @@ -193,7 +193,7 @@ def build_and_run(coreclr_args, output_mch_name): # On the last try, exit on fail exit_on_fail = try_num + 1 == num_tries (_, _, return_code) = run_command( - [dotnet_exe, "restore", project_file, "--packages", artifacts_packages_directory, "-p:TargetFramework=" + tfm], + [dotnet_exe, "restore", project_file, "--packages", artifacts_packages_directory, "-p:TargetFrameworks=netstandard2.0;" + tfm], _exit_on_fail=exit_on_fail, _env=env_copy) if return_code == 0: # It succeeded! From 249c6e19855e3d1b65c6b0e6a78ea48bef551c60 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 22 Nov 2024 08:47:56 +0100 Subject: [PATCH 04/10] Invoke dotnet restore multiple times --- src/coreclr/scripts/superpmi_benchmarks.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/coreclr/scripts/superpmi_benchmarks.py b/src/coreclr/scripts/superpmi_benchmarks.py index 47eeb0deae9ce2..bb65190259af3c 100644 --- a/src/coreclr/scripts/superpmi_benchmarks.py +++ b/src/coreclr/scripts/superpmi_benchmarks.py @@ -188,17 +188,22 @@ def build_and_run(coreclr_args, output_mch_name): # If `dotnet restore` fails, retry. tfm = "net9.0" + tfms_to_restore = [tfm, "netstandard2.0"] num_tries = 3 for try_num in range(num_tries): # On the last try, exit on fail exit_on_fail = try_num + 1 == num_tries - (_, _, return_code) = run_command( - [dotnet_exe, "restore", project_file, "--packages", artifacts_packages_directory, "-p:TargetFrameworks=netstandard2.0;" + tfm], - _exit_on_fail=exit_on_fail, _env=env_copy) - if return_code == 0: - # It succeeded! + all_succeeded = True + for tfm_to_restore in tfms_to_restore: + (_, _, return_code) = run_command( + [dotnet_exe, "restore", project_file, "--packages", artifacts_packages_directory, "-p:TargetFramework=" + tfm_to_restore], + _exit_on_fail=exit_on_fail, _env=env_copy) + if return_code != 0: + all_succeeded = False + print("Restoring {} try {} of {} failed with error code {}: trying again".format(tfm_to_restore, try_num + 1, num_tries, return_code)) + + if all_succeeded: break - print("Try {} of {} failed with error code {}: trying again".format(try_num + 1, num_tries, return_code)) # Sleep 5 seconds before trying again time.sleep(5) From f580ec681151a81e1fb85c675e05edc69bd6ceb5 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 22 Nov 2024 12:07:48 +0100 Subject: [PATCH 05/10] Switch to a retrying dotnet build --- src/coreclr/scripts/superpmi_benchmarks.py | 26 ++++++++-------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/coreclr/scripts/superpmi_benchmarks.py b/src/coreclr/scripts/superpmi_benchmarks.py index bb65190259af3c..61f9874dc35992 100644 --- a/src/coreclr/scripts/superpmi_benchmarks.py +++ b/src/coreclr/scripts/superpmi_benchmarks.py @@ -187,31 +187,23 @@ def build_and_run(coreclr_args, output_mch_name): env_copy["NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY"] = "9,2000" # If `dotnet restore` fails, retry. - tfm = "net9.0" - tfms_to_restore = [tfm, "netstandard2.0"] num_tries = 3 for try_num in range(num_tries): # On the last try, exit on fail exit_on_fail = try_num + 1 == num_tries - all_succeeded = True - for tfm_to_restore in tfms_to_restore: - (_, _, return_code) = run_command( - [dotnet_exe, "restore", project_file, "--packages", artifacts_packages_directory, "-p:TargetFramework=" + tfm_to_restore], - _exit_on_fail=exit_on_fail, _env=env_copy) - if return_code != 0: - all_succeeded = False - print("Restoring {} try {} of {} failed with error code {}: trying again".format(tfm_to_restore, try_num + 1, num_tries, return_code)) - - if all_succeeded: + (_, _, return_code) = run_command( + [dotnet_exe, "build", project_file, "--configuration", "Release", + "--framework", "net9.0", "/p:NuGetPackageRoot=" + artifacts_packages_directory, + "-o", artifacts_directory], _exit_on_fail=exit_on_fail) + + if return_code == 0: break + + print("Build try {} of {} failed with error code {}: trying again".format(try_num + 1, num_tries, return_code)) + # Sleep 5 seconds before trying again time.sleep(5) - run_command( - [dotnet_exe, "build", project_file, "--configuration", "Release", - "--framework", tfm, "--no-restore", "/p:NuGetPackageRoot=" + artifacts_packages_directory, - "-o", artifacts_directory], _exit_on_fail=True) - # This is specifically for PowerShell.Benchmarks. # Because we are using '--corerun' when running the benchmarks, it is not able to resolve the PowerShell dependent assemblies. # To make this work, we create a directory called 'core_root' in the directory where the benchmarks_dll lives. From 7029061edd1601f16084ca452ec3aee269660144 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 22 Nov 2024 12:09:57 +0100 Subject: [PATCH 06/10] Nit --- src/coreclr/scripts/superpmi_benchmarks.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/coreclr/scripts/superpmi_benchmarks.py b/src/coreclr/scripts/superpmi_benchmarks.py index 61f9874dc35992..ce13f7b1600533 100644 --- a/src/coreclr/scripts/superpmi_benchmarks.py +++ b/src/coreclr/scripts/superpmi_benchmarks.py @@ -197,10 +197,9 @@ def build_and_run(coreclr_args, output_mch_name): "-o", artifacts_directory], _exit_on_fail=exit_on_fail) if return_code == 0: + # It succeeded! break - - print("Build try {} of {} failed with error code {}: trying again".format(try_num + 1, num_tries, return_code)) - + print("Try {} of {} failed with error code {}: trying again".format(try_num + 1, num_tries, return_code)) # Sleep 5 seconds before trying again time.sleep(5) From c02446644d147998f682911256b8278bc8b6c04c Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Nov 2024 16:25:50 +0100 Subject: [PATCH 07/10] Set PERFLAB_TARGET_FRAMEWORKS --- src/coreclr/scripts/superpmi_benchmarks.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/coreclr/scripts/superpmi_benchmarks.py b/src/coreclr/scripts/superpmi_benchmarks.py index ce13f7b1600533..d74b6385b3f97e 100644 --- a/src/coreclr/scripts/superpmi_benchmarks.py +++ b/src/coreclr/scripts/superpmi_benchmarks.py @@ -186,16 +186,18 @@ def build_and_run(coreclr_args, output_mch_name): # Using environment variable specified in https://github.com/NuGet/NuGet.Client/pull/4259. env_copy["NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY"] = "9,2000" + + tfm = "net9.0" + env_copy["PERFLAB_TARGET_FRAMEWORKS"] = tfm + # If `dotnet restore` fails, retry. num_tries = 3 for try_num in range(num_tries): # On the last try, exit on fail exit_on_fail = try_num + 1 == num_tries (_, _, return_code) = run_command( - [dotnet_exe, "build", project_file, "--configuration", "Release", - "--framework", "net9.0", "/p:NuGetPackageRoot=" + artifacts_packages_directory, - "-o", artifacts_directory], _exit_on_fail=exit_on_fail) - + [dotnet_exe, "restore", project_file, "--packages", artifacts_packages_directory], + _exit_on_fail=exit_on_fail, _env=env_copy) if return_code == 0: # It succeeded! break @@ -203,6 +205,11 @@ def build_and_run(coreclr_args, output_mch_name): # Sleep 5 seconds before trying again time.sleep(5) + run_command( + [dotnet_exe, "build", project_file, "--configuration", "Release", + "--framework", tfm, "--no-restore", "/p:NuGetPackageRoot=" + artifacts_packages_directory, + "-o", artifacts_directory], _exit_on_fail=True) + # This is specifically for PowerShell.Benchmarks. # Because we are using '--corerun' when running the benchmarks, it is not able to resolve the PowerShell dependent assemblies. # To make this work, we create a directory called 'core_root' in the directory where the benchmarks_dll lives. From 13a148a96143cfd4775722f6e70dce0cd2a1f0e5 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Nov 2024 16:29:18 +0100 Subject: [PATCH 08/10] Set env var for both invocations --- src/coreclr/scripts/superpmi_benchmarks.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/coreclr/scripts/superpmi_benchmarks.py b/src/coreclr/scripts/superpmi_benchmarks.py index d74b6385b3f97e..31e0fe0dbbba08 100644 --- a/src/coreclr/scripts/superpmi_benchmarks.py +++ b/src/coreclr/scripts/superpmi_benchmarks.py @@ -178,17 +178,17 @@ def build_and_run(coreclr_args, output_mch_name): # Start with a "dotnet --info" to see what we've got. run_command([dotnet_exe, "--info"]) - env_copy = os.environ.copy() + env = os.environ.copy() + + tfm = "net9.0" + env["PERFLAB_TARGET_FRAMEWORKS"] = tfm + if is_windows: # Try to work around problem with random NuGet failures in "dotnet restore": # error NU3037: Package 'System.Runtime 4.1.0' from source 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json': # The repository primary signature validity period has expired. [C:\h\w\A3B008C0\w\B581097F\u\performance\src\benchmarks\micro\MicroBenchmarks.csproj] # Using environment variable specified in https://github.com/NuGet/NuGet.Client/pull/4259. - env_copy["NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY"] = "9,2000" - - - tfm = "net9.0" - env_copy["PERFLAB_TARGET_FRAMEWORKS"] = tfm + env["NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY"] = "9,2000" # If `dotnet restore` fails, retry. num_tries = 3 @@ -197,7 +197,7 @@ def build_and_run(coreclr_args, output_mch_name): exit_on_fail = try_num + 1 == num_tries (_, _, return_code) = run_command( [dotnet_exe, "restore", project_file, "--packages", artifacts_packages_directory], - _exit_on_fail=exit_on_fail, _env=env_copy) + _exit_on_fail=exit_on_fail, _env=env) if return_code == 0: # It succeeded! break @@ -208,7 +208,7 @@ def build_and_run(coreclr_args, output_mch_name): run_command( [dotnet_exe, "build", project_file, "--configuration", "Release", "--framework", tfm, "--no-restore", "/p:NuGetPackageRoot=" + artifacts_packages_directory, - "-o", artifacts_directory], _exit_on_fail=True) + "-o", artifacts_directory], _exit_on_fail=True, _env=env) # This is specifically for PowerShell.Benchmarks. # Because we are using '--corerun' when running the benchmarks, it is not able to resolve the PowerShell dependent assemblies. From 28d25245ca249a067f82159fdadfab7d3cb34fc3 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Nov 2024 18:04:46 +0100 Subject: [PATCH 09/10] Set PERFLAB_TARGET_FRAMEWORKS globally in environment --- src/coreclr/scripts/superpmi_benchmarks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/coreclr/scripts/superpmi_benchmarks.py b/src/coreclr/scripts/superpmi_benchmarks.py index 31e0fe0dbbba08..ef0ca68a3dd9af 100644 --- a/src/coreclr/scripts/superpmi_benchmarks.py +++ b/src/coreclr/scripts/superpmi_benchmarks.py @@ -178,17 +178,17 @@ def build_and_run(coreclr_args, output_mch_name): # Start with a "dotnet --info" to see what we've got. run_command([dotnet_exe, "--info"]) - env = os.environ.copy() - tfm = "net9.0" - env["PERFLAB_TARGET_FRAMEWORKS"] = tfm + os.environ["PERFLAB_TARGET_FRAMEWORKS"] = tfm + + env_for_restore = os.environ.copy() if is_windows: # Try to work around problem with random NuGet failures in "dotnet restore": # error NU3037: Package 'System.Runtime 4.1.0' from source 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json': # The repository primary signature validity period has expired. [C:\h\w\A3B008C0\w\B581097F\u\performance\src\benchmarks\micro\MicroBenchmarks.csproj] # Using environment variable specified in https://github.com/NuGet/NuGet.Client/pull/4259. - env["NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY"] = "9,2000" + env_for_restore["NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY"] = "9,2000" # If `dotnet restore` fails, retry. num_tries = 3 @@ -197,7 +197,7 @@ def build_and_run(coreclr_args, output_mch_name): exit_on_fail = try_num + 1 == num_tries (_, _, return_code) = run_command( [dotnet_exe, "restore", project_file, "--packages", artifacts_packages_directory], - _exit_on_fail=exit_on_fail, _env=env) + _exit_on_fail=exit_on_fail, _env=env_for_restore) if return_code == 0: # It succeeded! break @@ -208,7 +208,7 @@ def build_and_run(coreclr_args, output_mch_name): run_command( [dotnet_exe, "build", project_file, "--configuration", "Release", "--framework", tfm, "--no-restore", "/p:NuGetPackageRoot=" + artifacts_packages_directory, - "-o", artifacts_directory], _exit_on_fail=True, _env=env) + "-o", artifacts_directory], _exit_on_fail=True) # This is specifically for PowerShell.Benchmarks. # Because we are using '--corerun' when running the benchmarks, it is not able to resolve the PowerShell dependent assemblies. From aa002fc6f390cee51c06a3e89170d9a6498c6cad Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 26 Nov 2024 11:59:54 +0100 Subject: [PATCH 10/10] Fix collect --- src/coreclr/scripts/superpmi.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/coreclr/scripts/superpmi.py b/src/coreclr/scripts/superpmi.py index f2b81a06e39fdd..ca3e0dd6101457 100644 --- a/src/coreclr/scripts/superpmi.py +++ b/src/coreclr/scripts/superpmi.py @@ -330,6 +330,8 @@ def add_core_root_arguments(parser, build_type_default, build_type_help): replay_common_parser = argparse.ArgumentParser(add_help=False) +# NOTE: When adding arguments here, also make sure that they are set when collect does its replay: +# see the "collect" case in setup_args. replay_common_parser.add_argument("-mch_files", metavar="MCH_FILE", nargs='+', help=replay_mch_files_help) replay_common_parser.add_argument("-filter", nargs='+', help=filter_help) replay_common_parser.add_argument("-product_location", help=product_location_help) @@ -4754,6 +4756,11 @@ def verify_base_diff_args(): lambda unused: True, "Unable to set produce_repro") + coreclr_args.verify(args, + "details", # The replay code checks this, so make sure it's set + lambda unused: True, + "Unable to set details") + coreclr_args.verify(args, "collection_command", lambda unused: True,