From 201a82167740232d96fa11c7eed2b02b204936b8 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Thu, 16 May 2024 18:57:54 +0100 Subject: [PATCH 1/7] Add bm_runner 'trialrun' subcommand. --- benchmarks/bm_runner.py | 61 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/benchmarks/bm_runner.py b/benchmarks/bm_runner.py index 2c18de4a41..dd17cc2f26 100644 --- a/benchmarks/bm_runner.py +++ b/benchmarks/bm_runner.py @@ -537,6 +537,65 @@ def func(args: argparse.Namespace) -> None: _subprocess_runner([args.asv_sub_command, *args.asv_args], asv=True) +class TrialRun(_SubParserGenerator): + name = "trialrun" + description = ( + "Fast trial-run a given benchmark, to check it works : " + "in a provided or latest-lockfile environment, " + "with no repeats for accuracy of measurement." + ) + epilog = ( + "e.g. python bm_runner.py trialrun " + "MyBenchmarks.time_calc /tmp/testpython/bin/python" + "\n NOTE: setting $DATA_GEN_PYTHON is equivalent to the " + "'runpath' argument." + ) + + def add_arguments(self) -> None: + self.subparser.add_argument( + "benchmark", + type=str, + help=( + "A benchmark name, possibly including wildcards, " + "as supported by the ASV '--benchmark' argument." + ) + ) + self.subparser.add_argument( + "runpath", + type=str, + nargs="?", + help=( + "A path to an existing python environment, " + "to completely bypass environment building." + ) + ) + + @staticmethod + def func(args: argparse.Namespace) -> None: + print(args) + if args.runpath: + # Shortcut creation of a data-gen environment + # - which is also the trial-run env. + environ["DATA_GEN_PYTHON"] = args.runpath + _setup_common() + # get path of data-gen environment, setup by previous call + python_path = environ["DATA_GEN_PYTHON"] + # allow 'on-demand' benchmarks + environ["ON_DEMAND_BENCHMARKS"] = "1" + asv_command = [ + "run", + "--bench", + args.benchmark, + # no repeats for timing accuracy + "--quick", + # show any errors + "-e", + # do not build a unique env : run test in data-gen environment + "--environment", f"existing:{python_path}" + ] + args.asv_args + _subprocess_runner(asv_command, asv=True) + + class GhPost(_SubParserGenerator): name = "_gh_post" description = ( @@ -566,7 +625,7 @@ def main(): ) subparsers = parser.add_subparsers(required=True) - for gen in (Overnight, Branch, CPerf, SPerf, Custom, GhPost): + for gen in (Overnight, Branch, CPerf, SPerf, Custom, TrialRun, GhPost): _ = gen(subparsers).subparser parsed = parser.parse_args() From d8b13b8f28b528cc3ffb94496ffa5a1abe80f2ba Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Fri, 17 May 2024 11:53:59 +0100 Subject: [PATCH 2/7] Remove debug. --- benchmarks/bm_runner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/bm_runner.py b/benchmarks/bm_runner.py index dd17cc2f26..18d2f7fab4 100644 --- a/benchmarks/bm_runner.py +++ b/benchmarks/bm_runner.py @@ -558,7 +558,7 @@ def add_arguments(self) -> None: help=( "A benchmark name, possibly including wildcards, " "as supported by the ASV '--benchmark' argument." - ) + ), ) self.subparser.add_argument( "runpath", @@ -567,12 +567,11 @@ def add_arguments(self) -> None: help=( "A path to an existing python environment, " "to completely bypass environment building." - ) + ), ) @staticmethod def func(args: argparse.Namespace) -> None: - print(args) if args.runpath: # Shortcut creation of a data-gen environment # - which is also the trial-run env. @@ -591,7 +590,8 @@ def func(args: argparse.Namespace) -> None: # show any errors "-e", # do not build a unique env : run test in data-gen environment - "--environment", f"existing:{python_path}" + "--environment", + f"existing:{python_path}", ] + args.asv_args _subprocess_runner(asv_command, asv=True) From ab09e95760eae1f0e7060316d9480d54a1220d38 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Fri, 17 May 2024 15:33:24 +0100 Subject: [PATCH 3/7] Review changes. --- benchmarks/bm_runner.py | 18 +++++++++++------- noxfile.py | 4 ++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/benchmarks/bm_runner.py b/benchmarks/bm_runner.py index 18d2f7fab4..322906c3d8 100644 --- a/benchmarks/bm_runner.py +++ b/benchmarks/bm_runner.py @@ -325,7 +325,7 @@ def add_arguments(self) -> None: def add_asv_arguments(self) -> None: self.subparser.add_argument( "asv_args", - nargs=argparse.REMAINDER, + nargs="*", help="Any number of arguments to pass down to the ASV benchmark command.", ) @@ -547,8 +547,12 @@ class TrialRun(_SubParserGenerator): epilog = ( "e.g. python bm_runner.py trialrun " "MyBenchmarks.time_calc /tmp/testpython/bin/python" - "\n NOTE: setting $DATA_GEN_PYTHON is equivalent to the " + "\n\n NOTE#1: setting $DATA_GEN_PYTHON is equivalent to the " "'runpath' argument." + "\n NOTE#2: setting $OVERRIDE_TEST_DATA_REPOSITORY avoids the runner " + "installing iris-test-data." + "\n NOTE#3: setting $BENCHMARK_DATA may be desirable to specify " + "where is safe to create potentially large (Gb) test data." ) def add_arguments(self) -> None: @@ -557,7 +561,7 @@ def add_arguments(self) -> None: type=str, help=( "A benchmark name, possibly including wildcards, " - "as supported by the ASV '--benchmark' argument." + "as supported by the ASV '--bench' argument." ), ) self.subparser.add_argument( @@ -565,7 +569,7 @@ def add_arguments(self) -> None: type=str, nargs="?", help=( - "A path to an existing python environment, " + "A path to an existing python executable, " "to completely bypass environment building." ), ) @@ -575,7 +579,8 @@ def func(args: argparse.Namespace) -> None: if args.runpath: # Shortcut creation of a data-gen environment # - which is also the trial-run env. - environ["DATA_GEN_PYTHON"] = args.runpath + python_path = Path(args.runpath).resolve() + environ["DATA_GEN_PYTHON"] = str(python_path) _setup_common() # get path of data-gen environment, setup by previous call python_path = environ["DATA_GEN_PYTHON"] @@ -587,8 +592,7 @@ def func(args: argparse.Namespace) -> None: args.benchmark, # no repeats for timing accuracy "--quick", - # show any errors - "-e", + "--show-stderr", # do not build a unique env : run test in data-gen environment "--environment", f"existing:{python_path}", diff --git a/noxfile.py b/noxfile.py index 770298d319..d74a964e94 100644 --- a/noxfile.py +++ b/noxfile.py @@ -313,5 +313,5 @@ def benchmarks(session: nox.sessions.Session): ) session.error(message) session.install("asv", "nox") - with session.chdir(Path(__file__).parent / "benchmarks"): - session.run("python", "bm_runner.py", *session.posargs) + bm_runner_path = Path(__file__).parent / "benchmarks" / "bm_runner.py" + session.run("python", bm_runner_path, *session.posargs) From dcd39827d1f6ac122f7d52fb6e438d01588847a0 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Sun, 19 May 2024 00:38:44 +0100 Subject: [PATCH 4/7] Reinstate use of argparse.REMAINDER --- benchmarks/bm_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/bm_runner.py b/benchmarks/bm_runner.py index 322906c3d8..fb2e2a171a 100644 --- a/benchmarks/bm_runner.py +++ b/benchmarks/bm_runner.py @@ -325,7 +325,7 @@ def add_arguments(self) -> None: def add_asv_arguments(self) -> None: self.subparser.add_argument( "asv_args", - nargs="*", + nargs=argparse.REMAINDER, help="Any number of arguments to pass down to the ASV benchmark command.", ) From 7bae96bc31673cfbaef7a65d694367d5ce6f6567 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 20 May 2024 11:21:43 +0100 Subject: [PATCH 5/7] Make trialrun 'runpath' arg non-optional. --- benchmarks/bm_runner.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/benchmarks/bm_runner.py b/benchmarks/bm_runner.py index fb2e2a171a..04401dfba9 100644 --- a/benchmarks/bm_runner.py +++ b/benchmarks/bm_runner.py @@ -547,8 +547,8 @@ class TrialRun(_SubParserGenerator): epilog = ( "e.g. python bm_runner.py trialrun " "MyBenchmarks.time_calc /tmp/testpython/bin/python" - "\n\n NOTE#1: setting $DATA_GEN_PYTHON is equivalent to the " - "'runpath' argument." + "\n NOTE: 'runpath' is also used for any data-generation, i.e. it" + "replaces $DATA_GEN_PYTHON during this run." "\n NOTE#2: setting $OVERRIDE_TEST_DATA_REPOSITORY avoids the runner " "installing iris-test-data." "\n NOTE#3: setting $BENCHMARK_DATA may be desirable to specify " @@ -567,7 +567,6 @@ def add_arguments(self) -> None: self.subparser.add_argument( "runpath", type=str, - nargs="?", help=( "A path to an existing python executable, " "to completely bypass environment building." From 5847a7a35523c0244e42790adfb3103372cca6ac Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 20 May 2024 12:10:09 +0100 Subject: [PATCH 6/7] Better help documentation. --- benchmarks/bm_runner.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/benchmarks/bm_runner.py b/benchmarks/bm_runner.py index 04401dfba9..afa8b6ff29 100644 --- a/benchmarks/bm_runner.py +++ b/benchmarks/bm_runner.py @@ -547,12 +547,7 @@ class TrialRun(_SubParserGenerator): epilog = ( "e.g. python bm_runner.py trialrun " "MyBenchmarks.time_calc /tmp/testpython/bin/python" - "\n NOTE: 'runpath' is also used for any data-generation, i.e. it" - "replaces $DATA_GEN_PYTHON during this run." - "\n NOTE#2: setting $OVERRIDE_TEST_DATA_REPOSITORY avoids the runner " - "installing iris-test-data." - "\n NOTE#3: setting $BENCHMARK_DATA may be desirable to specify " - "where is safe to create potentially large (Gb) test data." + "\n\nNOTE: 'runpath' also replaces $DATA_GEN_PYTHON during the run." ) def add_arguments(self) -> None: @@ -624,7 +619,20 @@ def add_asv_arguments(self) -> None: def main(): parser = ArgumentParser( description="Run the Iris performance benchmarks (using Airspeed Velocity).", - epilog="More help is available within each sub-command.", + epilog=( + "More help is available within each sub-command." + "\n\nNOTE(1): iris-test-data is downloaded and cached within the " + "benchmarks code directory.\n Set $OVERRIDE_TEST_DATA_REPOSITORY " + "to avoid the cost of this." + "\nNOTE(2): a separate python environment is created to write " + "benchmark test data.\n Set $DATA_GEN_PYTHON to avoid the cost " + "of this." + "\nNOTE(3): test data is cached within the " + "benchmarks code directory, and uses a lot of disk space " + "of disk space (Gb).\n Set $BENCHMARK_DATA to specify where this " + "space can be safely allocated." + ), + formatter_class=argparse.RawTextHelpFormatter, ) subparsers = parser.add_subparsers(required=True) From ca21d914788b5a2caa03859af4ac576308e3c0c9 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 20 May 2024 14:33:13 +0100 Subject: [PATCH 7/7] Review changes. --- benchmarks/bm_runner.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/benchmarks/bm_runner.py b/benchmarks/bm_runner.py index afa8b6ff29..116a7cf193 100644 --- a/benchmarks/bm_runner.py +++ b/benchmarks/bm_runner.py @@ -546,7 +546,7 @@ class TrialRun(_SubParserGenerator): ) epilog = ( "e.g. python bm_runner.py trialrun " - "MyBenchmarks.time_calc /tmp/testpython/bin/python" + "MyBenchmarks.time_calc ${DATA_GEN_PYTHON}" "\n\nNOTE: 'runpath' also replaces $DATA_GEN_PYTHON during the run." ) @@ -621,12 +621,12 @@ def main(): description="Run the Iris performance benchmarks (using Airspeed Velocity).", epilog=( "More help is available within each sub-command." - "\n\nNOTE(1): iris-test-data is downloaded and cached within the " - "benchmarks code directory.\n Set $OVERRIDE_TEST_DATA_REPOSITORY " - "to avoid the cost of this." - "\nNOTE(2): a separate python environment is created to write " - "benchmark test data.\n Set $DATA_GEN_PYTHON to avoid the cost " + "\n\nNOTE(1): a separate python environment is created to " + "construct test files.\n Set $DATA_GEN_PYTHON to avoid the cost " "of this." + "\nNOTE(2): iris-test-data is downloaded and cached within the " + "data generation environment.\n Set " + "$OVERRIDE_TEST_DATA_REPOSITORY to avoid the cost of this." "\nNOTE(3): test data is cached within the " "benchmarks code directory, and uses a lot of disk space " "of disk space (Gb).\n Set $BENCHMARK_DATA to specify where this "