Skip to content

Add bm_runner 'trialrun' subcommand. #5957

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
trexfeathers merged 8 commits into from
May 20, 2024
Merged

Add bm_runner 'trialrun' subcommand. #5957

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
201a821
Add bm_runner 'trialrun' subcommand.
pp-mo May 16, 2024
d8b13b8
Remove debug.
pp-mo May 17, 2024
ab09e95
Review changes.
pp-mo May 17, 2024
dcd3982
Reinstate use of argparse.REMAINDER
pp-mo May 18, 2024
7bae96b
Make trialrun 'runpath' arg non-optional.
pp-mo May 20, 2024
5847a7a
Better help documentation.
pp-mo May 20, 2024
ca21d91
Review changes.
pp-mo May 20, 2024
b5d45a2
Merge branch 'main' into benchmark_trialrun
trexfeathers May 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 72 additions & 2 deletions benchmarks/bm_runner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -544,6 +544,63 @@ def func(args: argparse.Namespace) -> None:
_subprocess_runner([args.asv_sub_command, *args.asv_args], asv=True)


class TrialRun(_SubParserGenerator):
name = "trialrun"
description = (
"Fast trial-run a given benchmark, to check it works : "
"in a provided or latest-lockfile environment, "
"with no repeats for accuracy of measurement."
)
epilog = (
"e.g. python bm_runner.py trialrun "
"MyBenchmarks.time_calc ${DATA_GEN_PYTHON}"
"\n\nNOTE: 'runpath' also replaces $DATA_GEN_PYTHON during the run."
)

def add_arguments(self) -> None:
self.subparser.add_argument(
"benchmark",
type=str,
help=(
"A benchmark name, possibly including wildcards, "
"as supported by the ASV '--bench' argument."
),
)
self.subparser.add_argument(
"runpath",
type=str,
help=(
"A path to an existing python executable, "
"to completely bypass environment building."
),
)

@staticmethod
def func(args: argparse.Namespace) -> None:
if args.runpath:
# Shortcut creation of a data-gen environment
# - which is also the trial-run env.
python_path = Path(args.runpath).resolve()
environ["DATA_GEN_PYTHON"] = str(python_path)
_setup_common()
# get path of data-gen environment, setup by previous call
python_path = environ["DATA_GEN_PYTHON"]
# allow 'on-demand' benchmarks
environ["ON_DEMAND_BENCHMARKS"] = "1"
asv_command = [
"run",
"--bench",
args.benchmark,
# no repeats for timing accuracy
"--quick",
"--show-stderr",
# do not build a unique env : run test in data-gen environment
"--environment",
f"existing:{python_path}",
] + args.asv_args
_subprocess_runner(asv_command, asv=True)


class GhPost(_SubParserGenerator):
name = "_gh_post"
description = (
Expand All @@ -569,11 +626,24 @@ def add_asv_arguments(self) -> None:
def main():
parser = ArgumentParser(
description="Run the Iris performance benchmarks (using Airspeed Velocity).",
epilog="More help is available within each sub-command.",
epilog=(
"More help is available within each sub-command."
"\n\nNOTE(1): a separate python environment is created to "
"construct test files.\n Set $DATA_GEN_PYTHON to avoid the cost "
"of this."
"\nNOTE(2): iris-test-data is downloaded and cached within the "
"data generation environment.\n Set "
"$OVERRIDE_TEST_DATA_REPOSITORY to avoid the cost of this."
"\nNOTE(3): test data is cached within the "
"benchmarks code directory, and uses a lot of disk space "
"of disk space (Gb).\n Set $BENCHMARK_DATA to specify where this "
"space can be safely allocated."
),
formatter_class=argparse.RawTextHelpFormatter,
)
subparsers = parser.add_subparsers(required=True)

for gen in (Overnight, Branch, CPerf, SPerf, Custom, GhPost):
for gen in (Overnight, Branch, CPerf, SPerf, Custom, TrialRun, GhPost):
_ = gen(subparsers).subparser

parsed = parser.parse_args()
Expand Down
4 changes: 2 additions & 2 deletions noxfile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,5 +313,5 @@ def benchmarks(session: nox.sessions.Session):
)
session.error(message)
session.install("asv", "nox")
with session.chdir(Path(__file__).parent / "benchmarks"):
session.run("python", "bm_runner.py", *session.posargs)
bm_runner_path = Path(__file__).parent / "benchmarks" / "bm_runner.py"
session.run("python", bm_runner_path, *session.posargs)