From f12f5c32d17753919fd9489ab5251055724da9d7 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 29 Oct 2024 21:04:52 -0700 Subject: [PATCH 1/9] wip: helper to run an arbitrary interpreter or interpreter from a binary Run a specific interpreter: * `bazel run @rules_python//tools/run --@rules_python//python/config_settings:python_version=3.12` Run interpreter from a binary: * `bazel run @rules_python//tools/run --@rules_python//tools/run:bin=//my:binary` --- tools/run/BUILD.bazel | 10 ++++++++++ tools/run/run.bzl | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 tools/run/BUILD.bazel create mode 100644 tools/run/run.bzl diff --git a/tools/run/BUILD.bazel b/tools/run/BUILD.bazel new file mode 100644 index 0000000000..941d06e52d --- /dev/null +++ b/tools/run/BUILD.bazel @@ -0,0 +1,10 @@ +load(":run.bzl", "interpreter") + +interpreter( + name = "run", +) + +label_flag( + name = "bin", + build_setting_default = "//python:none", +) diff --git a/tools/run/run.bzl b/tools/run/run.bzl new file mode 100644 index 0000000000..786d4b5324 --- /dev/null +++ b/tools/run/run.bzl @@ -0,0 +1,40 @@ +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("//python:py_runtime_info.bzl", "PyRuntimeInfo") +load("//python/private:sentinel.bzl", "SentinelInfo") +load("//python/private:toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE") + +def _interpreter_impl(ctx): + if SentinelInfo in ctx.attr.binary: + toolchain = ctx.toolchains[TARGET_TOOLCHAIN_TYPE] + runtime = toolchain.py3_runtime + else: + runtime = ctx.attr.binary[PyRuntimeInfo] + + # NOTE: We name the output filename after the underlying file name + # because of things like pyenv: they use $0 to determine what to + # re-exec. If it's not a recognized name, then they fail. + if runtime.interpreter: + executable = ctx.actions.declare_file(runtime.interpreter.basename) + ctx.actions.symlink(output = executable, target_file = runtime.interpreter, is_executable = True) + else: + executable = ctx.actions.declare_symlink(paths.basename(runtime.interpreter_path)) + ctx.actions.symlink(output = executable, target_path = runtime.interpreter_path) + + return [ + DefaultInfo( + executable = executable, + runfiles = ctx.runfiles([executable], transitive_files = runtime.files), + ), + ] + +interpreter = rule( + implementation = _interpreter_impl, + toolchains = [TARGET_TOOLCHAIN_TYPE], + executable = True, + attrs = { + "binary": attr.label( + default = "//tools/run:bin", + ), + }, +) From 75076262173f614f80a103bd60e8fe5167cea084 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Wed, 27 Nov 2024 12:17:19 -0800 Subject: [PATCH 2/9] Basic REPL support $ bazel run //python/bin:repl $ bazel run //python/bin:repl --//python/bin:repl_dep=//python/runfiles --- python/bin/BUILD.bazel | 14 ++++++++++++++ python/bin/repl.py | 17 +++++++++++++++++ python/private/sentinel.bzl | 4 +++- 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 python/bin/BUILD.bazel create mode 100644 python/bin/repl.py diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel new file mode 100644 index 0000000000..a1ba9e6698 --- /dev/null +++ b/python/bin/BUILD.bazel @@ -0,0 +1,14 @@ +load("//python:defs.bzl", "py_binary") + +py_binary( + name = "repl", + srcs = ["repl.py"], + deps = [ + ":repl_dep", + ], +) + +label_flag( + name = "repl_dep", + build_setting_default = "//python:none", +) diff --git a/python/bin/repl.py b/python/bin/repl.py new file mode 100644 index 0000000000..00c7dec02f --- /dev/null +++ b/python/bin/repl.py @@ -0,0 +1,17 @@ +import code +import os +from pathlib import Path + +# Manually implement PYTHONSTARTUP support. We can't just invoke the python +# binary directly as it would skip the bootstrap scripts. +python_startup = os.getenv("PYTHONSTARTUP") +if python_startup: + try: + source = Path(python_startup).read_text() + except Exception as error: + print(f"{type(error).__name__}: {error}") + else: + compiled_code = compile(source, filename=python_startup, mode="exec") + eval(compiled_code, {}) + +code.InteractiveConsole().interact() diff --git a/python/private/sentinel.bzl b/python/private/sentinel.bzl index 6d753e1983..4e4d8e78a2 100644 --- a/python/private/sentinel.bzl +++ b/python/private/sentinel.bzl @@ -18,6 +18,8 @@ Label attributes with defaults cannot accept None, otherwise they fall back to using the default. A sentinel allows detecting an intended None value. """ +load(":py_info.bzl", "PyInfo") + SentinelInfo = provider( doc = "Indicates this was the sentinel target.", fields = [], @@ -25,6 +27,6 @@ SentinelInfo = provider( def _sentinel_impl(ctx): _ = ctx # @unused - return [SentinelInfo()] + return [SentinelInfo(), PyInfo(transitive_sources=depset())] sentinel = rule(implementation = _sentinel_impl) From 9730855c9c965798ce3a022ad087cef4002a3f8c Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 1 Dec 2024 11:39:26 -0800 Subject: [PATCH 3/9] experiment with site packages --- python/private/site_init_template.py | 1 + python/private/stage2_bootstrap_template.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/python/private/site_init_template.py b/python/private/site_init_template.py index 7a32210bff..e6eadfe8be 100644 --- a/python/private/site_init_template.py +++ b/python/private/site_init_template.py @@ -27,6 +27,7 @@ # Runfiles-relative path to the coverage tool entry point, if any. _COVERAGE_TOOL = "%coverage_tool%" +print("Hi") def _is_verbose(): return bool(os.environ.get("RULES_PYTHON_BOOTSTRAP_VERBOSE")) diff --git a/python/private/stage2_bootstrap_template.py b/python/private/stage2_bootstrap_template.py index d2c7497795..dcbcb253b8 100644 --- a/python/private/stage2_bootstrap_template.py +++ b/python/private/stage2_bootstrap_template.py @@ -24,6 +24,9 @@ # Runfiles-relative path to the main Python source file. MAIN = "%main%" +# Whether this script is used as a sitecustomize script. +USED_AS_SITECUSTOMIZE = "%used_as_sitecustomize%" + # ===== Template substitutions end ===== @@ -375,6 +378,8 @@ def main(): if runfiles_envkey: os.environ[runfiles_envkey] = runfiles_envvalue + sys.path[0:0] = prepend_path_entries + main_filename = os.path.join(module_space, main_rel_path) main_filename = get_windows_path_with_unc_prefix(main_filename) assert os.path.exists(main_filename), ( @@ -386,8 +391,6 @@ def main(): sys.stdout.flush() - sys.path[0:0] = prepend_path_entries - if os.environ.get("COVERAGE_DIR"): import _bazel_site_init coverage_enabled = _bazel_site_init.COVERAGE_SETUP From 286b4f1400cddb3f8cba2b708e4d641988a9655d Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 1 Dec 2024 12:30:57 -0800 Subject: [PATCH 4/9] consolidate files a bit --- python/bin/BUILD.bazel | 10 ++++++++++ tools/run/run.bzl => python/bin/interpreter.bzl | 0 tools/run/BUILD.bazel | 10 ---------- 3 files changed, 10 insertions(+), 10 deletions(-) rename tools/run/run.bzl => python/bin/interpreter.bzl (100%) delete mode 100644 tools/run/BUILD.bazel diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index a1ba9e6698..5745fc8eee 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -1,4 +1,14 @@ load("//python:defs.bzl", "py_binary") +load(":interpreter.bzl", "interpreter") + +interpreter( + name = "interpreter", +) + +label_flag( + name = "bin", + build_setting_default = "//python:none", +) py_binary( name = "repl", diff --git a/tools/run/run.bzl b/python/bin/interpreter.bzl similarity index 100% rename from tools/run/run.bzl rename to python/bin/interpreter.bzl diff --git a/tools/run/BUILD.bazel b/tools/run/BUILD.bazel deleted file mode 100644 index 941d06e52d..0000000000 --- a/tools/run/BUILD.bazel +++ /dev/null @@ -1,10 +0,0 @@ -load(":run.bzl", "interpreter") - -interpreter( - name = "run", -) - -label_flag( - name = "bin", - build_setting_default = "//python:none", -) From 314447d6f634539c38ef213544b36370db383274 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 15 Dec 2024 15:37:57 -0800 Subject: [PATCH 5/9] revert REPL-related things --- python/bin/BUILD.bazel | 13 ------------- python/bin/repl.py | 17 ----------------- python/private/sentinel.bzl | 4 +--- python/private/site_init_template.py | 1 - python/private/stage2_bootstrap_template.py | 7 ++----- 5 files changed, 3 insertions(+), 39 deletions(-) delete mode 100644 python/bin/repl.py diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index 5745fc8eee..4ee55b512a 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -9,16 +9,3 @@ label_flag( name = "bin", build_setting_default = "//python:none", ) - -py_binary( - name = "repl", - srcs = ["repl.py"], - deps = [ - ":repl_dep", - ], -) - -label_flag( - name = "repl_dep", - build_setting_default = "//python:none", -) diff --git a/python/bin/repl.py b/python/bin/repl.py deleted file mode 100644 index 00c7dec02f..0000000000 --- a/python/bin/repl.py +++ /dev/null @@ -1,17 +0,0 @@ -import code -import os -from pathlib import Path - -# Manually implement PYTHONSTARTUP support. We can't just invoke the python -# binary directly as it would skip the bootstrap scripts. -python_startup = os.getenv("PYTHONSTARTUP") -if python_startup: - try: - source = Path(python_startup).read_text() - except Exception as error: - print(f"{type(error).__name__}: {error}") - else: - compiled_code = compile(source, filename=python_startup, mode="exec") - eval(compiled_code, {}) - -code.InteractiveConsole().interact() diff --git a/python/private/sentinel.bzl b/python/private/sentinel.bzl index 4e4d8e78a2..6d753e1983 100644 --- a/python/private/sentinel.bzl +++ b/python/private/sentinel.bzl @@ -18,8 +18,6 @@ Label attributes with defaults cannot accept None, otherwise they fall back to using the default. A sentinel allows detecting an intended None value. """ -load(":py_info.bzl", "PyInfo") - SentinelInfo = provider( doc = "Indicates this was the sentinel target.", fields = [], @@ -27,6 +25,6 @@ SentinelInfo = provider( def _sentinel_impl(ctx): _ = ctx # @unused - return [SentinelInfo(), PyInfo(transitive_sources=depset())] + return [SentinelInfo()] sentinel = rule(implementation = _sentinel_impl) diff --git a/python/private/site_init_template.py b/python/private/site_init_template.py index e6eadfe8be..7a32210bff 100644 --- a/python/private/site_init_template.py +++ b/python/private/site_init_template.py @@ -27,7 +27,6 @@ # Runfiles-relative path to the coverage tool entry point, if any. _COVERAGE_TOOL = "%coverage_tool%" -print("Hi") def _is_verbose(): return bool(os.environ.get("RULES_PYTHON_BOOTSTRAP_VERBOSE")) diff --git a/python/private/stage2_bootstrap_template.py b/python/private/stage2_bootstrap_template.py index dcbcb253b8..d2c7497795 100644 --- a/python/private/stage2_bootstrap_template.py +++ b/python/private/stage2_bootstrap_template.py @@ -24,9 +24,6 @@ # Runfiles-relative path to the main Python source file. MAIN = "%main%" -# Whether this script is used as a sitecustomize script. -USED_AS_SITECUSTOMIZE = "%used_as_sitecustomize%" - # ===== Template substitutions end ===== @@ -378,8 +375,6 @@ def main(): if runfiles_envkey: os.environ[runfiles_envkey] = runfiles_envvalue - sys.path[0:0] = prepend_path_entries - main_filename = os.path.join(module_space, main_rel_path) main_filename = get_windows_path_with_unc_prefix(main_filename) assert os.path.exists(main_filename), ( @@ -391,6 +386,8 @@ def main(): sys.stdout.flush() + sys.path[0:0] = prepend_path_entries + if os.environ.get("COVERAGE_DIR"): import _bazel_site_init coverage_enabled = _bazel_site_init.COVERAGE_SETUP From e07a528ef5e8c36e956b636af0568de2efaf8e16 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 15 Dec 2024 18:15:14 -0800 Subject: [PATCH 6/9] add integration test --- .bazelrc | 4 +- python/BUILD.bazel | 1 + python/bin/BUILD.bazel | 13 +++- python/bin/interpreter.bzl | 2 +- tests/integration/BUILD.bazel | 5 ++ tests/integration/interpreter/BUILD.bazel | 13 ++++ tests/integration/interpreter/MODULE.bazel | 21 +++++++ tests/integration/interpreter/WORKSPACE | 0 tests/integration/interpreter_test.py | 72 ++++++++++++++++++++++ tests/integration/runner.py | 3 +- 10 files changed, 129 insertions(+), 5 deletions(-) create mode 100644 tests/integration/interpreter/BUILD.bazel create mode 100644 tests/integration/interpreter/MODULE.bazel create mode 100644 tests/integration/interpreter/WORKSPACE create mode 100644 tests/integration/interpreter_test.py diff --git a/.bazelrc b/.bazelrc index ada5c5a0a7..bfb07ced41 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,8 +4,8 @@ # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, execute # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/interpreter,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/interpreter,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered test --test_output=errors diff --git a/python/BUILD.bazel b/python/BUILD.bazel index b747e2fbc7..8ffa166728 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -35,6 +35,7 @@ filegroup( name = "distribution", srcs = glob(["**"]) + [ "//python/api:distribution", + "//python/bin:distribution", "//python/cc:distribution", "//python/config_settings:distribution", "//python/constraints:distribution", diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index 4ee55b512a..309e0e11bf 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -1,11 +1,22 @@ load("//python:defs.bzl", "py_binary") load(":interpreter.bzl", "interpreter") +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "distribution", + srcs = glob(["**"]), + visibility = ["//python:__pkg__"], +) + interpreter( name = "interpreter", + binary = ":interpreter_src", ) +# The user can modify this flag to source different interpreters for the +# `interpreter` target above. label_flag( - name = "bin", + name = "interpreter_src", build_setting_default = "//python:none", ) diff --git a/python/bin/interpreter.bzl b/python/bin/interpreter.bzl index 786d4b5324..f73d32362f 100644 --- a/python/bin/interpreter.bzl +++ b/python/bin/interpreter.bzl @@ -34,7 +34,7 @@ interpreter = rule( executable = True, attrs = { "binary": attr.label( - default = "//tools/run:bin", + mandatory = True, ), }, ) diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel index d178e0f01c..637420a634 100644 --- a/tests/integration/BUILD.bazel +++ b/tests/integration/BUILD.bazel @@ -120,6 +120,11 @@ rules_python_integration_test( py_main = "custom_commands_test.py", ) +rules_python_integration_test( + name = "interpreter_test", + py_main = "interpreter_test.py", +) + py_library( name = "runner_lib", srcs = ["runner.py"], diff --git a/tests/integration/interpreter/BUILD.bazel b/tests/integration/interpreter/BUILD.bazel new file mode 100644 index 0000000000..c9b29fc948 --- /dev/null +++ b/tests/integration/interpreter/BUILD.bazel @@ -0,0 +1,13 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/integration/interpreter/MODULE.bazel b/tests/integration/interpreter/MODULE.bazel new file mode 100644 index 0000000000..5bea8126aa --- /dev/null +++ b/tests/integration/interpreter/MODULE.bazel @@ -0,0 +1,21 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module(name = "module_under_test") + +bazel_dep(name = "rules_python", version = "0.0.0") +local_path_override( + module_name = "rules_python", + path = "../../..", +) diff --git a/tests/integration/interpreter/WORKSPACE b/tests/integration/interpreter/WORKSPACE new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/interpreter_test.py b/tests/integration/interpreter_test.py new file mode 100644 index 0000000000..bce33f26f3 --- /dev/null +++ b/tests/integration/interpreter_test.py @@ -0,0 +1,72 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import unittest + +from tests.integration import runner + + +class InterpreterTest(runner.TestCase): + def _run_version_test(self, expected_version): + """Validates that we can successfully execute arbitrary code from the CLI.""" + result = self.run_bazel( + "run", + f"--@rules_python//python/config_settings:python_version={expected_version}", + "@rules_python//python/bin:interpreter", + input = "\r".join([ + "import sys", + "v = sys.version_info", + "print(f'version: {v.major}.{v.minor}')", + ]), + ) + self.assert_result_matches(result, f"version: {expected_version}") + + def test_run_interpreter_3_10(self): + self._run_version_test("3.10") + + def test_run_interpreter_3_11(self): + self._run_version_test("3.11") + + def test_run_interpreter_3_12(self): + self._run_version_test("3.12") + + def _run_module_test(self, version): + """Validates that we can successfully invoke a module from the CLI.""" + result = self.run_bazel( + "run", + f"--@rules_python//python/config_settings:python_version={version}", + "@rules_python//python/bin:interpreter", + "--", + "-m", + "json.tool", + input = '{"json":"obj"}', + ) + self.assert_result_matches(result, r'{\n "json": "obj"\n}') + + def test_run_module_3_10(self): + self._run_module_test("3.10") + + def test_run_module_3_11(self): + self._run_module_test("3.11") + + def test_run_module_3_12(self): + self._run_module_test("3.12") + + + +if __name__ == "__main__": + # Enabling this makes the runner log subprocesses as the test goes along. + # logging.basicConfig(level = "INFO") + unittest.main() diff --git a/tests/integration/runner.py b/tests/integration/runner.py index 9414a865c0..5895828863 100644 --- a/tests/integration/runner.py +++ b/tests/integration/runner.py @@ -86,7 +86,7 @@ def setUp(self): "RUNFILES_DIR": os.environ["TEST_SRCDIR"] } - def run_bazel(self, *args: str, check: bool = True) -> ExecuteResult: + def run_bazel(self, *args: str, input=None, check: bool = True) -> ExecuteResult: """Run a bazel invocation. Args: @@ -104,6 +104,7 @@ def run_bazel(self, *args: str, check: bool = True) -> ExecuteResult: proc_result = subprocess.run( args=args, text=True, + input=input, capture_output=True, cwd=cwd, env=env, From 91d66df12cf33cf35c6296b1be50120238a661e5 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 15 Dec 2024 19:02:37 -0800 Subject: [PATCH 7/9] add some docs --- docs/toolchains.md | 34 ++++++++++++++++++++++++++- tests/integration/interpreter_test.py | 2 ++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/toolchains.md b/docs/toolchains.md index db4c6ba07a..a018e74a19 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -313,7 +313,7 @@ provide `Python.h`. This is typically implemented using {obj}`py_cc_toolchain()`, which provides {obj}`ToolchainInfo` with the field `py_cc_toolchain` set, which is a -{obj}`PyCcToolchainInfo` provider instance. +{obj}`PyCcToolchainInfo` provider instance. This toolchain type is intended to hold only _target configuration_ values relating to the C/C++ information for the Python runtime. As such, when defining @@ -467,3 +467,35 @@ Currently the following flags are used to influence toolchain selection: * {obj}`--@rules_python//python/config_settings:py_linux_libc` for selecting the Linux libc variant. * {obj}`--@rules_python//python/config_settings:py_freethreaded` for selecting the freethreaded experimental Python builds available from `3.13.0` onwards. + +## Accessing the underlying interpreter + +To access the interpreter that bazel manages, you can use the +`@rules_python//python/bin:interpreter` target. This is a binary target with +the executable pointing at the `python3` binary plus the relevent runfiles. + +``` +$ bazel run @rules_python//python/bin:interpreter +Python 3.11.1 (main, Jan 16 2023, 22:41:20) [Clang 15.0.7 ] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> +$ bazel run @rules_python//python/bin:interpreter --@rules_python//python/config_settings:python_version=3.12 +Python 3.12.0 (main, Oct 3 2023, 01:27:23) [Clang 17.0.1 ] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> +``` + +You can also access a specific binary's interpreter this way by using the +`@rules_python//python/bin:interpreter_src` target. + +``` +$ bazel run @rules_python//python/bin:interpreter --@rules_python//python/bin:interpreter_src=//path/to:bin +Python 3.11.1 (main, Jan 16 2023, 22:41:20) [Clang 15.0.7 ] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> +``` + +:::{note} +The interpreter target does not provide access to any modules from `py_*` +targets on its own. Work is ongoing to support that. +::: diff --git a/tests/integration/interpreter_test.py b/tests/integration/interpreter_test.py index bce33f26f3..ab5cd10306 100644 --- a/tests/integration/interpreter_test.py +++ b/tests/integration/interpreter_test.py @@ -44,6 +44,7 @@ def test_run_interpreter_3_12(self): def _run_module_test(self, version): """Validates that we can successfully invoke a module from the CLI.""" + # Pass unformatted JSON to the json.tool module. result = self.run_bazel( "run", f"--@rules_python//python/config_settings:python_version={version}", @@ -53,6 +54,7 @@ def _run_module_test(self, version): "json.tool", input = '{"json":"obj"}', ) + # Validate that we get formatted JSON back. self.assert_result_matches(result, r'{\n "json": "obj"\n}') def test_run_module_3_10(self): From c71629fc342e92d6a4288f9bf58941f7bd6b64cf Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 5 Jan 2025 19:31:31 -0800 Subject: [PATCH 8/9] rename to :python and incorporate readme feedback --- docs/toolchains.md | 16 ++++++++-------- python/bin/BUILD.bazel | 8 ++++---- tests/integration/interpreter_test.py | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/toolchains.md b/docs/toolchains.md index a018e74a19..819b4e9613 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -468,34 +468,34 @@ Currently the following flags are used to influence toolchain selection: * {obj}`--@rules_python//python/config_settings:py_freethreaded` for selecting the freethreaded experimental Python builds available from `3.13.0` onwards. -## Accessing the underlying interpreter +## Running the underlying interpreter -To access the interpreter that bazel manages, you can use the -`@rules_python//python/bin:interpreter` target. This is a binary target with +To run the interpreter that Bazel will use, you can use the +`@rules_python//python/bin:python` target. This is a binary target with the executable pointing at the `python3` binary plus the relevent runfiles. ``` -$ bazel run @rules_python//python/bin:interpreter +$ bazel run @rules_python//python/bin:python Python 3.11.1 (main, Jan 16 2023, 22:41:20) [Clang 15.0.7 ] on linux Type "help", "copyright", "credits" or "license" for more information. >>> -$ bazel run @rules_python//python/bin:interpreter --@rules_python//python/config_settings:python_version=3.12 +$ bazel run @rules_python//python/bin:python --@rules_python//python/config_settings:python_version=3.12 Python 3.12.0 (main, Oct 3 2023, 01:27:23) [Clang 17.0.1 ] on linux Type "help", "copyright", "credits" or "license" for more information. >>> ``` You can also access a specific binary's interpreter this way by using the -`@rules_python//python/bin:interpreter_src` target. +`@rules_python//python/bin:python_src` target. ``` -$ bazel run @rules_python//python/bin:interpreter --@rules_python//python/bin:interpreter_src=//path/to:bin +$ bazel run @rules_python//python/bin:python --@rules_python//python/bin:interpreter_src=//path/to:bin Python 3.11.1 (main, Jan 16 2023, 22:41:20) [Clang 15.0.7 ] on linux Type "help", "copyright", "credits" or "license" for more information. >>> ``` :::{note} -The interpreter target does not provide access to any modules from `py_*` +The `python` target does not provide access to any modules from `py_*` targets on its own. Work is ongoing to support that. ::: diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index 309e0e11bf..a90773b743 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -10,13 +10,13 @@ filegroup( ) interpreter( - name = "interpreter", - binary = ":interpreter_src", + name = "python", + binary = ":python_src", ) # The user can modify this flag to source different interpreters for the -# `interpreter` target above. +# `python` target above. label_flag( - name = "interpreter_src", + name = "python_src", build_setting_default = "//python:none", ) diff --git a/tests/integration/interpreter_test.py b/tests/integration/interpreter_test.py index ab5cd10306..a7f015305d 100644 --- a/tests/integration/interpreter_test.py +++ b/tests/integration/interpreter_test.py @@ -24,7 +24,7 @@ def _run_version_test(self, expected_version): result = self.run_bazel( "run", f"--@rules_python//python/config_settings:python_version={expected_version}", - "@rules_python//python/bin:interpreter", + "@rules_python//python/bin:python", input = "\r".join([ "import sys", "v = sys.version_info", @@ -48,7 +48,7 @@ def _run_module_test(self, version): result = self.run_bazel( "run", f"--@rules_python//python/config_settings:python_version={version}", - "@rules_python//python/bin:interpreter", + "@rules_python//python/bin:python", "--", "-m", "json.tool", From 2ee2311120a3591c1b4114c0c4b1e989b7bf24d7 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 5 Jan 2025 19:42:37 -0800 Subject: [PATCH 9/9] move implementation to //python/private --- python/bin/interpreter.bzl | 41 ++-------------------------------- python/private/interpreter.bzl | 40 +++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 39 deletions(-) create mode 100644 python/private/interpreter.bzl diff --git a/python/bin/interpreter.bzl b/python/bin/interpreter.bzl index f73d32362f..40282e0790 100644 --- a/python/bin/interpreter.bzl +++ b/python/bin/interpreter.bzl @@ -1,40 +1,3 @@ -load("@bazel_skylib//lib:paths.bzl", "paths") -load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") -load("//python:py_runtime_info.bzl", "PyRuntimeInfo") -load("//python/private:sentinel.bzl", "SentinelInfo") -load("//python/private:toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE") +load("//python/private:interpreter.bzl", _interpeter="interpreter") -def _interpreter_impl(ctx): - if SentinelInfo in ctx.attr.binary: - toolchain = ctx.toolchains[TARGET_TOOLCHAIN_TYPE] - runtime = toolchain.py3_runtime - else: - runtime = ctx.attr.binary[PyRuntimeInfo] - - # NOTE: We name the output filename after the underlying file name - # because of things like pyenv: they use $0 to determine what to - # re-exec. If it's not a recognized name, then they fail. - if runtime.interpreter: - executable = ctx.actions.declare_file(runtime.interpreter.basename) - ctx.actions.symlink(output = executable, target_file = runtime.interpreter, is_executable = True) - else: - executable = ctx.actions.declare_symlink(paths.basename(runtime.interpreter_path)) - ctx.actions.symlink(output = executable, target_path = runtime.interpreter_path) - - return [ - DefaultInfo( - executable = executable, - runfiles = ctx.runfiles([executable], transitive_files = runtime.files), - ), - ] - -interpreter = rule( - implementation = _interpreter_impl, - toolchains = [TARGET_TOOLCHAIN_TYPE], - executable = True, - attrs = { - "binary": attr.label( - mandatory = True, - ), - }, -) +interpreter = _interpeter diff --git a/python/private/interpreter.bzl b/python/private/interpreter.bzl new file mode 100644 index 0000000000..8754b47cb2 --- /dev/null +++ b/python/private/interpreter.bzl @@ -0,0 +1,40 @@ +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("//python:py_runtime_info.bzl", "PyRuntimeInfo") +load(":sentinel.bzl", "SentinelInfo") +load(":toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE") + +def _interpreter_impl(ctx): + if SentinelInfo in ctx.attr.binary: + toolchain = ctx.toolchains[TARGET_TOOLCHAIN_TYPE] + runtime = toolchain.py3_runtime + else: + runtime = ctx.attr.binary[PyRuntimeInfo] + + # NOTE: We name the output filename after the underlying file name + # because of things like pyenv: they use $0 to determine what to + # re-exec. If it's not a recognized name, then they fail. + if runtime.interpreter: + executable = ctx.actions.declare_file(runtime.interpreter.basename) + ctx.actions.symlink(output = executable, target_file = runtime.interpreter, is_executable = True) + else: + executable = ctx.actions.declare_symlink(paths.basename(runtime.interpreter_path)) + ctx.actions.symlink(output = executable, target_path = runtime.interpreter_path) + + return [ + DefaultInfo( + executable = executable, + runfiles = ctx.runfiles([executable], transitive_files = runtime.files), + ), + ] + +interpreter = rule( + implementation = _interpreter_impl, + toolchains = [TARGET_TOOLCHAIN_TYPE], + executable = True, + attrs = { + "binary": attr.label( + mandatory = True, + ), + }, +)