feat: compile source files at build time (#1902)

This implements precompiling: performing Python source to byte code
compilation at build time. This allows improved program startup time by
allowing the byte code compilation step to be skipped at runtime.

Precompiling is disabled by default, for now. A subsequent release will
enable it by default. This allows the necessary flags and attributes to
become available so users can opt-out prior to it being enabled by
default. Similarly, `//python:features.bzl` is introduced to allow
feature detection.

This implementation is made to serve a variety of use cases, so there
are several attributes and flags to control behavior. The main use cases
being served are:
* Large mono-repos that need to incrementally enable/disable
precompiling.
* Remote execution builds, where persistent workers aren't easily
available.
* Environments where toolchains are custom defined instead of using the
ones created by rules_python.

To that end, there are several attributes and flags to control behavior,
and the toolchains allow customizing the tools used.

Fixes #1761

.bazelignore

@@ -25,4 +25,5 @@ examples/pip_parse_vendored/bazel-pip_parse_vendored
 examples/py_proto_library/bazel-py_proto_library
 tests/integration/compile_pip_requirements/bazel-compile_pip_requirements
 tests/integration/ignore_root_user_error/bazel-ignore_root_user_error
+tests/integration/local_toolchains/bazel-local_toolchains
 tests/integration/pip_repository_entry_points/bazel-pip_repository_entry_points

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
 
 test --test_output=errors
 

CHANGELOG.md

@@ -22,6 +22,9 @@ A brief description of the categories of changes:
 [x.x.x]: https://github.com/bazelbuild/rules_python/releases/tag/x.x.x
 
 ### Changed
+* (toolchains) Optional toolchain dependency: `py_binary`, `py_test`, and
+  `py_library` now depend on the `//python:exec_tools_toolchain_type` for build
+  tools.
 
 * (deps): Bumped `bazel_skylib` to 1.6.1.
 * (bzlmod): The `python` and internal `rules_python` extensions have been
@@ -37,6 +40,28 @@ A brief description of the categories of changes:
   for this package before will be deleted automatically.
 
 ### Added
+* (rules) Precompiling Python source at build time is available. but is
+  disabled by default, for now. Set
+  `@rules_python//python/config_settings:precompile=enabled` to enable it
+  by default. A subsequent release will enable it by default. See the
+  [Precompiling docs][precompile-docs] and API reference docs for more
+  information on precompiling. Note this requires Bazel 7+ and the Pystar rule
+  implementation enabled.
+  ([#1761](https://github.com/bazelbuild/rules_python/issues/1761))
+* (rules) Attributes and flags to control precompile behavior: `precompile`,
+  `precompile_optimize_level`, `precompile_source_retention`,
+  `precompile_invalidation_mode`, and `pyc_collection`
+* (toolchains) The target runtime toolchain (`//python:toolchain_type`) has
+  two new optional attributes: `pyc_tag` (tells the pyc filename infix to use) and
+  `implementation_name` (tells the Python implementation name).
+* (toolchains) A toolchain type for build tools has been added:
+  `//python:exec_tools_toolchain_type`.
+* (providers) `PyInfo` has two new attributes: `direct_pyc_files` and
+  `transitive_pyc_files`, which tell the pyc files a target makes available
+  directly and transitively, respectively.
+* `//python:features.bzl` added to allow easy feature-detection in the future.
+
+[precompile-docs]: /precompiling
 
 ## [0.32.2] - 2024-05-14
 

docs/sphinx/index.md

@@ -60,6 +60,7 @@ pypi-dependencies
 toolchains
 pip
 coverage
+precompiling
 gazelle
 Contributing <contributing>
 support

docs/sphinx/precompiling.md

@@ -0,0 +1,95 @@
+# Precompiling
+
+Precompiling is compiling Python source files (`.py` files) into byte code (`.pyc`
+files) at build
+time instead of runtime. Doing it at build time can improve performance by
+skipping that work at runtime.
+
+Precompiling is enabled by default, so there typically isn't anything special
+you must do to use it.
+
+
+## Overhead of precompiling
+
+While precompiling helps runtime performance, it has two main costs:
+1. Increasing the size (count and disk usage) of runfiles. It approximately
+   double the count of the runfiles because for every `.py` file, there is also
+   a `.pyc` file. Compiled files are generally around the same size as the
+   source files, so it approximately doubles the disk usage.
+2. Precompiling requires running an extra action at build time. While
+   compiling itself isn't that expensive, the overhead can become noticable
+   as more files need to be compiled.
+
+## Binary-level opt-in
+
+Because of the costs of precompiling, it may not be feasible to globally enable it
+for your repo for everything. For example, some binaries may be
+particularly large, and doubling the number of runfiles isn't doable.
+
+If this is the case, there's an alternative way to more selectively and
+incrementally control precompiling on a per-binry basis.
+
+To use this approach, the two basic steps are:
+1. Disable pyc files from being automatically added to runfiles:
+   `--@rules_python//python/config_settings:precompile_add_to_runfiles=decided_elsewhere`,
+2. Set the `pyc_collection` attribute on the binaries/tests that should or should
+   not use precompiling.
+
+The default for the `pyc_collection` attribute is controlled by a flag, so you
+can use an opt-in or opt-out approach by setting the flag:
+* targets must opt-out: `--@rules_python//python/config_settings:pyc_collection=include_pyc`,
+* targets must opt-in: `--@rules_python//python/config_settings:pyc_collection=disabled`,
+
+## Advanced precompiler customization
+
+The default implementation of the precompiler is a persistent, multiplexed,
+sandbox-aware, cancellation-enabled, json-protocol worker that uses the same
+interpreter as the target toolchain. This works well for local builds, but may
+not work as well for remote execution builds. To customize the precompiler, two
+mechanisms are available:
+
+* The exec tools toolchain allows customizing the precompiler binary used with
+  the `precompiler` attribute. Arbitrary binaries are supported.
+* The execution requirements can be customized using
+  `--@rules_python//tools/precompiler:execution_requirements`. This is a list
+  flag that can be repeated. Each entry is a key=value that is added to the
+  execution requirements of the `PyPrecompile` action. Note that this flag
+  is specific to the rules_python precompiler. If a custom binary is used,
+  this flag will have to be propagated from the custom binary using the
+  `testing.ExecutionInfo` provider; refer to the `py_interpreter_program` an
+
+The default precompiler implementation is an asynchronous/concurrent
+implementation. If you find it has bugs or hangs, please report them. In the
+meantime, the flag `--worker_extra_flag=PyPrecompile=--worker_impl=serial` can
+be used to switch to a synchronous/serial implementation that may not perform
+as well, but is less likely to have issues.
+
+The `execution_requirements` keys of most relevance are:
+* `supports-workers`: 1 or 0, to indicate if a regular persistent worker is
+  desired.
+* `supports-multiplex-workers`: 1 o 0, to indicate if a multiplexed persistent
+  worker is desired.
+* `requires-worker-protocol`: json or proto; the rules_python precompiler
+  currently only supports json.
+* `supports-multiplex-sandboxing`: 1 or 0, to indicate if sanboxing is of the
+  worker is supported.
+* `supports-worker-cancellation`: 1 or 1, to indicate if requests to the worker
+  can be cancelled.
+
+Note that any execution requirements values can be specified in the flag.
+
+## Known issues, caveats, and idiosyncracies
+
+* Precompiling requires Bazel 7+ with the Pystar rule implementation enabled.
+* Mixing rules_python PyInfo with Bazel builtin PyInfo will result in pyc files
+  being dropped.
+* Precompiled files may not be used in certain cases prior to Python 3.11. This
+  occurs due Python adding the directory of the binary's main `.py` file, which
+  causes the module to be found in the workspace source directory instead of
+  within the binary's runfiles directory (where the pyc files are). This can
+  usually be worked around by removing `sys.path[0]` (or otherwise ensuring the
+  runfiles directory comes before the repos source directory in `sys.path`).
+* The pyc filename does not include the optimization level (e.g.
+  `foo.cpython-39.opt-2.pyc`). This works fine (it's all byte code), but also
+  means the interpreter `-O` argument can't be used -- doing so will cause the
+  interpreter to look for the non-existent `opt-N` named files.

python/BUILD.bazel

@@ -71,6 +71,11 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "features_bzl",
+    srcs = ["features.bzl"],
+)
+
 bzl_library(
     name = "packaging_bzl",
     srcs = ["packaging.bzl"],
@@ -292,6 +297,11 @@ alias(
     actual = "@bazel_tools//tools/python:toolchain_type",
 )
 
+toolchain_type(
+    name = "exec_tools_toolchain_type",
+    visibility = ["//visibility:public"],
+)
+
 # Definitions for a Python toolchain that, at execution time, attempts to detect
 # a platform runtime having the appropriate major Python version. Consider this
 # a toolchain of last resort.

python/config_settings/BUILD.bazel

@@ -1,3 +1,11 @@
+load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
+load(
+    "//python/private:flags.bzl",
+    "PrecompileAddToRunfilesFlag",
+    "PrecompileFlag",
+    "PrecompileSourceRetentionFlag",
+    "PycCollectionFlag",
+)
 load(":config_settings.bzl", "construct_config_settings")
 
 filegroup(
@@ -12,3 +20,35 @@ filegroup(
 construct_config_settings(
     name = "construct_config_settings",
 )
+
+string_flag(
+    name = "precompile",
+    build_setting_default = PrecompileFlag.AUTO,
+    values = sorted(PrecompileFlag.__members__.values()),
+    # NOTE: Only public because its an implicit dependency
+    visibility = ["//visibility:public"],
+)
+
+string_flag(
+    name = "precompile_source_retention",
+    build_setting_default = PrecompileSourceRetentionFlag.KEEP_SOURCE,
+    values = sorted(PrecompileSourceRetentionFlag.__members__.values()),
+    # NOTE: Only public because its an implicit dependency
+    visibility = ["//visibility:public"],
+)
+
+string_flag(
+    name = "precompile_add_to_runfiles",
+    build_setting_default = PrecompileAddToRunfilesFlag.ALWAYS,
+    values = sorted(PrecompileAddToRunfilesFlag.__members__.values()),
+    # NOTE: Only public because its an implicit dependency
+    visibility = ["//visibility:public"],
+)
+
+string_flag(
+    name = "pyc_collection",
+    build_setting_default = PycCollectionFlag.DISABLED,
+    values = sorted(PycCollectionFlag.__members__.values()),
+    # NOTE: Only public because its an implicit dependency
+    visibility = ["//visibility:public"],
+)

tests/support/test_platforms.bzl → python/features.bzl

@@ -1,4 +1,4 @@
-# Copyright 2023 The Bazel Authors. All rights reserved.
+# Copyright 2024 The Bazel Authors. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,10 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Constants for referring to platforms."""
+"""Allows detecting of rules_python features that aren't easily detected."""
 
-# Explicit Label() calls are required so that it resolves in @rules_python
-# context instead of e.g. the @rules_testing context.
-MAC = Label("//tests/support:mac")
-LINUX = Label("//tests/support:linux")
-WINDOWS = Label("//tests/support:windows")
+features = struct(
+    precompile = True,
+)

python/private/BUILD.bazel

@@ -86,11 +86,25 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "enum_bzl",
+    srcs = ["enum.bzl"],
+)
+
 bzl_library(
     name = "envsubst_bzl",
     srcs = ["envsubst.bzl"],
 )
 
+bzl_library(
+    name = "flags_bzl",
+    srcs = ["flags.bzl"],
+    deps = [
+        ":enum_bzl",
+        "@bazel_skylib//rules:common_settings",
+    ],
+)
+
 bzl_library(
     name = "full_version_bzl",
     srcs = ["full_version.bzl"],
@@ -166,6 +180,18 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "py_exec_tools_toolchain_bzl",
+    srcs = ["py_exec_tools_toolchain.bzl"],
+    deps = ["//python/private/common:providers_bzl"],
+)
+
+bzl_library(
+    name = "py_interpreter_program_bzl",
+    srcs = ["py_interpreter_program.bzl"],
+    deps = ["@bazel_skylib//rules:common_settings"],
+)
+
 bzl_library(
     name = "py_package_bzl",
     srcs = ["py_package.bzl"],
@@ -192,6 +218,7 @@ bzl_library(
     name = "py_toolchain_suite_bzl",
     srcs = ["py_toolchain_suite.bzl"],
     deps = [
+        ":toolchain_types_bzl",
         "@bazel_skylib//lib:selects",
     ],
 )
@@ -256,6 +283,11 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "toolchain_types_bzl",
+    srcs = ["toolchain_types.bzl"],
+)
+
 bzl_library(
     name = "util_bzl",
     srcs = ["util.bzl"],

python/private/common/BUILD.bazel

@@ -32,7 +32,10 @@ bzl_library(
         ":providers_bzl",
         ":py_internal_bzl",
         ":semantics_bzl",
+        "//python/private:enum_bzl",
+        "//python/private:flags_bzl",
         "//python/private:reexports_bzl",
+        "@bazel_skylib//rules:common_settings",
     ],
 )
 
@@ -46,9 +49,12 @@ bzl_library(
     name = "common_bazel_bzl",
     srcs = ["common_bazel.bzl"],
     deps = [
+        ":attributes_bzl",
         ":common_bzl",
         ":providers_bzl",
         ":py_internal_bzl",
+        "//python/private:py_interpreter_program_bzl",
+        "//python/private:toolchain_types_bzl",
         "@bazel_skylib//lib:paths",
     ],
 )
@@ -124,8 +130,11 @@ bzl_library(
         ":common_bzl",
         ":providers_bzl",
         ":py_internal_bzl",
+        "//python/private:flags_bzl",
         "//python/private:rules_cc_srcs_bzl",
+        "//python/private:toolchain_types_bzl",
         "@bazel_skylib//lib:dicts",
+        "@bazel_skylib//rules:common_settings",
     ],
 )
 
@@ -143,7 +152,10 @@ bzl_library(
         ":common_bzl",
         ":providers_bzl",
         ":py_internal_bzl",
+        "//python/private:flags_bzl",
+        "//python/private:toolchain_types_bzl",
         "@bazel_skylib//lib:dicts",
+        "@bazel_skylib//rules:common_settings",
     ],
 )