From 282adb588b0e1bc4da6c8b350486d14a9a8155d3 Mon Sep 17 00:00:00 2001 From: Timothy Trippel Date: Fri, 6 May 2022 19:39:17 -0700 Subject: [PATCH] [bazel] use patched python rules and build wheels cache This repository is brought in as a Bazel dependency in the OpenTitan project. In order to support offline (air-gapped) Bazel builds of OpenTitan, all dependencies must also build in an air-gapped environment. Since this repository make use of the Bazel Python rules, Bazel will try to fetch and install Python packages using pip, will not not cache them in Bazel's repository cache (as Bazel has no knowledge of pip). Therefore a similar approach to lowRISC/opentitan:#12091 must be taken: that is, a custom repository rule must be added to cache Python wheels, so that Bazel will not attempt to fetch them in the airgapped environment if they are already present. This partially addresses lowRISC/opentitan:#12538. Signed-off-by: Timothy Trippel --- BUILD | 6 +-- rules/pip.bzl | 122 ++++++++++++++++++++++++++++++++++++++++++------ rules/repos.bzl | 6 +-- 3 files changed, 115 insertions(+), 19 deletions(-) diff --git a/BUILD b/BUILD index bac3b3e..833f2cf 100644 --- a/BUILD +++ b/BUILD @@ -6,12 +6,12 @@ load("//rules:rules.bzl", "licence_check", "yapf_check") licence_check( name = "licence-check", - licence = ''' + exclude_patterns = [".style.yapf"], + licence = """ Copyright lowRISC contributors. Licensed under the Apache License, Version 2.0, see LICENSE for details. SPDX-License-Identifier: Apache-2.0 - ''', - exclude_patterns = [".style.yapf"], + """, ) yapf_check( diff --git a/rules/pip.bzl b/rules/pip.bzl index 6dcb08d..b73ced4 100644 --- a/rules/pip.bzl +++ b/rules/pip.bzl @@ -7,17 +7,113 @@ load("@rules_python//python:pip.bzl", "pip_install") load("@python3//:defs.bzl", "interpreter") +_WHEEL_BUILD_FILE_CONTENTS = """\ +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "all_wheels", + srcs = glob(["**/*.whl"]) +) +""" + +def _pip_wheel_impl(rctx): + # First, check if an existing pre-built Python wheels repo exists, and if + # so, use it instead of building one. + python_wheel_repo_path = rctx.os.environ.get( + "BAZEL_PYTHON_WHEELS_REPO", + None, + ) + if python_wheel_repo_path: + rctx.report_progress("Mounting existing Python wheels repo") + rctx.symlink(python_wheel_repo_path, ".") + return + + # If a pre-built Python wheels repo does not exist, we need to build it. + rctx.report_progress("No Python wheels repo detected, building it instead") + + # First, we install the Python wheel package so we can build other wheels. + args = [ + rctx.path(rctx.attr.python_interpreter), + "-m", + "pip", + "install", + "wheel", + ] + rctx.report_progress("Installing the Python wheel package") + result = rctx.execute( + args, + timeout = rctx.attr.timeout, + quiet = rctx.attr.quiet, + ) + if result.return_code: + fail("pip_wheel failed: {} ({})".format(result.stdout, result.stderr)) + + # Next, we download/build all the Python wheels for each requirement. + args = [ + rctx.path(rctx.attr.python_interpreter), + "-m", + "pip", + "wheel", + "-r", + rctx.path(rctx.attr.requirements), + "-w", + "./", + ] + rctx.report_progress("Pre-building Python wheels") + result = rctx.execute( + args, + timeout = rctx.attr.timeout, + quiet = rctx.attr.quiet, + ) + if result.return_code: + fail("pip_wheel failed: {} ({})".format(result.stdout, result.stderr)) + + # We need a BUILD file to load the downloaded Python packages. + rctx.file( + "BUILD.bazel", + _WHEEL_BUILD_FILE_CONTENTS, + ) + +pip_wheel = repository_rule( + implementation = _pip_wheel_impl, + attrs = { + "python_interpreter": attr.label( + default = interpreter, + allow_single_file = True, + doc = "Python interpreter to use.", + ), + "requirements": attr.label( + default = "//:requirements.txt", + allow_single_file = True, + doc = "Python requirements file describing package dependencies.", + ), + "quiet": attr.bool( + default = True, + doc = "If True, suppress printing stdout/stderr to the terminal.", + ), + "timeout": attr.int( + default = 300, + doc = "Timeout (in seconds) on the rule's execution duration.", + ), + }, + environ = ["BAZEL_PYTHON_WHEELS_REPO"], +) + def lowrisc_misc_linters_pip_dependencies(): - """ - Declares workspaces linting rules depend on. - Make sure to call this in your WORKSPACE file. - - Make sure to call lowrisc_misc_linters_dependencies() from - deps.bzl first. - """ - - pip_install( - name = "lowrisc_misc_linters_pip", - python_interpreter_target = interpreter, - requirements = Label("//:requirements.txt"), - ) + """ + Declares workspaces linting rules depend on. + Make sure to call this in your WORKSPACE file. + + Make sure to call lowrisc_misc_linters_dependencies() from + deps.bzl first. + """ + pip_wheel( + name = "lowrisc_misc_linters_wheels", + ) + pip_install( + name = "lowrisc_misc_linters_pip", + python_interpreter_target = interpreter, + requirements = Label("//:requirements.txt"), + find_links = "@lowrisc_misc_linters_wheels//:all_wheels", + extra_pip_args = ["--no-index"], + ) diff --git a/rules/repos.bzl b/rules/repos.bzl index 3921b09..4de0a99 100644 --- a/rules/repos.bzl +++ b/rules/repos.bzl @@ -12,7 +12,7 @@ def lowrisc_misc_linters_repos(): if not native.existing_rule("rules_python"): http_archive( name = "rules_python", - sha256 = "9fcf91dbcc31fde6d1edb15f117246d912c33c36f44cf681976bd886538deba6", - strip_prefix = "rules_python-0.8.0", - url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.8.0.tar.gz", + sha256 = "9e9a58cff49f80afd1c9fcc7137b719531f7a7427cce4fda1d30ca27b4a46a8a", + strip_prefix = "rules_python-07c3f8547abbd5b97839a48af226a0fbcfaa5e7c", + url = "https://github.com/lowRISC/rules_python/archive/07c3f8547abbd5b97839a48af226a0fbcfaa5e7c.tar.gz", )