docs(examples): Add example of using a src dir and separate tests dir with gazelle
#1842
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| # Load all of the various rules and functions that we end up using. | ||
| load("@bazel_gazelle//:def.bzl", "gazelle") | ||
| load("@pypi//:requirements.bzl", "all_whl_requirements") | ||
| load("@rules_python//python:pip.bzl", "compile_pip_requirements") | ||
| load("@rules_python_gazelle_plugin//manifest:defs.bzl", "gazelle_python_manifest") | ||
| load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping") | ||
|
|
||
| # This filegroup is only needed if | ||
| # (a) you're using pyproject.toml to store your dependencies and | ||
| # (b) you have [tool.setuptools.package-dir] and [tool.setuptools.packages.find] | ||
| # configured for `src` dirs per https://setuptools.pypa.io/en/stable/userguide/package_discovery.html | ||
| # (which you probably do if you're looking at this example). | ||
| # This target gets used later by compile_pip_requirements. | ||
| filegroup( | ||
| name = "src_dir", | ||
| srcs = ["src"], | ||
| ) | ||
|
|
||
| ###### | ||
| # Gazelle | ||
| # | ||
| # Gazelle is an automatic BUILD(.bazel) file generator. Run via: | ||
| # bazel run //:gazelle | ||
| ###### | ||
|
|
||
| # Comments that start with "# gazelle:XYZ" are called *directives*. Some directives | ||
| # can and should be set here, in the same bazel package (BUILD file) that defines | ||
| # gazelle, while other directives (such as "# gazelle:python_root") should be | ||
| # defined in a BUILD file specific to that part of the folder tree. See | ||
| # src/BUILD for such an example - it's how we define that the "src" dir should | ||
| # be the root of python files and thus get added to sys.path. | ||
|
|
||
| # This directive tells gazelle that our tests are named "test_foo.py" instead | ||
| # of "foo_test.py". | ||
| # gazelle:python_test_naming_convention test_$package_name$ | ||
|
|
||
| # This directive tells gazelle to make a single bazel target per python file. | ||
| # The default is to make a single bazel target per python _package_). | ||
| # gazelle:python_generation_mode file | ||
|
|
||
| ###### End Gazelle Directives ###### | ||
|
|
||
| # This rule will compile the project requirements into a lock file that | ||
| # contains versions and hashes. The lock file ends up getting used when | ||
| # installing dependencies via pip. | ||
| # bazel run //:requirements.update | ||
| compile_pip_requirements( | ||
| # Name this target. This will be how you run with `bazel run //:<name>.update` | ||
| name = "requirements", | ||
| # See comment about filegroup above. If both (a) and (b) are true, you need this otherwise | ||
| # the compiling will fail with "error in 'egg_base' option: 'src' does not exist or | ||
| # is not a directory". | ||
| data = [":src_dir"], | ||
| # Optional. Tell pip_tools to be more verbose. | ||
| # extra_args = ["-v"], | ||
| # If you store requirements in a separate file, name that file in `src`. | ||
| # Otherwise, gazelle will pull from pyproject.toml's [project.dependencies] section. | ||
| # src = "requirements.in", | ||
| requirements_txt = "requirements.lock", | ||
| ) | ||
|
|
||
| # This rule fetches the metadata for python packages we depend on. That data is | ||
| # required for the gazelle_python_manifest rule to update our manifest file. | ||
| modules_mapping( | ||
| # Name this target. This name is used in `gazelle_python_manifest.modules_mapping` below. | ||
| name = "modules_map", | ||
| wheels = all_whl_requirements, | ||
| ) | ||
|
|
||
| # Gazelle python extension needs a manifest file mapping from | ||
| # an import to the installed package that provides it. This target updates the | ||
| # "gazelle_python.yaml" file when run. The file must already exist. | ||
| # This target produces two targets: | ||
| # bazel run //:gazelle_python_manifest.update | ||
| # bazel run //:gazelle_python_manifest.test | ||
| gazelle_python_manifest( | ||
| # Name this target. This will be how you run with `bazel run //:<name>.update` | ||
| name = "gazelle_python_manifest", | ||
| # Same as `modules_mapping.name` (with ":"), above. | ||
| modules_mapping = ":modules_map", | ||
| # This is what we called our `pip_parse` rule, where third-party | ||
| # python libraries are loaded in BUILD files. | ||
| pip_repository_name = "pypi", | ||
| # This should point to wherever we declare our python dependencies. | ||
| # It's the same as what we passed to the pip.parse rule in MODULE.bazel and | ||
| # is the same filename that we used in the `requirements_txt` attribute of | ||
| # `compile_pip_requirements`, above.) | ||
| # This argument is optional. If provided, the `.test` target is very | ||
| # fast because it just has to check an integrity field. If not provided, | ||
| # the integrity field is not added to the manifest which can help avoid | ||
| # merge conflicts in large repos. | ||
| requirements = "//:requirements.lock", | ||
| ) | ||
|
|
||
| # Make a target for running gazelle. | ||
| # bazel run //:gazelle | ||
| # or: | ||
| # bazel run //:gazelle update # Note: "update" is the arg, not part of the target | ||
| gazelle( | ||
| name = "gazelle", | ||
| gazelle = "@rules_python_gazelle_plugin//python:gazelle_binary", | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,69 @@ | ||||||
| # Define metadata about this repository/project. | ||||||
| module( | ||||||
| name = "example_bzlmod_python_src_dir_with_separate_tests_dir", | ||||||
| version = "0.0.0", | ||||||
| compatibility_level = 1, | ||||||
| ) | ||||||
|
|
||||||
| # Install rules_python, which allows us to define how bazel should work with python files. | ||||||
| # See https://github.com/bazelbuild/rules_python/blob/main/examples/bzlmod/MODULE.bazel | ||||||
|
There was a problem hiding this comment. Why not use relative file references? At least in
Suggested change
There was a problem hiding this comment. +1, updated here and for gazelle/README.md below. |
||||||
| # In the old WORKSPACE file, this would be 4 items: | ||||||
| # 1. `load` the http_archive rule | ||||||
| # 2. run the http_archive rule, grapping rules_python from github | ||||||
| # 3. load the py_repositories target from rules_python | ||||||
| # 4. execute py_respositories() | ||||||
|
There was a problem hiding this comment. We could just link to the relevant example (e.g. There was a problem hiding this comment. Relative file references added, but I kept the original list because I find it easier to follow if I don't have to jump around to various other examples. LMK if you feel strongly about removing the list. |
||||||
| bazel_dep(name = "rules_python", version = "0.31.0") | ||||||
|
|
||||||
| ### THIS BLOCK IS ONLY REQUIRED FOR EXAMPLES ### | ||||||
| # This `local_path_override` allows us to use this repo's version of rules_python. | ||||||
| # For usual setups you should remove this local_path_override block. | ||||||
| local_path_override( | ||||||
| module_name = "rules_python", | ||||||
| path = "../..", | ||||||
| ) | ||||||
| ### END EXAMPLE-ONLY BLOCK ### | ||||||
|
|
||||||
| # Gazelle for auto BUILD generation. See | ||||||
| # https://github.com/bazelbuild/rules_python/blob/main/gazelle/README.md | ||||||
| # First install the gazelle config that's specific to python. | ||||||
| bazel_dep(name = "rules_python_gazelle_plugin", version = "0.31.0") # same version as rules_python | ||||||
|
|
||||||
| ### THIS BLOCK IS ONLY REQUIRED FOR EXAMPLES ### | ||||||
| # This `local_path_override` allows us to use this repo's version of rules_python. | ||||||
| # For usual setups you should remove this local_path_override block. | ||||||
| local_path_override( | ||||||
| module_name = "rules_python_gazelle_plugin", | ||||||
| path = "../../gazelle", | ||||||
| ) | ||||||
| ### END EXAMPLE-ONLY BLOCK ### | ||||||
|
|
||||||
| # Then install gazelle itself. | ||||||
| bazel_dep(name = "gazelle", version = "0.35.0", repo_name = "bazel_gazelle") | ||||||
|
|
||||||
| # Initialize the python toolchain using the rules_python extension. | ||||||
| # This is similar to the "python_register_toolchains" function in WORKSPACE. | ||||||
| # It creates a hermetic python rather than relying on a system-installed interpreter. | ||||||
| python = use_extension("@rules_python//python/extensions:python.bzl", "python") | ||||||
| python.toolchain( | ||||||
| configure_coverage_tool = True, | ||||||
| python_version = "3.9", | ||||||
| ) | ||||||
|
|
||||||
| # Enable pip | ||||||
| pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") | ||||||
|
|
||||||
| # Configure how we fetch python dependencies via pip | ||||||
| pip.parse( | ||||||
| # Use the bazel downloader for pulling pypi packages. | ||||||
| experimental_index_url = "https://pypi.org/simple", | ||||||
|
|
||||||
| # This hub name is what gets used in other BUILD files with `load()`. | ||||||
| hub_name = "pypi", | ||||||
| python_version = "3.9", | ||||||
| # The file that contains the python dependencies, versions, and hashes. | ||||||
| # This target needs to be the same as what's passed to `gazelle_python_manifest.requirements` | ||||||
| # in ./BUILD.bazel. | ||||||
| requirements_lock = "//:requirements.lock", | ||||||
| ) | ||||||
|
|
||||||
| # Same as WORKSPACE install_deps() - actually install the python deps. | ||||||
|
There was a problem hiding this comment. This is not technically correct. Here we just expose the repo to be used by the module and the install happens lazily by bazel. I do think the intention is good - to draw parallels between There was a problem hiding this comment. Ah, good to know! Updated, PTAL and check for correctness. |
||||||
| use_repo(pip, "pypi") | ||||||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| # Using a `src` dir and separate `tests` dir, with bzlmod | ||
|
|
||
| This example highlights how to set up `MODULE.bazel`, `BUILD.bazel`, and `gazelle` to work with | ||
| a python `src` directory and a separate `tests` directory[^1]. | ||
|
|
||
|
|
||
| Run tests by first `cd`ing into this directory and then running `bazel test`: | ||
|
|
||
| ```shell | ||
| $ cd examples/bzlmod_python_src_dir_with_separate_tests_dir | ||
| $ bazel test --test_output=errors //... | ||
| ``` | ||
|
|
||
| Everything should pass. | ||
|
|
||
| Try changing `tests/test_my_python_module.py`'s assert to a different value and run | ||
| `bazel test` again. You'll see a test failure, yay! | ||
|
|
||
|
|
||
| [^1]: This is how the [Python Packaging User Guide][pypa-tutorial] recommends new python libraries | ||
| be set up. | ||
|
|
||
| [pypa-tutorial]: https://github.com/pypa/packaging.python.org/blob/091e45c8f78614307ccfdc061a6e562d669b178b/source/tutorials/packaging-projects.rst | ||
|
|
||
|
|
||
| ## Details | ||
|
|
||
| The folder structure, prior to adding Bazel, is: | ||
|
|
||
| ``` | ||
| ./ | ||
| ├── pyproject.toml | ||
| ├── README.md | ||
| ├── src/ | ||
| │ └── my_package/ | ||
| │ ├── __init__.py | ||
| │ └── my_python_module.py | ||
| └── tests/ | ||
| ├── __init__.py | ||
| └── test_my_python_module.py | ||
| ``` | ||
|
|
||
| After adding files and configuration for Bazel and gazelle: | ||
|
|
||
| ``` | ||
| packaging_tutorial/ | ||
| ├── BUILD.bazel # New | ||
| ├── gazelle_python.yaml # New, empty | ||
| ├── MODULE.bazel # New | ||
| ├── pyproject.toml | ||
| ├── README.md | ||
| ├── requirements.lock # New, empty | ||
| ├── src/ | ||
| │ ├── BUILD.bazel # New | ||
| │ └── mypackage/ | ||
| │ ├── __init__.py | ||
| │ └── my_python_module.py | ||
| └── tests/ | ||
| ├── __init__.py | ||
| └── test_my_python_module.py | ||
| ``` | ||
|
|
||
| After running Gazelle: | ||
|
|
||
| ```shell | ||
| $ bazel run //:requirements.update | ||
| $ bazel run //:gazelle_python_manifest.update | ||
| $ bazel run //:gazelle | ||
| ``` | ||
|
|
||
| ``` | ||
| packaging_tutorial/ | ||
| ├── BUILD.bazel | ||
| ├── gazelle_python.yaml # Updated by 'bazel run //:gazelle_python_manifest.update' | ||
| ├── MODULE.bazel | ||
| ├── MODULE.bazel.lock # New, not included in git repo | ||
| ├── pyproject.toml | ||
| ├── README.md | ||
| ├── requirements.lock # Updated by 'bazel run //:requirements.update' | ||
| ├── src/ | ||
| │ ├── BUILD.bazel | ||
| │ └── mypackage/ | ||
| │ ├── __init__.py | ||
| │ ├── BUILD.bazel # New, added by 'bazel run //:gazelle' | ||
| │ └── my_python_module.py | ||
| └── tests/ | ||
| ├── __init__.py | ||
| ├── BUILD.bazel # New, added by 'bazel run //:gazelle' | ||
| └── test_my_python_module.py | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| # GENERATED FILE - DO NOT EDIT! | ||
| # | ||
| # To update this file, run: | ||
| # bazel run //:gazelle_python_manifest.update | ||
|
|
||
| manifest: | ||
| modules_mapping: | ||
| pathspec: pathspec | ||
| pathspec.gitignore: pathspec | ||
| pathspec.pathspec: pathspec | ||
| pathspec.pattern: pathspec | ||
| pathspec.patterns: pathspec | ||
| pathspec.patterns.gitwildmatch: pathspec | ||
| pathspec.util: pathspec | ||
| pip_repository: | ||
| name: pypi | ||
| integrity: 05245d78ed551ea7a050bc567024326e6d9256b8b8d356f855c3f29af654685a |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| [build-system] | ||
| requires = ["setuptools>=61.0"] | ||
| build-backend = "setuptools.build_meta" | ||
|
|
||
| [project] | ||
| name = "my_package" | ||
| version = "0.0.1" | ||
| description = "Example of using Bazel with python `src` and `tests` dir" | ||
| dependencies = [ | ||
| "pathspec==0.12.1", | ||
| ] | ||
|
|
||
| [tool.setuptools] | ||
| package-dir = {"" = "src"} | ||
|
|
||
| [tool.setuptools.packages.find] | ||
| where = ["src"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| # | ||
| # This file is autogenerated by pip-compile with Python 3.9 | ||
| # by the following command: | ||
| # | ||
| # bazel run //:requirements.update | ||
| # | ||
| pathspec==0.12.1 \ | ||
| --hash=sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08 \ | ||
| --hash=sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712 | ||
| # via my-package (pyproject.toml) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| # This directive tells Gazelle to use this dir (`src`) as the root python path. | ||
| # gazelle:python_root | ||
|
|
||
| # This directive tells Gazelle to append "//tests:__subpackages__" to the | ||
| # visibility of all python targets. | ||
| # gazelle:python_visibility //tests:__subpackages__ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| load("@rules_python//python:defs.bzl", "py_library") | ||
|
|
||
| py_library( | ||
| name = "my_python_module", | ||
| srcs = ["my_python_module.py"], | ||
| imports = [".."], | ||
| visibility = [ | ||
| "//src:__subpackages__", | ||
| "//tests:__subpackages__", | ||
| ], | ||
| deps = ["@pypi//pathspec"], | ||
| ) |
Empty file.
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Having an empty
WORKSPACEfile may fix some of the issues you saw with the pre-commit hook, I think.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Hmm... still no dice. Same issue.
Given that CI is passing, I'm not too concerned about the pre-commit check. My assumption is that it's something specific to my computer/setup rather than specific to this branch, as
pre-commit run --all-filesalso fails on the latestmaincommit 3730803.