From 952880642e157cc1dadba82579e7b18dde9b9160 Mon Sep 17 00:00:00 2001 From: Rasrack Date: Tue, 25 Apr 2023 19:20:00 +0200 Subject: [PATCH] fix: compile_pip_requirements test from external repositories (#1124) Previously when running the compile_pip_requirements test from an external repository the test failed. This was due to the fact that paths in the annotations of the lock file contained paths from the external repository. Another problem was that the requirement files could not be located. --- .bazelci/presubmit.yml | 38 +++++++++++ .../build_file_generation/gazelle_python.yaml | 2 +- .../requirements_lock.txt | 2 +- .../requirements_windows.txt | 2 +- examples/bzlmod/requirements_lock.txt | 12 ++-- examples/bzlmod/requirements_windows.txt | 12 ++-- examples/pip_install/requirements.txt | 8 +-- examples/pip_install/requirements_windows.txt | 6 +- examples/pip_parse/requirements_lock.txt | 6 +- examples/pip_parse_vendored/requirements.txt | 2 +- .../requirements.txt | 4 +- .../test/requirements_parser_tests.bzl | 6 +- python/pip_install/requirements.bzl | 3 +- .../dependency_resolver.py | 63 +++++++++++++++---- .../requirements_lock.txt | 4 +- .../.bazelrc | 1 + .../.gitignore | 1 + .../BUILD.bazel | 0 .../README.md | 3 + .../WORKSPACE | 36 +++++++++++ .../requirements.txt | 6 +- .../requirements_windows.txt | 6 +- 22 files changed, 170 insertions(+), 53 deletions(-) create mode 100644 tests/compile_pip_requirements_test_from_external_workspace/.bazelrc create mode 100644 tests/compile_pip_requirements_test_from_external_workspace/.gitignore create mode 100644 tests/compile_pip_requirements_test_from_external_workspace/BUILD.bazel create mode 100644 tests/compile_pip_requirements_test_from_external_workspace/README.md create mode 100644 tests/compile_pip_requirements_test_from_external_workspace/WORKSPACE diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 0e9feab093..7b5ba8b2f2 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -509,3 +509,41 @@ tasks: name: ignore_root_user_error integration tests on Windows working_directory: tests/ignore_root_user_error platform: windows + + integration_compile_pip_requirements_test_from_external_repo_ubuntu_min: + <<: *minimum_supported_version + name: compile_pip_requirements test from external repo on Ubuntu using minimum supported Bazel version + working_directory: tests/compile_pip_requirements_test_from_external_workspace + platform: ubuntu2004 + shell_commands: + # Assert that @external_repository//:requirements_test does the right thing. + - "bazel test @external_repository//..." + integration_compile_pip_requirements_test_from_external_repo_ubuntu: + name: compile_pip_requirements test from external repo on Ubuntu + working_directory: tests/compile_pip_requirements_test_from_external_workspace + platform: ubuntu2004 + shell_commands: + # Assert that @external_repository//:requirements_test does the right thing. + - "bazel test @external_repository//..." + integration_compile_pip_requirements_test_from_external_repo_debian: + name: compile_pip_requirements test from external repo on Debian + working_directory: tests/compile_pip_requirements_test_from_external_workspace + platform: debian11 + shell_commands: + # Assert that @external_repository//:requirements_test does the right thing. + - "bazel test @external_repository//..." + integration_compile_pip_requirements_test_from_external_repo_macos: + name: compile_pip_requirements test from external repo on macOS + working_directory: tests/compile_pip_requirements_test_from_external_workspace + platform: macos + shell_commands: + # Assert that @external_repository//:requirements_test does the right thing. + - "bazel test @external_repository//..." + integration_compile_pip_requirements_test_from_external_repo_windows: + name: compile_pip_requirements test from external repo on Windows + working_directory: tests/compile_pip_requirements_test_from_external_workspace + platform: windows + shell_commands: + # Assert that @external_repository//:requirements_test does the right thing. + - "bazel test @external_repository//..." + diff --git a/examples/build_file_generation/gazelle_python.yaml b/examples/build_file_generation/gazelle_python.yaml index b57e9f02bc..1000757ea5 100644 --- a/examples/build_file_generation/gazelle_python.yaml +++ b/examples/build_file_generation/gazelle_python.yaml @@ -115,4 +115,4 @@ manifest: pip_repository: name: pip use_pip_repository_aliases: true -integrity: 85f073e37e31339508aaaf5e0d5472adae5148fd5f054e9cc586343c026660e1 +integrity: 030d6d99b56c32d6577e616b617260d0a93588af791269162e43391a5a4fa576 diff --git a/examples/build_file_generation/requirements_lock.txt b/examples/build_file_generation/requirements_lock.txt index f73827a36e..443db71ddc 100644 --- a/examples/build_file_generation/requirements_lock.txt +++ b/examples/build_file_generation/requirements_lock.txt @@ -11,7 +11,7 @@ click==8.1.3 \ flask==2.2.2 \ --hash=sha256:642c450d19c4ad482f96729bd2a8f6d32554aa1e231f4f6b4e7e5264b16cca2b \ --hash=sha256:b9c46cc36662a7949f34b52d8ec7bb59c0d74ba08ba6cb9ce9adc1d8676d9526 - # via -r ./requirements.in + # via -r requirements.in importlib-metadata==5.2.0 \ --hash=sha256:0eafa39ba42bf225fc00e67f701d71f85aead9f878569caf13c3724f704b970f \ --hash=sha256:404d48d62bba0b7a77ff9d405efd91501bef2e67ff4ace0bed40a0cf28c3c7cd diff --git a/examples/build_file_generation/requirements_windows.txt b/examples/build_file_generation/requirements_windows.txt index fc097141c5..bdd536fdcf 100644 --- a/examples/build_file_generation/requirements_windows.txt +++ b/examples/build_file_generation/requirements_windows.txt @@ -15,7 +15,7 @@ colorama==0.4.6 \ flask==2.2.2 \ --hash=sha256:642c450d19c4ad482f96729bd2a8f6d32554aa1e231f4f6b4e7e5264b16cca2b \ --hash=sha256:b9c46cc36662a7949f34b52d8ec7bb59c0d74ba08ba6cb9ce9adc1d8676d9526 - # via -r ./requirements.in + # via -r requirements.in importlib-metadata==5.2.0 \ --hash=sha256:0eafa39ba42bf225fc00e67f701d71f85aead9f878569caf13c3724f704b970f \ --hash=sha256:404d48d62bba0b7a77ff9d405efd91501bef2e67ff4ace0bed40a0cf28c3c7cd diff --git a/examples/bzlmod/requirements_lock.txt b/examples/bzlmod/requirements_lock.txt index 482402ffb8..2160fe1163 100644 --- a/examples/bzlmod/requirements_lock.txt +++ b/examples/bzlmod/requirements_lock.txt @@ -64,12 +64,12 @@ platformdirs==2.6.0 \ pylint==2.15.9 \ --hash=sha256:18783cca3cfee5b83c6c5d10b3cdb66c6594520ffae61890858fe8d932e1c6b4 \ --hash=sha256:349c8cd36aede4d50a0754a8c0218b43323d13d5d88f4b2952ddfe3e169681eb - # via -r ./requirements.in + # via -r requirements.in python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via - # -r ./requirements.in + # -r requirements.in # s3cmd python-magic==0.4.27 \ --hash=sha256:c1ba14b08e4a5f5c31a302b7721239695b2f0f058d125bd5ce1ee36b9d9d3c3b \ @@ -120,11 +120,11 @@ pyyaml==6.0 \ requests==2.25.1 \ --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \ --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e - # via -r ./requirements.in + # via -r requirements.in s3cmd==2.1.0 \ --hash=sha256:49cd23d516b17974b22b611a95ce4d93fe326feaa07320bd1d234fed68cbccfa \ --hash=sha256:966b0a494a916fc3b4324de38f089c86c70ee90e8e1cae6d59102103a4c0cc03 - # via -r ./requirements.in + # via -r requirements.in setuptools==65.6.3 \ --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \ --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75 @@ -136,7 +136,7 @@ six==1.16.0 \ tabulate==0.9.0 \ --hash=sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c \ --hash=sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f - # via -r ./requirements.in + # via -r requirements.in tomli==2.0.1 \ --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f @@ -224,4 +224,4 @@ wrapt==1.14.1 \ yamllint==1.28.0 \ --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b - # via -r ./requirements.in + # via -r requirements.in diff --git a/examples/bzlmod/requirements_windows.txt b/examples/bzlmod/requirements_windows.txt index 41187b9475..06cfdc332c 100644 --- a/examples/bzlmod/requirements_windows.txt +++ b/examples/bzlmod/requirements_windows.txt @@ -68,12 +68,12 @@ platformdirs==2.6.0 \ pylint==2.15.9 \ --hash=sha256:18783cca3cfee5b83c6c5d10b3cdb66c6594520ffae61890858fe8d932e1c6b4 \ --hash=sha256:349c8cd36aede4d50a0754a8c0218b43323d13d5d88f4b2952ddfe3e169681eb - # via -r ./requirements.in + # via -r requirements.in python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via - # -r ./requirements.in + # -r requirements.in # s3cmd python-magic==0.4.27 \ --hash=sha256:c1ba14b08e4a5f5c31a302b7721239695b2f0f058d125bd5ce1ee36b9d9d3c3b \ @@ -124,11 +124,11 @@ pyyaml==6.0 \ requests==2.25.1 \ --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \ --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e - # via -r ./requirements.in + # via -r requirements.in s3cmd==2.1.0 \ --hash=sha256:49cd23d516b17974b22b611a95ce4d93fe326feaa07320bd1d234fed68cbccfa \ --hash=sha256:966b0a494a916fc3b4324de38f089c86c70ee90e8e1cae6d59102103a4c0cc03 - # via -r ./requirements.in + # via -r requirements.in setuptools==65.6.3 \ --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \ --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75 @@ -140,7 +140,7 @@ six==1.16.0 \ tabulate==0.9.0 \ --hash=sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c \ --hash=sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f - # via -r ./requirements.in + # via -r requirements.in tomli==2.0.1 \ --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f @@ -228,4 +228,4 @@ wrapt==1.14.1 \ yamllint==1.28.0 \ --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b - # via -r ./requirements.in + # via -r requirements.in diff --git a/examples/pip_install/requirements.txt b/examples/pip_install/requirements.txt index ca8d5943a7..495a32a637 100644 --- a/examples/pip_install/requirements.txt +++ b/examples/pip_install/requirements.txt @@ -7,7 +7,7 @@ boto3==1.14.63 \ --hash=sha256:25c716b7c01d4664027afc6a6418a06459e311a610c7fd39a030a1ced1b72ce4 \ --hash=sha256:37158c37a151eab5b9080968305621a40168171fda9584d50a309ceb4e5e6964 - # via -r ./requirements.in + # via -r requirements.in botocore==1.17.63 \ --hash=sha256:40f13f6c9c29c307a9dc5982739e537ddce55b29787b90c3447b507e3283bcd6 \ --hash=sha256:aa88eafc6295132f4bc606f1df32b3248e0fa611724c0a216aceda767948ac75 @@ -84,7 +84,7 @@ pyyaml==6.0 \ s3cmd==2.1.0 \ --hash=sha256:49cd23d516b17974b22b611a95ce4d93fe326feaa07320bd1d234fed68cbccfa \ --hash=sha256:966b0a494a916fc3b4324de38f089c86c70ee90e8e1cae6d59102103a4c0cc03 - # via -r ./requirements.in + # via -r requirements.in s3transfer==0.3.7 \ --hash=sha256:35627b86af8ff97e7ac27975fe0a98a312814b46c6333d8a6b889627bcd80994 \ --hash=sha256:efa5bd92a897b6a8d5c1383828dca3d52d0790e0756d49740563a3fb6ed03246 @@ -100,11 +100,11 @@ six==1.16.0 \ tree-sitter==0.20.0 ; sys_platform != "win32" \ --hash=sha256:1940f64be1e8c9c3c0e34a2258f1e4c324207534d5b1eefc5ab2960a9d98f668 \ --hash=sha256:51a609a7c1bd9d9e75d92ee128c12c7852ae70a482900fbbccf3d13a79e0378c - # via -r ./requirements.in + # via -r requirements.in urllib3==1.25.11 \ --hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \ --hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e # via botocore yamllint==1.26.3 \ --hash=sha256:3934dcde484374596d6b52d8db412929a169f6d9e52e20f9ade5bf3523d9b96e - # via -r ./requirements.in + # via -r requirements.in diff --git a/examples/pip_install/requirements_windows.txt b/examples/pip_install/requirements_windows.txt index c4279cb6d7..b87192f9d0 100644 --- a/examples/pip_install/requirements_windows.txt +++ b/examples/pip_install/requirements_windows.txt @@ -7,7 +7,7 @@ boto3==1.14.63 \ --hash=sha256:25c716b7c01d4664027afc6a6418a06459e311a610c7fd39a030a1ced1b72ce4 \ --hash=sha256:37158c37a151eab5b9080968305621a40168171fda9584d50a309ceb4e5e6964 - # via -r ./requirements.in + # via -r requirements.in botocore==1.17.63 \ --hash=sha256:40f13f6c9c29c307a9dc5982739e537ddce55b29787b90c3447b507e3283bcd6 \ --hash=sha256:aa88eafc6295132f4bc606f1df32b3248e0fa611724c0a216aceda767948ac75 @@ -84,7 +84,7 @@ pyyaml==6.0 \ s3cmd==2.1.0 \ --hash=sha256:49cd23d516b17974b22b611a95ce4d93fe326feaa07320bd1d234fed68cbccfa \ --hash=sha256:966b0a494a916fc3b4324de38f089c86c70ee90e8e1cae6d59102103a4c0cc03 - # via -r ./requirements.in + # via -r requirements.in s3transfer==0.3.7 \ --hash=sha256:35627b86af8ff97e7ac27975fe0a98a312814b46c6333d8a6b889627bcd80994 \ --hash=sha256:efa5bd92a897b6a8d5c1383828dca3d52d0790e0756d49740563a3fb6ed03246 @@ -103,4 +103,4 @@ urllib3==1.25.11 \ # via botocore yamllint==1.26.3 \ --hash=sha256:3934dcde484374596d6b52d8db412929a169f6d9e52e20f9ade5bf3523d9b96e - # via -r ./requirements.in + # via -r requirements.in diff --git a/examples/pip_parse/requirements_lock.txt b/examples/pip_parse/requirements_lock.txt index d60295c0bf..3cbe57f28c 100644 --- a/examples/pip_parse/requirements_lock.txt +++ b/examples/pip_parse/requirements_lock.txt @@ -73,11 +73,11 @@ pyyaml==6.0 \ requests==2.25.1 \ --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \ --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e - # via -r ./requirements.in + # via -r requirements.in s3cmd==2.1.0 \ --hash=sha256:49cd23d516b17974b22b611a95ce4d93fe326feaa07320bd1d234fed68cbccfa \ --hash=sha256:966b0a494a916fc3b4324de38f089c86c70ee90e8e1cae6d59102103a4c0cc03 - # via -r ./requirements.in + # via -r requirements.in setuptools==65.6.3 \ --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \ --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75 @@ -92,4 +92,4 @@ urllib3==1.26.13 \ # via requests yamllint==1.26.3 \ --hash=sha256:3934dcde484374596d6b52d8db412929a169f6d9e52e20f9ade5bf3523d9b96e - # via -r ./requirements.in + # via -r requirements.in diff --git a/examples/pip_parse_vendored/requirements.txt b/examples/pip_parse_vendored/requirements.txt index 6a70e036b4..ff1a3633a2 100644 --- a/examples/pip_parse_vendored/requirements.txt +++ b/examples/pip_parse_vendored/requirements.txt @@ -19,7 +19,7 @@ idna==3.4 \ requests==2.28.1 \ --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 \ --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349 - # via -r ./requirements.in + # via -r requirements.in urllib3==1.26.13 \ --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc \ --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8 diff --git a/examples/pip_repository_annotations/requirements.txt b/examples/pip_repository_annotations/requirements.txt index f599f7a457..9fde0a922f 100644 --- a/examples/pip_repository_annotations/requirements.txt +++ b/examples/pip_repository_annotations/requirements.txt @@ -21,7 +21,7 @@ idna==3.4 \ requests[security]==2.28.1 \ --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 \ --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349 - # via -r ./requirements.in + # via -r requirements.in urllib3==1.26.13 \ --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc \ --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8 @@ -29,4 +29,4 @@ urllib3==1.26.13 \ wheel==0.38.4 \ --hash=sha256:965f5259b566725405b05e7cf774052044b1ed30119b5d586b2703aafe8719ac \ --hash=sha256:b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8 - # via -r ./requirements.in + # via -r requirements.in diff --git a/python/pip_install/private/test/requirements_parser_tests.bzl b/python/pip_install/private/test/requirements_parser_tests.bzl index 13c40c2956..c13ec204fb 100644 --- a/python/pip_install/private/test/requirements_parser_tests.bzl +++ b/python/pip_install/private/test/requirements_parser_tests.bzl @@ -177,11 +177,11 @@ pyyaml==6.0 \ requests==2.25.1 \ --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \ --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e - # via -r ./requirements.in + # via -r requirements.in s3cmd==2.1.0 \ --hash=sha256:49cd23d516b17974b22b611a95ce4d93fe326feaa07320bd1d234fed68cbccfa \ --hash=sha256:966b0a494a916fc3b4324de38f089c86c70ee90e8e1cae6d59102103a4c0cc03 - # via -r ./requirements.in + # via -r requirements.in six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 @@ -192,7 +192,7 @@ urllib3==1.26.7 \ # via requests yamllint==1.26.3 \ --hash=sha256:3934dcde484374596d6b52d8db412929a169f6d9e52e20f9ade5bf3523d9b96e - # via -r ./requirements.in + # via -r requirements.in # The following packages are considered to be unsafe in a requirements file: setuptools==59.6.0 \ diff --git a/python/pip_install/requirements.bzl b/python/pip_install/requirements.bzl index dd38c9df5b..7594471897 100644 --- a/python/pip_install/requirements.bzl +++ b/python/pip_install/requirements.bzl @@ -75,7 +75,7 @@ def compile_pip_requirements( # where it appears, which is to say, in @rules_python pip_compile = Label("//python/pip_install/tools/dependency_resolver:dependency_resolver.py") - loc = "$(rootpath {})" + loc = "$(rlocationpath {})" args = [ loc.format(requirements_in), @@ -99,6 +99,7 @@ def compile_pip_requirements( requirement("importlib_metadata"), requirement("zipp"), requirement("more_itertools"), + Label("//python/runfiles:runfiles"), ] + extra_deps tags = tags or [] diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/pip_install/tools/dependency_resolver/dependency_resolver.py index e636febd93..89e355806c 100644 --- a/python/pip_install/tools/dependency_resolver/dependency_resolver.py +++ b/python/pip_install/tools/dependency_resolver/dependency_resolver.py @@ -23,6 +23,8 @@ import piptools.writer as piptools_writer from piptools.scripts.compile import cli +from python.runfiles import runfiles + # Replace the os.replace function with shutil.copy to work around os.replace not being able to # replace or move files across filesystems. os.replace = shutil.copy @@ -66,6 +68,15 @@ def _select_golden_requirements_file( return requirements_txt +def _locate(bazel_runfiles, file): + """Look up the file via Rlocation""" + + if not file: + return file + + return bazel_runfiles.Rlocation(file) + + if __name__ == "__main__": if len(sys.argv) < 4: print( @@ -75,6 +86,7 @@ def _select_golden_requirements_file( sys.exit(1) parse_str_none = lambda s: None if s == "None" else s + bazel_runfiles = runfiles.Create() requirements_in = sys.argv.pop(1) requirements_txt = sys.argv.pop(1) @@ -83,10 +95,25 @@ def _select_golden_requirements_file( requirements_windows = parse_str_none(sys.argv.pop(1)) update_target_label = sys.argv.pop(1) - # The requirements_in file could be generated, so we will need to remove the - # absolute prefixes in the locked requirements output file. - requirements_in_path = Path(requirements_in) - resolved_requirements_in = str(requirements_in_path.resolve()) + resolved_requirements_in = _locate(bazel_runfiles, requirements_in) + resolved_requirements_txt = _locate(bazel_runfiles, requirements_txt) + + # Files in the runfiles directory has the following naming schema: + # Main repo: __main__/ + # External repo: / + # We want to strip both __main__ and from the absolute prefix + # to keep the requirements lock file agnostic. + repository_prefix = requirements_txt[: requirements_txt.index("/") + 1] + absolute_path_prefix = resolved_requirements_txt[ + : -(len(requirements_txt) - len(repository_prefix)) + ] + + # As requirements_in might contain references to generated files we want to + # use the runfiles file first. Thus, we need to compute the relative path + # from the execution root. + # Note: Windows cannot reference generated files without runfiles support enabled. + requirements_in_relative = requirements_in[len(repository_prefix) :] + requirements_txt_relative = requirements_txt[len(repository_prefix) :] # Before loading click, set the locale for its parser. # If it leaks through to the system setting, it may fail: @@ -112,7 +139,7 @@ def _select_golden_requirements_file( ) # Those two files won't necessarily be on the same filesystem, so we can't use os.replace # or shutil.copyfile, as they will fail with OSError: [Errno 18] Invalid cross-device link. - shutil.copy(requirements_txt, requirements_out) + shutil.copy(resolved_requirements_txt, requirements_out) update_command = os.getenv("CUSTOM_COMPILE_COMMAND") or "bazel run %s" % ( update_target_label, @@ -123,24 +150,33 @@ def _select_golden_requirements_file( sys.argv.append("--generate-hashes") sys.argv.append("--output-file") - sys.argv.append(requirements_txt if UPDATE else requirements_out) + sys.argv.append(requirements_txt_relative if UPDATE else requirements_out) sys.argv.append( - requirements_in if requirements_in_path.exists() else resolved_requirements_in + requirements_in_relative + if Path(requirements_in_relative).exists() + else resolved_requirements_in ) + print(sys.argv) if UPDATE: - print("Updating " + requirements_txt) + print("Updating " + requirements_txt_relative) if "BUILD_WORKSPACE_DIRECTORY" in os.environ: workspace = os.environ["BUILD_WORKSPACE_DIRECTORY"] - requirements_txt_tree = os.path.join(workspace, requirements_txt) + requirements_txt_tree = os.path.join(workspace, requirements_txt_relative) # In most cases, requirements_txt will be a symlink to the real file in the source tree. # If symlinks are not enabled (e.g. on Windows), then requirements_txt will be a copy, # and we should copy the updated requirements back to the source tree. - if not os.path.samefile(requirements_txt, requirements_txt_tree): + if not os.path.samefile(resolved_requirements_txt, requirements_txt_tree): atexit.register( - lambda: shutil.copy(requirements_txt, requirements_txt_tree) + lambda: shutil.copy( + resolved_requirements_txt, requirements_txt_tree + ) ) cli() + requirements_txt_relative_path = Path(requirements_txt_relative) + content = requirements_txt_relative_path.read_text() + content = content.replace(absolute_path_prefix, "") + requirements_txt_relative_path.write_text(content) else: # cli will exit(0) on success try: @@ -153,7 +189,7 @@ def _select_golden_requirements_file( print( "pip-compile exited with code 2. This means that pip-compile found " "incompatible requirements or could not find a version that matches " - f"the install requirement in {requirements_in}.", + f"the install requirement in {requirements_in_relative}.", file=sys.stderr, ) sys.exit(1) @@ -164,8 +200,9 @@ def _select_golden_requirements_file( requirements_darwin, requirements_windows, ) - golden = open(golden_filename).readlines() + golden = open(_locate(bazel_runfiles, golden_filename)).readlines() out = open(requirements_out).readlines() + out = [line.replace(absolute_path_prefix, "") for line in out] if golden != out: import difflib diff --git a/tests/compile_pip_requirements/requirements_lock.txt b/tests/compile_pip_requirements/requirements_lock.txt index 8f7037ce7a..4ca4a11f3e 100644 --- a/tests/compile_pip_requirements/requirements_lock.txt +++ b/tests/compile_pip_requirements/requirements_lock.txt @@ -7,8 +7,8 @@ pip==22.3.1 \ --hash=sha256:65fd48317359f3af8e593943e6ae1506b66325085ea64b706a998c6e83eeaf38 \ --hash=sha256:908c78e6bc29b676ede1c4d57981d490cb892eb45cd8c214ab6298125119e077 - # via -r ./requirements.in + # via -r requirements.in setuptools==65.6.3 \ --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \ --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75 - # via -r ./requirements_extra.in + # via -r requirements_extra.in diff --git a/tests/compile_pip_requirements_test_from_external_workspace/.bazelrc b/tests/compile_pip_requirements_test_from_external_workspace/.bazelrc new file mode 100644 index 0000000000..b98fc09774 --- /dev/null +++ b/tests/compile_pip_requirements_test_from_external_workspace/.bazelrc @@ -0,0 +1 @@ +test --test_output=errors diff --git a/tests/compile_pip_requirements_test_from_external_workspace/.gitignore b/tests/compile_pip_requirements_test_from_external_workspace/.gitignore new file mode 100644 index 0000000000..ac51a054d2 --- /dev/null +++ b/tests/compile_pip_requirements_test_from_external_workspace/.gitignore @@ -0,0 +1 @@ +bazel-* diff --git a/tests/compile_pip_requirements_test_from_external_workspace/BUILD.bazel b/tests/compile_pip_requirements_test_from_external_workspace/BUILD.bazel new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/compile_pip_requirements_test_from_external_workspace/README.md b/tests/compile_pip_requirements_test_from_external_workspace/README.md new file mode 100644 index 0000000000..8ce77ca1f1 --- /dev/null +++ b/tests/compile_pip_requirements_test_from_external_workspace/README.md @@ -0,0 +1,3 @@ +# compile_pip_requirements_test_from_external_workspace + +This test checks that compile_pip_requirements test can be run from external workspaces without invalidating the lock file. diff --git a/tests/compile_pip_requirements_test_from_external_workspace/WORKSPACE b/tests/compile_pip_requirements_test_from_external_workspace/WORKSPACE new file mode 100644 index 0000000000..35686a1d30 --- /dev/null +++ b/tests/compile_pip_requirements_test_from_external_workspace/WORKSPACE @@ -0,0 +1,36 @@ +local_repository( + name = "rules_python", + path = "../..", +) + +load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains") + +py_repositories() + +load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies") + +pip_install_dependencies() + +python_register_toolchains( + name = "python39", + python_version = "3.9", +) + +load("@python39//:defs.bzl", "interpreter") +load("@rules_python//python:pip.bzl", "pip_parse") + +local_repository( + name = "external_repository", + path = "../compile_pip_requirements", +) + +pip_parse( + name = "pypi", + python_interpreter_target = interpreter, + requirements_lock = "@external_repository//:requirements_lock.txt", +) + +load("@pypi//:requirements.bzl", "install_deps") + +# Initialize repositories for all packages in requirements_lock.txt. +install_deps() diff --git a/tests/pip_repository_entry_points/requirements.txt b/tests/pip_repository_entry_points/requirements.txt index 90b717e7a7..a93facc03b 100644 --- a/tests/pip_repository_entry_points/requirements.txt +++ b/tests/pip_repository_entry_points/requirements.txt @@ -170,7 +170,7 @@ setuptools==59.6.0 \ --hash=sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373 \ --hash=sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e # via - # -r ./requirements.in + # -r requirements.in # sphinx # yamllint snowballstemmer==2.2.0 \ @@ -180,7 +180,7 @@ snowballstemmer==2.2.0 \ sphinx==4.3.2 \ --hash=sha256:0a8836751a68306b3fe97ecbe44db786f8479c3bf4b80e3a7f5c838657b4698c \ --hash=sha256:6a11ea5dd0bdb197f9c2abc2e0ce73e01340464feaece525e64036546d24c851 - # via -r ./requirements.in + # via -r requirements.in sphinxcontrib-applehelp==1.0.2 \ --hash=sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a \ --hash=sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58 @@ -212,4 +212,4 @@ urllib3==1.26.7 \ yamllint==1.28.0 \ --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b - # via -r ./requirements.in + # via -r requirements.in diff --git a/tests/pip_repository_entry_points/requirements_windows.txt b/tests/pip_repository_entry_points/requirements_windows.txt index 14c3dc3274..651e2b5e56 100644 --- a/tests/pip_repository_entry_points/requirements_windows.txt +++ b/tests/pip_repository_entry_points/requirements_windows.txt @@ -174,7 +174,7 @@ setuptools==59.6.0 \ --hash=sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373 \ --hash=sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e # via - # -r ./requirements.in + # -r requirements.in # sphinx # yamllint snowballstemmer==2.2.0 \ @@ -184,7 +184,7 @@ snowballstemmer==2.2.0 \ sphinx==4.3.2 \ --hash=sha256:0a8836751a68306b3fe97ecbe44db786f8479c3bf4b80e3a7f5c838657b4698c \ --hash=sha256:6a11ea5dd0bdb197f9c2abc2e0ce73e01340464feaece525e64036546d24c851 - # via -r ./requirements.in + # via -r requirements.in sphinxcontrib-applehelp==1.0.2 \ --hash=sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a \ --hash=sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58 @@ -216,4 +216,4 @@ urllib3==1.26.7 \ yamllint==1.28.0 \ --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b - # via -r ./requirements.in + # via -r requirements.in