Use real rules_python

Initial motivation was to use py_proto_library from rules_python, but then a yak came along.

Fixes: bazelbuild#9029

Closes bazelbuild#17545.

PiperOrigin-RevId: 513834100
Change-Id: I11a99381e1169a9fb7a7a3eaa733ddd348ebac2b

BUILD

@@ -1,7 +1,7 @@
 # Bazel - Google's Build System
 
 load("//tools/distributions:distribution_rules.bzl", "distrib_jar_filegroup")
-load("//tools/python:private/defs.bzl", "py_binary")
+load("@rules_python//python:defs.bzl", "py_binary")
 load("@rules_license//rules:license.bzl", "license")
 load("@rules_pkg//pkg:tar.bzl", "pkg_tar")
 

MODULE.bazel

@@ -19,6 +19,8 @@ bazel_dep(name = "rules_cc", version = "0.0.6")
 bazel_dep(name = "rules_java", version = "5.4.1")
 bazel_dep(name = "rules_proto", version = "5.3.0-21.7")
 bazel_dep(name = "rules_jvm_external", version = "4.5")
+bazel_dep(name = "rules_python", version = "0.19.0")
+
 single_version_override(
     module_name = "rules_jvm_external",
     patch_strip = 1,
@@ -50,3 +52,5 @@ local_path_override(
     module_name = "googleapis",
     path = "./third_party/googleapis",
 )
+
+register_toolchains("@bazel_tools//tools/python:autodetecting_toolchain")

WORKSPACE

@@ -96,16 +96,6 @@ dist_http_archive(
     patch_cmds_win = EXPORT_WORKSPACE_IN_BUILD_BAZEL_FILE_WIN,
 )
 
-# This is a mock version of bazelbuild/rules_python that contains only
-# @rules_python//python:defs.bzl. It is used by protobuf.
-# TODO(#9029): We could potentially replace this with the real @rules_python.
-new_local_repository(
-    name = "rules_python",
-    build_file = "//third_party/rules_python:BUILD",
-    path = "./third_party/rules_python",
-    workspace_file = "//third_party/rules_python:rules_python.WORKSPACE",
-)
-
 local_repository(
     name = "googleapis",
     path = "./third_party/googleapis/",
@@ -214,6 +204,16 @@ dist_http_archive(
     patch_cmds_win = EXPORT_WORKSPACE_IN_BUILD_FILE_WIN,
 )
 
+dist_http_archive(
+    name = "rules_python",
+    patch_cmds = EXPORT_WORKSPACE_IN_BUILD_BAZEL_FILE,
+    patch_cmds_win = EXPORT_WORKSPACE_IN_BUILD_BAZEL_FILE_WIN,
+)
+
+load("@rules_python//python:repositories.bzl", "py_repositories")
+
+py_repositories()
+
 dist_http_archive(
     name = "zstd-jni",
     build_file = "//third_party:zstd-jni/zstd-jni.BUILD",
@@ -722,11 +722,11 @@ maven_install(
         "com.google.protobuf:protobuf-java",
         "com.google.protobuf:protobuf-javalite",
     ],
+    fail_if_repin_required = True,
+    maven_install_json = "//:maven_install.json",
     repositories = [
         "https://repo1.maven.org/maven2",
     ],
-    maven_install_json = "//:maven_install.json",
-    fail_if_repin_required = True,
     strict_visibility = True,
 )
 

distdir_deps.bzl

@@ -355,6 +355,13 @@ DIST_DEPS = {
         ],
         "package_version": "4.5",
     },
+    "rules_python": {
+        "sha256": "ffc7b877c95413c82bfd5482c017edcf759a6250d8b24e82f41f3c8b8d9e287e",
+        "strip_prefix": "rules_python-0.19.0",
+        "urls": ["https://github.com/bazelbuild/rules_python/releases/download/0.19.0/rules_python-0.19.0.tar.gz"],
+        "archive": "rules_python-0.19.0.tar.gz",
+        "used_in": ["additional_distfiles", "test_WORKSPACE_files"],
+    },
     "desugar_jdk_libs": {
         # Commit 5847d6a06302136d95a14b4cbd4b55a9c9f1436e of 2021-03-10
         "archive": "5847d6a06302136d95a14b4cbd4b55a9c9f1436e.zip",

scripts/docs/BUILD

@@ -1,4 +1,4 @@
-load("//tools/python:private/defs.bzl", "py_binary", "py_library")
+load("@rules_python//python:defs.bzl", "py_binary", "py_library")
 
 BUILD_SCM_REV_CMD = "$$(grep BUILD_SCM_REVISION bazel-out/volatile-status.txt | sed 's/^BUILD_SCM_REVISION //')"
 

src/BUILD

@@ -1,6 +1,6 @@
 # Packaging
 
-load("//tools/python:private/defs.bzl", "py_binary", "py_library")
+load("@rules_python//python:defs.bzl", "py_binary", "py_library")
 load(":embedded_tools.bzl", "srcsfile")
 load(":rule_size_test.bzl", "rule_size_test")
 load("//src:release_archive.bzl", "release_archive")

src/main/java/com/google/devtools/build/lib/bazel/rules/java/BUILD

@@ -96,6 +96,7 @@ gen_workspace_stanza(
         "rules_cc",
         "rules_java",
         "rules_proto",
+        "rules_python",
         "remotejdk11_linux",
         "remotejdk11_linux_aarch64",
         "remotejdk11_linux_ppc64le",

src/main/java/com/google/devtools/build/lib/bazel/rules/java/jdk.WORKSPACE.tmpl

@@ -284,4 +284,10 @@ maybe(
 {rules_proto}
 )
 
+# Needed only because of proguard_allowlister in java_tools.
+maybe(
+    http_archive,
+{rules_python}
+)
+
 register_toolchains("@bazel_tools//tools/jdk:all")

src/main/protobuf/BUILD

@@ -1,4 +1,4 @@
-load("@com_google_protobuf//:protobuf.bzl", "py_proto_library")
+load("@rules_python//python:proto.bzl", "py_proto_library")
 load("@rules_proto//proto:defs.bzl", "proto_library")
 load("//third_party/grpc:build_defs.bzl", "java_grpc_library")
 load("//third_party/grpc/bazel:cc_grpc_library.bzl", "cc_grpc_library")
@@ -47,10 +47,7 @@ proto_library(
 
 py_proto_library(
     name = "analysis_v2_py_proto",
-    srcs = ["analysis_v2.proto"],
-    default_runtime = "//third_party/protobuf:protobuf_python",
-    protoc = "//third_party/protobuf:protoc",
-    deps = [":build_pb_py"],
+    deps = [":analysis_v2_proto"],
 )
 
 java_proto_library(
@@ -187,9 +184,7 @@ cc_grpc_library(
 
 py_proto_library(
     name = "build_pb_py",
-    srcs = ["build.proto"],
-    default_runtime = "//third_party/protobuf:protobuf_python",
-    protoc = "//third_party/protobuf:protoc",
+    deps = [":build_proto"],
 )
 
 java_library_srcs(

src/test/java/com/google/devtools/build/lib/blackbox/bazel/DefaultToolsSetup.java

@@ -27,7 +27,12 @@
 public class DefaultToolsSetup implements ToolsSetup {
 
   private static ImmutableList<String> repos =
-      ImmutableList.<String>builder().add("rules_cc").add("rules_proto").add("rules_java").build();
+      ImmutableList.<String>builder()
+          .add("rules_cc")
+          .add("rules_proto")
+          .add("rules_java")
+          .add("rules_python")
+          .build();
 
   private ImmutableList<String> getRepositoryOverrides() {
     String sharedRepoHome = System.getenv("TEST_REPOSITORY_HOME");

src/test/java/com/google/devtools/build/lib/blackbox/framework/BUILD

@@ -46,6 +46,7 @@ gen_workspace_stanza(
     repos = [
         "rules_cc",
         "rules_proto",
+        "rules_python",
     ],
     visibility = ["//visibility:private"],
 )

src/test/java/com/google/devtools/build/lib/packages/util/BazelMockPythonSupport.java

@@ -38,7 +38,6 @@ public void setup(MockToolsConfig config) throws IOException {
     addTool(config, "tools/python/srcs_version.bzl");
     addTool(config, "tools/python/toolchain.bzl");
     addTool(config, "tools/python/utils.bzl");
-    addTool(config, "tools/python/private/defs.bzl");
     addTool(config, "tools/python/python_bootstrap_template.txt");
 
     config.create(

src/test/py/bazel/BUILD

@@ -1,5 +1,5 @@
 load("//:distdir_deps.bzl", "gen_workspace_stanza")
-load("//tools/python:private/defs.bzl", "py_library", "py_test")
+load("@rules_python//python:defs.bzl", "py_library", "py_test")
 
 package(default_visibility = ["//visibility:private"])
 
@@ -43,6 +43,7 @@ gen_workspace_stanza(
     repos = [
         "rules_license",
         "rules_cc",
+        "rules_python",
     ],
 )
 

src/test/py/bazel/test_base.py

@@ -58,6 +58,7 @@ class TestBase(unittest.TestCase):
       'rules_cc',
       'rules_java',
       'rules_proto',
+      'rules_python',
       'remotejdk11_linux_for_testing',
       'remotejdk11_linux_aarch64_for_testing',
       'remotejdk11_linux_ppc64le_for_testing',

src/test/shell/BUILD

@@ -1,5 +1,5 @@
 load("//:distdir_deps.bzl", "gen_workspace_stanza")
-load("//tools/python:private/defs.bzl", "py_test")
+load("@rules_python//python:defs.bzl", "py_test")
 
 package(default_visibility = ["//visibility:private"])
 
@@ -31,6 +31,7 @@ gen_workspace_stanza(
         "rules_license",
         "rules_pkg",
         "rules_proto",
+        "rules_python",
     ],
     template = "testenv.sh.tmpl",
     visibility = ["//visibility:public"],

src/test/shell/bazel/bazel_proto_library_test.sh

@@ -48,19 +48,6 @@ function write_workspace() {
     mkdir -p "$workspace"
   fi
 
-  cat >> "$workspace"WORKSPACE << EOF
-# TODO(#9029): May require some adjustment if/when we depend on the real
-# @rules_python in the real source tree, since this third_party/ package won't
-# be available.
-new_local_repository(
-    name = "rules_python",
-    path = "$(dirname $(rlocation io_bazel/third_party/rules_python/rules_python.WORKSPACE))",
-    build_file = "$(rlocation io_bazel/third_party/rules_python/BUILD)",
-    workspace_file = "$(rlocation io_bazel/third_party/rules_python/rules_python.WORKSPACE)",
-)
-
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-EOF
   cat $(rlocation io_bazel/src/tests/shell/bazel/rules_proto_stanza.txt) >> "$workspace"WORKSPACE
   cat >> "$workspace"WORKSPACE << EOF
 load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
@@ -70,7 +57,7 @@ rules_proto_toolchains()
 # @com_google_protobuf//:protoc depends on @io_bazel//third_party/zlib.
 new_local_repository(
     name = "io_bazel",
-    path = "$(dirname $(rlocation io_bazel/third_party/rules_python/rules_python.WORKSPACE))/../..",
+    path = "$(dirname $(rlocation io_bazel/third_party/zlib))/..",
     build_file_content = "# Intentionally left empty.",
     workspace_file_content = "workspace(name = 'io_bazel')",
 )

src/test/shell/integration/BUILD

@@ -41,6 +41,7 @@ gen_workspace_stanza(
     out = "rules_proto_stanza.txt",
     repos = [
         "rules_proto",
+        "rules_python",
     ],
 )
 

src/test/shell/integration/modify_execution_info_test.sh

@@ -151,18 +151,6 @@ Genrule=+requires-a,CppCompile=+requires-b,CppCompile=+requires-c \
 
 function test_modify_execution_info_various_types() {
   if [[ "$PRODUCT_NAME" = "bazel" ]]; then
-    # proto_library requires this external workspace.
-    cat >> WORKSPACE << EOF
-# TODO(#9029): May require some adjustment if/when we depend on the real
-# @rules_python in the real source tree, since this third_party/ package won't
-# be available.
-new_local_repository(
-    name = "rules_python",
-    path = "$(dirname $(rlocation io_bazel/third_party/rules_python/rules_python.WORKSPACE))",
-    build_file = "$(rlocation io_bazel/third_party/rules_python/BUILD)",
-    workspace_file = "$(rlocation io_bazel/third_party/rules_python/rules_python.WORKSPACE)",
-)
-EOF
     cat "$(rlocation "io_bazel/src/test/shell/integration/rules_proto_stanza.txt")" >>WORKSPACE
     cat >> WORKSPACE << EOF
 load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
@@ -172,7 +160,7 @@ rules_proto_toolchains()
 # @com_google_protobuf//:protoc depends on @io_bazel//third_party/zlib.
 new_local_repository(
     name = "io_bazel",
-    path = "$(dirname $(rlocation io_bazel/third_party/rules_python/rules_python.WORKSPACE))/../..",
+    path = "$(dirname $(rlocation io_bazel/third_party/zlib))/..",
     build_file_content = "# Intentionally left empty.",
     workspace_file_content = "workspace(name = 'io_bazel')",
 )

src/test/shell/testenv.sh.tmpl

@@ -561,6 +561,14 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 EOF
 }
 
+function add_rules_python_to_workspace() {
+  cat >> "$1"<<EOF
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+{rules_python}
+EOF
+}
+
 function create_workspace_with_default_repos() {
   write_workspace_file "${1:-WORKSPACE}" "${2:-main}"
   echo "$1"
@@ -576,6 +584,7 @@ EOF
   add_rules_license_to_workspace "WORKSPACE"
   add_rules_pkg_to_workspace "WORKSPACE"
   add_rules_proto_to_workspace "WORKSPACE"
+  add_rules_python_to_workspace "WORKSPACE"
 
   maybe_setup_python_windows_workspace
 }

tools/android/BUILD

@@ -1,4 +1,4 @@
-load("//tools/python:private/defs.bzl", "py_binary", "py_library", "py_test")
+load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
 
 package(default_visibility = ["//tools:__pkg__"])
 

tools/android/BUILD.tools

@@ -1,4 +1,4 @@
-load("//tools/python:private/defs.bzl", "py_binary", "py_library")
+load("@rules_python//python:defs.bzl", "py_binary", "py_library")
 load(":defs.bzl", "run_ijar", "run_singlejar")
 
 package(default_visibility = ["//visibility:public"])

tools/aquery_differ/BUILD

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 load("//tools/python:private/py_test_alias.bzl", "py_test_alias")
-load("//tools/python:private/defs.bzl", "py_binary", "py_library")
 
 package(default_visibility = ["//visibility:public"])
 

tools/build_defs/hash/BUILD

@@ -1,4 +1,4 @@
-load("//tools/python:private/defs.bzl", "py_binary")
+load("@rules_python//python:defs.bzl", "py_binary")
 
 licenses(["notice"])  # Apache 2.0
 

tools/build_defs/pkg/BUILD

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 load("//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
-load("//tools/python:private/defs.bzl", "py_binary", "py_library", "py_test")
+load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
 
 licenses(["notice"])
 

tools/build_rules/BUILD

@@ -1,4 +1,4 @@
-load("//tools/python:private/defs.bzl", "py_test")
+load("@rules_python//python:defs.bzl", "py_test")
 
 licenses(["notice"])  # Apache 2.0
 

tools/ctexplain/BUILD

@@ -1,6 +1,6 @@
 # Description:
 #   Tool for measuring how configuration transitions affect build graph size.
-load("//tools/python:private/defs.bzl", "py_binary", "py_library")
+load("@rules_python//python:defs.bzl", "py_binary", "py_library")
 
 package(default_visibility = ["//visibility:public"])
 

tools/jdk/BUILD.tools

@@ -14,7 +14,7 @@ load(
     "java_runtime_version_alias",
     "java_toolchain_alias",
 )
-load("//tools/python:private/defs.bzl", "py_binary", "py_test")
+load("@rules_python//python:defs.bzl", "py_binary", "py_test")
 
 package(default_visibility = ["//visibility:public"])
 

tools/mini_tar/BUILD

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 load("//tools/mini_tar:tar.bzl", "mini_tar")
-load("//tools/python:private/defs.bzl", "py_binary", "py_test")
 
 licenses(["notice"])
 

tools/python/BUILD

@@ -1,4 +1,4 @@
-load("//tools/python:private/defs.bzl", "py_test")
+load("@rules_python//python:defs.bzl", "py_test")
 load(":utils.bzl", "expand_pyversion_template")
 
 package(default_visibility = ["//visibility:public"])

tools/python/BUILD.tools

@@ -29,7 +29,6 @@ exports_files([
     "srcs_version.bzl",
     "toolchain.bzl",
     "utils.bzl",
-    "private/defs.bzl",
     "private/py_test_alias.bzl",
     # write_file.bzl fortunately doesn't need to be exposed because it's only
     # used in this BUILD file.
@@ -38,7 +37,7 @@ exports_files([
 filegroup(
     name = "bzl_srcs",
     srcs = glob(["*.bzl"]) + [
-        "private/defs.bzl",
+        "private/defs.bzl", # TODO(#9029): remove after the uses in bazel/third_party are removed
         "private/py_test_alias.bzl",
     ],
     visibility = ["//tools:__pkg__"],

tools/python/private/defs.bzl

@@ -31,6 +31,8 @@ WORKSPACE, and rely on those repos using the proper @rules_python-qualified
 label. (#9029 tracks possibly replacing the mock with the real thing.)
 """
 
+# TODO(#9029): the file can be deleted after the uses in bazel/third_party are removed
+
 _MIGRATION_TAG = "__PYTHON_RULES_MIGRATION_DO_NOT_USE_WILL_BREAK__"
 
 def _add_tags(attrs):