Skip to content

Commit

Permalink
Patch protobuf and grpc dependencies to be Python 3.10 compatible. (#…
Browse files Browse the repository at this point in the history
…5793)

This PR allows TensorBoard to build and run with Python 3.10.

There are incompatibilities in the protobuf and grpc dependencies, which are quite old but we are unable to upgrade (see: #5561 , Googlers: http://b/219030239). We patch the necessary fixes from those projects into our older versions using the `patches` property of the `http_archive` rule.

For grpc we patch grpc/grpc@dbe73c9 to avoid the following compile-time error:

```
Error in fail: Python Configuration Error: Problem getting python include path for /usr/local/google/home/bdubois/virtualenv/tensorboard/bin/python.
<string>:1: DeprecationWarning: The distutils package is deprecated and slated for removal in Python 3.12. Use setuptools or check PEP 632 for potential alternatives
<string>:1: DeprecationWarning: The distutils.sysconfig module is deprecated, use sysconfig instead
Is the Python binary path set up right? (See ./configure or /usr/local/google/home/bdubois/virtualenv/tensorboard/bin/python.) Is distutils installed? Are Python headers installed? Try installing python-dev or python3-dev on Debian-based systems. Try python-devel or python3-devel on Redhat-based systems.
```

For protobuf we patch protocolbuffers/protobuf@9d61ead to avoid the following runtime error at startup:

```
 File "/usr/local/google/home/bdubois/.cache/bazel/_bazel_bdubois/079646a57be11faea0b2bfefccb2a81a/execroot/org_tensorflow_tensorboard/bazel-out/k8-fastbuild/bin/tensorboard/dev.runfiles/com_google_protobuf/python/google/protobuf/descriptor.py", line 47, in <module>
    from google.protobuf.pyext import _message
AttributeError: module 'collections' has no attribute 'MutableSequence'
```

The changes patched into message.cc are not the latest. I considered also patching protocolbuffers/protobuf@624d29d to remove the references to PY_MAJOR_VERSION but that change is wide in scope.
  • Loading branch information
bmd3k authored Jul 6, 2022
1 parent 7e60aa4 commit 0334a6c
Show file tree
Hide file tree
Showing 3 changed files with 91 additions and 0 deletions.
40 changes: 40 additions & 0 deletions WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,24 @@ sass_repositories()
# might lead to test failures.
http_archive(
name = "com_google_protobuf",
patch_args = ["-p1"],
patches = [
# To maintain compatibility with python 3.10 and greater, we need to patch
# in the following protobuf change:
# https://github.com/grpc/grpc/commit/9d61eada0f47d7be793983638c4a29707b192d0c
#
# To reproduce the patch:
# ```
# $ git clone https://github.com/protocolbuffers/protobuf.git
# $ cd protobuf
# $ git checkout tags/v3.9.2 -b my-patch
# $ git cherry-pick 9d61eada0f47d7be793983638c4a29707b192d0c
# $ git diff HEAD~1 > protobuf.patch
# # Remove trailing whitespace to satisify whitespace_hygiene_test.py.
# $ sed -i 's/[[:space:]]*$//' protobuf.patch
# ```
"//third_party:protobuf.patch",
],
sha256 = "1fbf1c2962af287607232b2eddeaec9b4f4a7a6f5934e1a9276e9af76952f7e0",
strip_prefix = "protobuf-3.9.2",
urls = [
Expand All @@ -121,12 +139,34 @@ http_archive(
# gRPC.
http_archive(
name = "com_github_grpc_grpc",
patch_args = ["-p1"],
patches = [
# To maintain compatibility with python 3.10 and greater, we need to patch
# in the following grpc change:
# https://github.com/grpc/grpc/commit/dbe73c9004e483d24168c220cd589fe1824e72bc
#
# To reproduce the patch:
# ```
# $ git clone https://github.com/grpc/grpc.git
# $ cd grpc
# $ git checkout tags/v1.27.0-pre1 -b my-patch
# $ git cherry-pick dbe73c9004e483d24168c220cd589fe1824e72bc
# $ git diff HEAD~1 > grpc.patch
# ```
#
# Note that we choose tags/v1.27.0-pre1 as the base since the version in
# the archive is 1.27.0-dev. It's not necessarily an exact match but is
# the best guess we can make for a match.
"//third_party:grpc.patch",
],
sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f",
strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd",
urls = [
# Same as TF: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/workspace2.bzl#L492
# Currently we can't upgrade gRPC past 1.30.0 without also bumping protobuf to 3.12.0+:
# https://github.com/grpc/grpc/issues/23311.
#
# Inspecting the contents of this archive, the version is 1.27.0-dev.
"http://mirror.tensorflow.org/github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz",
"https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz",
],
Expand Down
33 changes: 33 additions & 0 deletions third_party/grpc.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
diff --git a/third_party/py/python_configure.bzl b/third_party/py/python_configure.bzl
index 34501c3212..7f178f4009 100644
--- a/third_party/py/python_configure.bzl
+++ b/third_party/py/python_configure.bzl
@@ -198,8 +198,8 @@ def _get_python_lib(repository_ctx, python_bin, lib_path_key):
" python_paths = os.getenv('PYTHONPATH').split(':')\n" + "try:\n" +
" library_paths = site.getsitepackages()\n" +
"except AttributeError:\n" +
- " from distutils.sysconfig import get_python_lib\n" +
- " library_paths = [get_python_lib()]\n" +
+ " import sysconfig\n" +
+ " library_paths = [sysconfig.get_path('purelib')]\n" +
"all_paths = set(python_paths + library_paths)\n" + "paths = []\n" +
"for path in all_paths:\n" + " if os.path.isdir(path):\n" +
" paths.append(path)\n" + "if len(paths) >=1:\n" +
@@ -232,14 +232,13 @@ def _get_python_include(repository_ctx, python_bin):
python_bin,
"-c",
"from __future__ import print_function;" +
- "from distutils import sysconfig;" +
- "print(sysconfig.get_python_inc())",
+ "import sysconfig;" +
+ "print(sysconfig.get_path('include'))",
],
error_msg = "Problem getting python include path for {}.".format(python_bin),
error_details = (
"Is the Python binary path set up right? " + "(See ./configure or " +
- python_bin + ".) " + "Is distutils installed? " +
- _HEADERS_HELP
+ python_bin + ".) " + _HEADERS_HELP
),
)
include_path = result.stdout.splitlines()[0]
18 changes: 18 additions & 0 deletions third_party/protobuf.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc
index 3530a9b37..c31fa8fcc 100644
--- a/python/google/protobuf/pyext/message.cc
+++ b/python/google/protobuf/pyext/message.cc
@@ -2991,8 +2991,12 @@ bool InitProto2MessageModule(PyObject *m) {
reinterpret_cast<PyObject*>(
&RepeatedCompositeContainer_Type));

- // Register them as collections.Sequence
+ // Register them as MutableSequence.
+#if PY_MAJOR_VERSION >= 3
+ ScopedPyObjectPtr collections(PyImport_ImportModule("collections.abc"));
+#else
ScopedPyObjectPtr collections(PyImport_ImportModule("collections"));
+#endif
if (collections == NULL) {
return false;
}

0 comments on commit 0334a6c

Please sign in to comment.