From 0334a6cac80489af0f9b65b79be208ac296afef0 Mon Sep 17 00:00:00 2001 From: Brian Dubois Date: Wed, 6 Jul 2022 12:03:55 -0400 Subject: [PATCH] Patch protobuf and grpc dependencies to be Python 3.10 compatible. (#5793) This PR allows TensorBoard to build and run with Python 3.10. There are incompatibilities in the protobuf and grpc dependencies, which are quite old but we are unable to upgrade (see: #5561 , Googlers: http://b/219030239). We patch the necessary fixes from those projects into our older versions using the `patches` property of the `http_archive` rule. For grpc we patch https://github.com/grpc/grpc/commit/dbe73c9004e483d24168c220cd589fe1824e72bc to avoid the following compile-time error: ``` Error in fail: Python Configuration Error: Problem getting python include path for /usr/local/google/home/bdubois/virtualenv/tensorboard/bin/python. :1: DeprecationWarning: The distutils package is deprecated and slated for removal in Python 3.12. Use setuptools or check PEP 632 for potential alternatives :1: DeprecationWarning: The distutils.sysconfig module is deprecated, use sysconfig instead Is the Python binary path set up right? (See ./configure or /usr/local/google/home/bdubois/virtualenv/tensorboard/bin/python.) Is distutils installed? Are Python headers installed? Try installing python-dev or python3-dev on Debian-based systems. Try python-devel or python3-devel on Redhat-based systems. ``` For protobuf we patch https://github.com/protocolbuffers/protobuf/commit/9d61eada0f47d7be793983638c4a29707b192d0c to avoid the following runtime error at startup: ``` File "/usr/local/google/home/bdubois/.cache/bazel/_bazel_bdubois/079646a57be11faea0b2bfefccb2a81a/execroot/org_tensorflow_tensorboard/bazel-out/k8-fastbuild/bin/tensorboard/dev.runfiles/com_google_protobuf/python/google/protobuf/descriptor.py", line 47, in from google.protobuf.pyext import _message AttributeError: module 'collections' has no attribute 'MutableSequence' ``` The changes patched into message.cc are not the latest. I considered also patching https://github.com/protocolbuffers/protobuf/commit/624d29d83306f3ce2c7e092dd44a891e04215e67 to remove the references to PY_MAJOR_VERSION but that change is wide in scope. --- WORKSPACE | 40 ++++++++++++++++++++++++++++++++++++++ third_party/grpc.patch | 33 +++++++++++++++++++++++++++++++ third_party/protobuf.patch | 18 +++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 third_party/grpc.patch create mode 100644 third_party/protobuf.patch diff --git a/WORKSPACE b/WORKSPACE index 390834bd385..a57df2dd66b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -110,6 +110,24 @@ sass_repositories() # might lead to test failures. http_archive( name = "com_google_protobuf", + patch_args = ["-p1"], + patches = [ + # To maintain compatibility with python 3.10 and greater, we need to patch + # in the following protobuf change: + # https://github.com/grpc/grpc/commit/9d61eada0f47d7be793983638c4a29707b192d0c + # + # To reproduce the patch: + # ``` + # $ git clone https://github.com/protocolbuffers/protobuf.git + # $ cd protobuf + # $ git checkout tags/v3.9.2 -b my-patch + # $ git cherry-pick 9d61eada0f47d7be793983638c4a29707b192d0c + # $ git diff HEAD~1 > protobuf.patch + # # Remove trailing whitespace to satisify whitespace_hygiene_test.py. + # $ sed -i 's/[[:space:]]*$//' protobuf.patch + # ``` + "//third_party:protobuf.patch", + ], sha256 = "1fbf1c2962af287607232b2eddeaec9b4f4a7a6f5934e1a9276e9af76952f7e0", strip_prefix = "protobuf-3.9.2", urls = [ @@ -121,12 +139,34 @@ http_archive( # gRPC. http_archive( name = "com_github_grpc_grpc", + patch_args = ["-p1"], + patches = [ + # To maintain compatibility with python 3.10 and greater, we need to patch + # in the following grpc change: + # https://github.com/grpc/grpc/commit/dbe73c9004e483d24168c220cd589fe1824e72bc + # + # To reproduce the patch: + # ``` + # $ git clone https://github.com/grpc/grpc.git + # $ cd grpc + # $ git checkout tags/v1.27.0-pre1 -b my-patch + # $ git cherry-pick dbe73c9004e483d24168c220cd589fe1824e72bc + # $ git diff HEAD~1 > grpc.patch + # ``` + # + # Note that we choose tags/v1.27.0-pre1 as the base since the version in + # the archive is 1.27.0-dev. It's not necessarily an exact match but is + # the best guess we can make for a match. + "//third_party:grpc.patch", + ], sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", urls = [ # Same as TF: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/workspace2.bzl#L492 # Currently we can't upgrade gRPC past 1.30.0 without also bumping protobuf to 3.12.0+: # https://github.com/grpc/grpc/issues/23311. + # + # Inspecting the contents of this archive, the version is 1.27.0-dev. "http://mirror.tensorflow.org/github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz", "https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz", ], diff --git a/third_party/grpc.patch b/third_party/grpc.patch new file mode 100644 index 00000000000..bcc1c7f6cd9 --- /dev/null +++ b/third_party/grpc.patch @@ -0,0 +1,33 @@ +diff --git a/third_party/py/python_configure.bzl b/third_party/py/python_configure.bzl +index 34501c3212..7f178f4009 100644 +--- a/third_party/py/python_configure.bzl ++++ b/third_party/py/python_configure.bzl +@@ -198,8 +198,8 @@ def _get_python_lib(repository_ctx, python_bin, lib_path_key): + " python_paths = os.getenv('PYTHONPATH').split(':')\n" + "try:\n" + + " library_paths = site.getsitepackages()\n" + + "except AttributeError:\n" + +- " from distutils.sysconfig import get_python_lib\n" + +- " library_paths = [get_python_lib()]\n" + ++ " import sysconfig\n" + ++ " library_paths = [sysconfig.get_path('purelib')]\n" + + "all_paths = set(python_paths + library_paths)\n" + "paths = []\n" + + "for path in all_paths:\n" + " if os.path.isdir(path):\n" + + " paths.append(path)\n" + "if len(paths) >=1:\n" + +@@ -232,14 +232,13 @@ def _get_python_include(repository_ctx, python_bin): + python_bin, + "-c", + "from __future__ import print_function;" + +- "from distutils import sysconfig;" + +- "print(sysconfig.get_python_inc())", ++ "import sysconfig;" + ++ "print(sysconfig.get_path('include'))", + ], + error_msg = "Problem getting python include path for {}.".format(python_bin), + error_details = ( + "Is the Python binary path set up right? " + "(See ./configure or " + +- python_bin + ".) " + "Is distutils installed? " + +- _HEADERS_HELP ++ python_bin + ".) " + _HEADERS_HELP + ), + ) + include_path = result.stdout.splitlines()[0] diff --git a/third_party/protobuf.patch b/third_party/protobuf.patch new file mode 100644 index 00000000000..61d1860188c --- /dev/null +++ b/third_party/protobuf.patch @@ -0,0 +1,18 @@ +diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc +index 3530a9b37..c31fa8fcc 100644 +--- a/python/google/protobuf/pyext/message.cc ++++ b/python/google/protobuf/pyext/message.cc +@@ -2991,8 +2991,12 @@ bool InitProto2MessageModule(PyObject *m) { + reinterpret_cast( + &RepeatedCompositeContainer_Type)); + +- // Register them as collections.Sequence ++ // Register them as MutableSequence. ++#if PY_MAJOR_VERSION >= 3 ++ ScopedPyObjectPtr collections(PyImport_ImportModule("collections.abc")); ++#else + ScopedPyObjectPtr collections(PyImport_ImportModule("collections")); ++#endif + if (collections == NULL) { + return false; + }