From 8cbe4e0f6889d4c6b69863479bd2da7b3679d810 Mon Sep 17 00:00:00 2001 From: Johnny Chavez <64660690+calderjo@users.noreply.github.com> Date: Thu, 8 Feb 2024 01:49:47 -0800 Subject: [PATCH 1/7] add patch for tf-df pt1 --- Dockerfile.tmpl | 57 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 4e04b695..00000d6b 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -202,18 +202,63 @@ RUN apt-get install -y default-jre && \ RUN pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o && /tmp/clean-layer.sh +# b/318672158 Use simply tensorflow-probability once > 0.23.0 is released. RUN pip install \ "tensorflow==${TENSORFLOW_VERSION}" \ "tensorflow-io==${TENSORFLOW_IO_VERSION}" \ - tensorflow_decision_forests \ + git+https://github.com/tensorflow/probability.git@fbc5ebe9b1d343113fb917010096cfd88b32eecf \ tensorflow_text \ - tensorflowjs \ - tensorflow_hub && \ + "tensorflow_hub>=0.16.0" \ + tf-keras && \ /tmp/clean-layer.sh -# TODO(b/318672158): Upgrade to Keras 3 once compatible with other TF libries. -# See blockers here: https://b.corp.google.com/issues/319722433#comment8 -RUN pip install keras keras-cv keras-nlp && \ +# b/318672158 Use simply tensorflow_decision_forests on next release, expected with tf 2.16 +RUN pip install tensorflow_decision_forests --no-deps && \ + /tmp/clean-layer.sh + +RUN sed -i "/import tensorflow_decision_forests as tfdf/a import tf_keras" /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/__init__.py && \ + sed -i -e "/import tensorflow as tf/a import tf_keras" \ + -e "/from yggdrasil_decision_forests.utils.distribute.implementations.grpc/a from tensorflow_decision_forests.keras import keras_internal" \ + -e '/try:/{:a;N;/backend = tf.keras.backend/!ba;d}'\ + /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/core.py && \ + sed -i -e "s/from typing import Optional, List, Dict, Any, Union, NamedTuple/from typing import Any, Dict, List, NamedTuple, Optional, Union/g" \ + -e "/import tensorflow as tf/a from tensorflow_decision_forests.keras import keras_internal" \ + -e "/import tensorflow as tf/a import tf_keras" \ + -e '/layers = tf.keras.layers/{:a;N;/backend = tf.keras.backend/!ba;d}' \ + /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/core_inference.py && \ + find /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests -type f -exec sed -i \ + -e "s/get_data_handler/keras_internal.get_data_handler/g" \ + -e 's/"models.Functional"/keras_internal.Functional/g' \ + -e "s/tf.keras.utils.unpack_x_y_sample_weight/keras_internal.unpack_x_y_sample_weight/g" \ + -e "s/tf.keras.utils.experimental/keras_internal/g" \ + {} \; && \ + sed -i -e "/import tensorflow as tf/a import tf_keras" \ + -e "/from tensorflow_decision_forests.keras import core/a from tensorflow_decision_forests.keras import keras_internal" \ + -e '/layers = tf.keras.layers/{:a;N;/callbacks = tf.keras.callbacks/!ba;d}' \ + /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/keras_test.py && \ + find /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras -type f -exec sed -i \ + -e "s/ layers.Input/ tf_keras.layers.Input/g" \ + -e "s/layers.minimum/tf_keras.layers.minimum/g" \ + -e "s/layers.Concatenate/tf_keras.layers.Concatenate/g" \ + -e "s/layers.Dense/tf_keras.layers.Dense/g" \ + -e "s/layers.experimental.preprocessing./tf_keras.layers./g" \ + -e "s/layers.DenseFeatures/keras_internal.layers.DenseFeatures/g" \ + -e "s/models.Model/tf_keras.models.Model/g" {} \; && \ + sed -i "s/ models.load_model/ tf_keras.models.load_model/g" /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/keras_test.py && \ + sed -i "/import tensorflow as tf/a import tf_keras" /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/test_runner.py && \ + sed -i "/import tensorflow as tf/a import tf_keras" /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/wrappers.py && \ + sed -i -e "/import tensorflow as tf/a import tf_keras" \ + -e "s/optimizer=optimizers.Adam()/optimizer=tf_keras.optimizers.Adam()/g" \ + /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/wrappers_pre_generated.py && \ + find /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests -type f -exec sed -i "s/tf.keras./tf_keras./g" {} \; + +ADD patches/keras_internal.py /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/keras_internal.py +ADD patches/keras_internal_test.py /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/keras_internal_test.py + +# Remove "--no-deps" flag and "namex" package once Keras 3.* is included in our base image. +# We ignore dependencies since tf2.15 and Keras 3.* should work despite pip saying it won't. +# Currently, keras tries to install a nightly version of tf 2.16: https://github.com/keras-team/keras/blob/fe2f54aa5bc42fb23a96449cf90434ab9bb6a2cd/requirements.txt#L2 +RUN pip install --no-deps "keras>3" keras-cv keras-nlp namex && \ /tmp/clean-layer.sh RUN pip install pysal From afb87038d820f1c883bc681b40fbe3b1a8cd6673 Mon Sep 17 00:00:00 2001 From: Johnny Chavez <64660690+calderjo@users.noreply.github.com> Date: Thu, 8 Feb 2024 01:50:55 -0800 Subject: [PATCH 2/7] Create keras_internal.py --- patches/keras_internal.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 patches/keras_internal.py diff --git a/patches/keras_internal.py b/patches/keras_internal.py new file mode 100644 index 00000000..e28127f9 --- /dev/null +++ b/patches/keras_internal.py @@ -0,0 +1,24 @@ +# Copyright 2021 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Access to Keras function with a different internal and external path.""" + +from tf_keras.src.engine import data_adapter as _data_adapter +from tf_keras.src.models import Functional +from tf_keras.layers import DenseFeatures +from tf_keras.src.utils.dataset_creator import DatasetCreator + + +unpack_x_y_sample_weight = _data_adapter.unpack_x_y_sample_weight +get_data_handler = _data_adapter.get_data_handler From 00c3e4de41c322d555fadb77f11adb5a546986b5 Mon Sep 17 00:00:00 2001 From: Johnny Chavez <64660690+calderjo@users.noreply.github.com> Date: Thu, 8 Feb 2024 01:51:27 -0800 Subject: [PATCH 3/7] Create keras_internal_test.py --- patches/keras_internal_test.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 patches/keras_internal_test.py diff --git a/patches/keras_internal_test.py b/patches/keras_internal_test.py new file mode 100644 index 00000000..edc33ec2 --- /dev/null +++ b/patches/keras_internal_test.py @@ -0,0 +1,23 @@ +# Copyright 2021 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tensorflow as tf +from tensorflow_decision_forests.keras import keras_internal + + +# Does nothing. Ensures keras_internal can be loaded. + +if __name__ == "__main__": + tf.test.main() + From b9f727c19505743e90cf35ad8b3dd0a3d8d2b48c Mon Sep 17 00:00:00 2001 From: Johnny Chavez <64660690+calderjo@users.noreply.github.com> Date: Thu, 8 Feb 2024 09:44:46 -0800 Subject: [PATCH 4/7] Create keras_patch.sh --- patches/keras_patch.sh | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 patches/keras_patch.sh diff --git a/patches/keras_patch.sh b/patches/keras_patch.sh new file mode 100644 index 00000000..9f219026 --- /dev/null +++ b/patches/keras_patch.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# The following "sed" are to patch the current version of tf-df with +# a fix for keras 3. In essence, replaces the use of package name "tf.keras" with +# "tf_keras" + +sed -i "/import tensorflow_decision_forests as tfdf/a import tf_keras" /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/__init__.py && \ +sed -i -e "/import tensorflow as tf/a import tf_keras" \ + -e "/from yggdrasil_decision_forests.utils.distribute.implementations.grpc/a from tensorflow_decision_forests.keras import keras_internal" \ + -e '/try:/{:a;N;/backend = tf.keras.backend/!ba;d}'\ + /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/core.py && \ +sed -i -e "s/from typing import Optional, List, Dict, Any, Union, NamedTuple/from typing import Any, Dict, List, NamedTuple, Optional, Union/g" \ + -e "/import tensorflow as tf/a from tensorflow_decision_forests.keras import keras_internal" \ + -e "/import tensorflow as tf/a import tf_keras" \ + -e '/layers = tf.keras.layers/{:a;N;/backend = tf.keras.backend/!ba;d}' \ + /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/core_inference.py && \ +find /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests -type f -exec sed -i \ + -e "s/get_data_handler/keras_internal.get_data_handler/g" \ + -e 's/"models.Functional"/keras_internal.Functional/g' \ + -e "s/tf.keras.utils.unpack_x_y_sample_weight/keras_internal.unpack_x_y_sample_weight/g" \ + -e "s/tf.keras.utils.experimental/keras_internal/g" \ + {} \; && \ +sed -i -e "/import tensorflow as tf/a import tf_keras" \ + -e "/from tensorflow_decision_forests.keras import core/a from tensorflow_decision_forests.keras import keras_internal" \ + -e '/layers = tf.keras.layers/{:a;N;/callbacks = tf.keras.callbacks/!ba;d}' \ + /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/keras_test.py && \ +find /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras -type f -exec sed -i \ + -e "s/ layers.Input/ tf_keras.layers.Input/g" \ + -e "s/layers.minimum/tf_keras.layers.minimum/g" \ + -e "s/layers.Concatenate/tf_keras.layers.Concatenate/g" \ + -e "s/layers.Dense/tf_keras.layers.Dense/g" \ + -e "s/layers.experimental.preprocessing./tf_keras.layers./g" \ + -e "s/layers.DenseFeatures/keras_internal.layers.DenseFeatures/g" \ + -e "s/models.Model/tf_keras.models.Model/g" {} \; && \ +sed -i "s/ models.load_model/ tf_keras.models.load_model/g" /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/keras_test.py && \ +sed -i "/import tensorflow as tf/a import tf_keras" /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/test_runner.py && \ +sed -i "/import tensorflow as tf/a import tf_keras" /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/wrappers.py && \ +sed -i -e "/import tensorflow as tf/a import tf_keras" \ + -e "s/optimizer=optimizers.Adam()/optimizer=tf_keras.optimizers.Adam()/g" \ + /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/wrappers_pre_generated.py && \ +find /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests -type f -exec sed -i "s/tf.keras./tf_keras./g" {} \; From bca30bdd51187eb73c256128300999e3b72905d7 Mon Sep 17 00:00:00 2001 From: Johnny Chavez <64660690+calderjo@users.noreply.github.com> Date: Thu, 8 Feb 2024 10:06:09 -0800 Subject: [PATCH 5/7] refactor sed commands --- Dockerfile.tmpl | 37 ++----------------------------------- 1 file changed, 2 insertions(+), 35 deletions(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 00000d6b..1bf51cdd 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -62,6 +62,7 @@ ENV KMP_SETTINGS=false ENV PIP_ROOT_USER_ACTION=ignore ADD clean-layer.sh /tmp/clean-layer.sh +ADD patches/keras_patch.sh /tmp/keras_patch.sh ADD patches/nbconvert-extensions.tpl /opt/kaggle/nbconvert-extensions.tpl ADD patches/template_conf.json /opt/kaggle/conf.json @@ -216,41 +217,7 @@ RUN pip install \ RUN pip install tensorflow_decision_forests --no-deps && \ /tmp/clean-layer.sh -RUN sed -i "/import tensorflow_decision_forests as tfdf/a import tf_keras" /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/__init__.py && \ - sed -i -e "/import tensorflow as tf/a import tf_keras" \ - -e "/from yggdrasil_decision_forests.utils.distribute.implementations.grpc/a from tensorflow_decision_forests.keras import keras_internal" \ - -e '/try:/{:a;N;/backend = tf.keras.backend/!ba;d}'\ - /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/core.py && \ - sed -i -e "s/from typing import Optional, List, Dict, Any, Union, NamedTuple/from typing import Any, Dict, List, NamedTuple, Optional, Union/g" \ - -e "/import tensorflow as tf/a from tensorflow_decision_forests.keras import keras_internal" \ - -e "/import tensorflow as tf/a import tf_keras" \ - -e '/layers = tf.keras.layers/{:a;N;/backend = tf.keras.backend/!ba;d}' \ - /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/core_inference.py && \ - find /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests -type f -exec sed -i \ - -e "s/get_data_handler/keras_internal.get_data_handler/g" \ - -e 's/"models.Functional"/keras_internal.Functional/g' \ - -e "s/tf.keras.utils.unpack_x_y_sample_weight/keras_internal.unpack_x_y_sample_weight/g" \ - -e "s/tf.keras.utils.experimental/keras_internal/g" \ - {} \; && \ - sed -i -e "/import tensorflow as tf/a import tf_keras" \ - -e "/from tensorflow_decision_forests.keras import core/a from tensorflow_decision_forests.keras import keras_internal" \ - -e '/layers = tf.keras.layers/{:a;N;/callbacks = tf.keras.callbacks/!ba;d}' \ - /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/keras_test.py && \ - find /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras -type f -exec sed -i \ - -e "s/ layers.Input/ tf_keras.layers.Input/g" \ - -e "s/layers.minimum/tf_keras.layers.minimum/g" \ - -e "s/layers.Concatenate/tf_keras.layers.Concatenate/g" \ - -e "s/layers.Dense/tf_keras.layers.Dense/g" \ - -e "s/layers.experimental.preprocessing./tf_keras.layers./g" \ - -e "s/layers.DenseFeatures/keras_internal.layers.DenseFeatures/g" \ - -e "s/models.Model/tf_keras.models.Model/g" {} \; && \ - sed -i "s/ models.load_model/ tf_keras.models.load_model/g" /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/keras_test.py && \ - sed -i "/import tensorflow as tf/a import tf_keras" /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/test_runner.py && \ - sed -i "/import tensorflow as tf/a import tf_keras" /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/wrappers.py && \ - sed -i -e "/import tensorflow as tf/a import tf_keras" \ - -e "s/optimizer=optimizers.Adam()/optimizer=tf_keras.optimizers.Adam()/g" \ - /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/wrappers_pre_generated.py && \ - find /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests -type f -exec sed -i "s/tf.keras./tf_keras./g" {} \; +RUN /tmp/keras_patch.sh ADD patches/keras_internal.py /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/keras_internal.py ADD patches/keras_internal_test.py /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/keras_internal_test.py From e1d8e9596e9775bf1d6fedf5edcd5f86bca71519 Mon Sep 17 00:00:00 2001 From: Johnny Chavez <64660690+calderjo@users.noreply.github.com> Date: Thu, 8 Feb 2024 10:06:45 -0800 Subject: [PATCH 6/7] Delete tests/test_tensorflowjs.py forgot to delete this test --- tests/test_tensorflowjs.py | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 tests/test_tensorflowjs.py diff --git a/tests/test_tensorflowjs.py b/tests/test_tensorflowjs.py deleted file mode 100644 index d0a5de6a..00000000 --- a/tests/test_tensorflowjs.py +++ /dev/null @@ -1,18 +0,0 @@ -import os -import subprocess -import unittest - -class TestTensorflowjs(unittest.TestCase): - def test_version(self): - result = subprocess.run([ - 'tensorflowjs_converter', - '--help', - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - print("tensorflowjs_converter help: ", result) - - self.assertEqual(0, result.returncode) - self.assertIn(b'tensorflowjs', result.stdout) From 8bcef0fd2b3cea247f13ccf9a9ffe4d446111689 Mon Sep 17 00:00:00 2001 From: Johnny Chavez <64660690+calderjo@users.noreply.github.com> Date: Thu, 8 Feb 2024 11:09:09 -0800 Subject: [PATCH 7/7] permission fix Dockerfile.tmpl --- Dockerfile.tmpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 1bf51cdd..0027fca1 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -217,7 +217,8 @@ RUN pip install \ RUN pip install tensorflow_decision_forests --no-deps && \ /tmp/clean-layer.sh -RUN /tmp/keras_patch.sh +RUN chmod +x /tmp/keras_patch.sh && \ + /tmp/keras_patch.sh ADD patches/keras_internal.py /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/keras_internal.py ADD patches/keras_internal_test.py /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/keras_internal_test.py