From 9d63670e58e0453a68e3e3e20eb6a26b69fec6bc Mon Sep 17 00:00:00 2001 From: Daksh Date: Fri, 7 Feb 2020 17:01:58 +0100 Subject: [PATCH 01/29] implemented first level of tensorflow configuration --- rasa/__main__.py | 4 ++ rasa/utils/tensorflow/__init__.py | 70 +++++++++++++++++++++++++++++++ rasa/utils/train_utils.py | 9 ---- 3 files changed, 74 insertions(+), 9 deletions(-) diff --git a/rasa/__main__.py b/rasa/__main__.py index 30fd6d1576cc..9c0017d4d383 100644 --- a/rasa/__main__.py +++ b/rasa/__main__.py @@ -8,6 +8,7 @@ from rasa.cli.arguments.default_arguments import add_logging_options from rasa.cli.utils import parse_last_positional_argument_as_model_path from rasa.utils.common import set_log_level +from rasa.utils.tensorflow import setup_tf_environment logger = logging.getLogger(__name__) @@ -68,6 +69,9 @@ def main() -> None: ) set_log_level(log_level) + # Set tensorflow environment + setup_tf_environment() + # insert current path in syspath so custom modules are found sys.path.insert(1, os.getcwd()) diff --git a/rasa/utils/tensorflow/__init__.py b/rasa/utils/tensorflow/__init__.py index e69de29bb2d1..5bb0a5ee9ca8 100644 --- a/rasa/utils/tensorflow/__init__.py +++ b/rasa/utils/tensorflow/__init__.py @@ -0,0 +1,70 @@ +import os +import tensorflow as tf +import logging + +logger = logging.getLogger(__name__) + + +def setup_gpu_environment(gpu_memory_config): + + if gpu_memory_config: + + # Parse GPU config + # gpu_config is of format "gpu_id_1:gpu_id_1_memory, gpu_id_2: gpu_id_2_memory" + # Parse it and store in a dictionary + parsed_gpu_config = { + instance.split(":")[0].strip(): int(instance.split(":")[1].strip()) + for instance in gpu_memory_config.split(",") + } + + physical_gpus = tf.config.list_physical_devices("GPU") + + # Logic taken from https://www.tensorflow.org/guide/gpu + if physical_gpus: + + for gpu_id, gpu_id_memory in parsed_gpu_config.items(): + try: + tf.config.experimental.set_virtual_device_configuration( + physical_gpus[int(gpu_id)], + [ + tf.config.experimental.VirtualDeviceConfiguration( + memory_limit=gpu_id_memory + ) + ], + ) + + except RuntimeError as e: + # Virtual devices must be set before GPUs have been initialized + raise RuntimeError( + "Error while setting up tensorflow environment. " + "Virtual devices must be set before GPUs have been initialized" + ) + + else: + logger.info( + "You have an environment variable GPU_MEMORY_ALLOC set but no GPUs were detected to configure" + ) + + +def setup_cpu_environment(inter_op_parallel_threads, intra_op_parallel_threads): + + if inter_op_parallel_threads: + tf.config.threading.set_inter_op_parallelism_threads( + int(inter_op_parallel_threads.strip()) + ) + + if intra_op_parallel_threads: + tf.config.threading.set_intra_op_parallelism_threads( + int(intra_op_parallel_threads.strip()) + ) + + +def setup_tf_environment(): + + # Get all env variables + gpu_memory_config = os.getenv("TF_GPU_MEMORY_ALLOC", None) + inter_op_parallel_threads = os.getenv("TF_INTER_OP_PARALLELISM_THREADS", None) + intra_op_parallel_threads = os.getenv("TF_INTRA_OP_PARALLELISM_THREADS", None) + + setup_gpu_environment(gpu_memory_config) + setup_cpu_environment(inter_op_parallel_threads, intra_op_parallel_threads) diff --git a/rasa/utils/train_utils.py b/rasa/utils/train_utils.py index 17ebcb124ab8..9477edc754dd 100644 --- a/rasa/utils/train_utils.py +++ b/rasa/utils/train_utils.py @@ -8,15 +8,6 @@ logger = logging.getLogger(__name__) -def load_tf_config(config: Dict[Text, Any]) -> Optional[tf.compat.v1.ConfigProto]: - """Prepare `tf.compat.v1.ConfigProto` for training""" - - if config.get("tf_config") is not None: - return tf.compat.v1.ConfigProto(**config.pop("tf_config")) - else: - return None - - def normalize(values: np.ndarray, ranking_length: Optional[int] = 0) -> np.ndarray: """Normalizes an array of positive numbers over the top `ranking_length` values. Other values will be set to 0. From 0f8b8a5a9f2d3e3f8bc7abff328b5341c3c158d0 Mon Sep 17 00:00:00 2001 From: Daksh Date: Wed, 12 Feb 2020 15:53:20 +0100 Subject: [PATCH 02/29] added test and refactored env variables --- rasa/__main__.py | 2 +- rasa/constants.py | 4 ++ rasa/utils/tensorflow/__init__.py | 67 ------------------------ rasa/utils/tensorflow/environment.py | 76 ++++++++++++++++++++++++++++ tests/utils/test_tf_environment.py | 16 ++++++ 5 files changed, 97 insertions(+), 68 deletions(-) create mode 100644 rasa/utils/tensorflow/environment.py create mode 100644 tests/utils/test_tf_environment.py diff --git a/rasa/__main__.py b/rasa/__main__.py index 9c0017d4d383..482807f1f261 100644 --- a/rasa/__main__.py +++ b/rasa/__main__.py @@ -8,7 +8,7 @@ from rasa.cli.arguments.default_arguments import add_logging_options from rasa.cli.utils import parse_last_positional_argument_as_model_path from rasa.utils.common import set_log_level -from rasa.utils.tensorflow import setup_tf_environment +from rasa.utils.tensorflow.environment import setup_tf_environment logger = logging.getLogger(__name__) diff --git a/rasa/constants.py b/rasa/constants.py index edfae6226e74..7cb9a9936507 100644 --- a/rasa/constants.py +++ b/rasa/constants.py @@ -59,3 +59,7 @@ DEFAULT_SESSION_EXPIRATION_TIME_IN_MINUTES = 60 DEFAULT_CARRY_OVER_SLOTS_TO_NEW_SESSION = True + +ENV_GPU_CONFIG = "TF_GPU_MEMORY_ALLOC" +ENV_CPU_INTER_OP_CONFIG = "TF_INTER_OP_PARALLELISM_THREADS" +ENV_CPU_INTRA_OP_CONFIG = "TF_INTRA_OP_PARALLELISM_THREADS" diff --git a/rasa/utils/tensorflow/__init__.py b/rasa/utils/tensorflow/__init__.py index 5bb0a5ee9ca8..b28b04f64312 100644 --- a/rasa/utils/tensorflow/__init__.py +++ b/rasa/utils/tensorflow/__init__.py @@ -1,70 +1,3 @@ -import os -import tensorflow as tf -import logging -logger = logging.getLogger(__name__) -def setup_gpu_environment(gpu_memory_config): - - if gpu_memory_config: - - # Parse GPU config - # gpu_config is of format "gpu_id_1:gpu_id_1_memory, gpu_id_2: gpu_id_2_memory" - # Parse it and store in a dictionary - parsed_gpu_config = { - instance.split(":")[0].strip(): int(instance.split(":")[1].strip()) - for instance in gpu_memory_config.split(",") - } - - physical_gpus = tf.config.list_physical_devices("GPU") - - # Logic taken from https://www.tensorflow.org/guide/gpu - if physical_gpus: - - for gpu_id, gpu_id_memory in parsed_gpu_config.items(): - try: - tf.config.experimental.set_virtual_device_configuration( - physical_gpus[int(gpu_id)], - [ - tf.config.experimental.VirtualDeviceConfiguration( - memory_limit=gpu_id_memory - ) - ], - ) - - except RuntimeError as e: - # Virtual devices must be set before GPUs have been initialized - raise RuntimeError( - "Error while setting up tensorflow environment. " - "Virtual devices must be set before GPUs have been initialized" - ) - - else: - logger.info( - "You have an environment variable GPU_MEMORY_ALLOC set but no GPUs were detected to configure" - ) - - -def setup_cpu_environment(inter_op_parallel_threads, intra_op_parallel_threads): - - if inter_op_parallel_threads: - tf.config.threading.set_inter_op_parallelism_threads( - int(inter_op_parallel_threads.strip()) - ) - - if intra_op_parallel_threads: - tf.config.threading.set_intra_op_parallelism_threads( - int(intra_op_parallel_threads.strip()) - ) - - -def setup_tf_environment(): - - # Get all env variables - gpu_memory_config = os.getenv("TF_GPU_MEMORY_ALLOC", None) - inter_op_parallel_threads = os.getenv("TF_INTER_OP_PARALLELISM_THREADS", None) - intra_op_parallel_threads = os.getenv("TF_INTRA_OP_PARALLELISM_THREADS", None) - - setup_gpu_environment(gpu_memory_config) - setup_cpu_environment(inter_op_parallel_threads, intra_op_parallel_threads) diff --git a/rasa/utils/tensorflow/environment.py b/rasa/utils/tensorflow/environment.py new file mode 100644 index 000000000000..acab6c726176 --- /dev/null +++ b/rasa/utils/tensorflow/environment.py @@ -0,0 +1,76 @@ +import logging +import os + +import tensorflow as tf +from rasa.constants import ( + ENV_GPU_CONFIG, + ENV_CPU_INTER_OP_CONFIG, + ENV_CPU_INTRA_OP_CONFIG, +) + +logger = logging.getLogger(__name__) + + +def setup_gpu_environment(gpu_memory_config): + + if gpu_memory_config: + + # Parse GPU config + # gpu_config is of format "gpu_id_1:gpu_id_1_memory, gpu_id_2: gpu_id_2_memory" + # Parse it and store in a dictionary + parsed_gpu_config = { + instance.split(":")[0].strip(): int(instance.split(":")[1].strip()) + for instance in gpu_memory_config.split(",") + } + + physical_gpus = tf.config.list_physical_devices("GPU") + + # Logic taken from https://www.tensorflow.org/guide/gpu + if physical_gpus: + + for gpu_id, gpu_id_memory in parsed_gpu_config.items(): + try: + tf.config.experimental.set_virtual_device_configuration( + physical_gpus[int(gpu_id)], + [ + tf.config.experimental.VirtualDeviceConfiguration( + memory_limit=gpu_id_memory + ) + ], + ) + + except RuntimeError as e: + # Virtual devices must be set before GPUs have been initialized + raise RuntimeError( + "Error while setting up tensorflow environment. " + "Virtual devices must be set before GPUs have been initialized" + ) + + else: + logger.info( + "You have an environment variable GPU_MEMORY_ALLOC set but no GPUs were detected to configure" + ) + + +def setup_cpu_environment(inter_op_parallel_threads, intra_op_parallel_threads): + + if inter_op_parallel_threads: + tf.config.threading.set_inter_op_parallelism_threads( + int(inter_op_parallel_threads.strip()) + ) + + if intra_op_parallel_threads: + tf.config.threading.set_intra_op_parallelism_threads( + int(intra_op_parallel_threads.strip()) + ) + + +def setup_tf_environment(): + + # Get all env variables + gpu_memory_config = os.getenv(ENV_GPU_CONFIG, None) + inter_op_parallel_threads = os.getenv(ENV_CPU_INTER_OP_CONFIG, None) + intra_op_parallel_threads = os.getenv(ENV_CPU_INTRA_OP_CONFIG, None) + + setup_gpu_environment(gpu_memory_config) + setup_cpu_environment(inter_op_parallel_threads, intra_op_parallel_threads) diff --git a/tests/utils/test_tf_environment.py b/tests/utils/test_tf_environment.py new file mode 100644 index 000000000000..acb58902a815 --- /dev/null +++ b/tests/utils/test_tf_environment.py @@ -0,0 +1,16 @@ +import tensorflow as tf +from rasa.utils.tensorflow.environment import ( + setup_cpu_environment, + setup_gpu_environment, +) + + +def test_tf_cpu_environment_setting(): + + inter_op_threads = "2" + intra_op_threads = "3" + + setup_cpu_environment(inter_op_threads, intra_op_threads) + + assert tf.config.threading.get_inter_op_parallelism_threads() == 2 + assert tf.config.threading.get_intra_op_parallelism_threads() == 3 From 3deda46aa51757780c069bccc079264a03fabd35 Mon Sep 17 00:00:00 2001 From: Daksh Date: Wed, 12 Feb 2020 16:04:16 +0100 Subject: [PATCH 03/29] remove extra lines --- rasa/utils/tensorflow/__init__.py | 3 --- tests/utils/test_tf_environment.py | 5 +---- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/rasa/utils/tensorflow/__init__.py b/rasa/utils/tensorflow/__init__.py index b28b04f64312..e69de29bb2d1 100644 --- a/rasa/utils/tensorflow/__init__.py +++ b/rasa/utils/tensorflow/__init__.py @@ -1,3 +0,0 @@ - - - diff --git a/tests/utils/test_tf_environment.py b/tests/utils/test_tf_environment.py index acb58902a815..14c0d2ba2e98 100644 --- a/tests/utils/test_tf_environment.py +++ b/tests/utils/test_tf_environment.py @@ -1,8 +1,5 @@ import tensorflow as tf -from rasa.utils.tensorflow.environment import ( - setup_cpu_environment, - setup_gpu_environment, -) +from rasa.utils.tensorflow.environment import setup_cpu_environment def test_tf_cpu_environment_setting(): From 75663a0baa0cce3184389d1c5ee0f413f2c69aad Mon Sep 17 00:00:00 2001 From: Daksh Date: Wed, 12 Feb 2020 16:10:12 +0100 Subject: [PATCH 04/29] added changelog --- changelog/5230.feature.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 changelog/5230.feature.rst diff --git a/changelog/5230.feature.rst b/changelog/5230.feature.rst new file mode 100644 index 000000000000..5d7599047663 --- /dev/null +++ b/changelog/5230.feature.rst @@ -0,0 +1,15 @@ +Refactor how GPU and CPU environments are configured for tensorflow 2.0 + +Environment variables to set and description is shown in the example below: + +.. code-block:: python + + # This specifies to use 1024 MB of memory from GPU with logical ID 0 and 2048 MB of memory from GPU with logical ID 1 + TF_GPU_MEMORY_ALLOC="0:1024, 1:2048" + + # Specifies that atmost 3 CPU threads can be used to parallelize multiple non-blocking operations + TF_INTER_OP_PARALLELISM_THREADS="3" + + # Specifies that atmost 2 CPU threads can be used to parallelize a particular operation. + TF_INTRA_OP_PARALLELISM_THREADS="2" + From 1ba52e09909c8a78ec239f9afb851222949be708 Mon Sep 17 00:00:00 2001 From: Daksh Date: Wed, 12 Feb 2020 16:12:39 +0100 Subject: [PATCH 05/29] removed unref variable --- rasa/utils/tensorflow/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/utils/tensorflow/environment.py b/rasa/utils/tensorflow/environment.py index acab6c726176..2afe9ba056c8 100644 --- a/rasa/utils/tensorflow/environment.py +++ b/rasa/utils/tensorflow/environment.py @@ -39,7 +39,7 @@ def setup_gpu_environment(gpu_memory_config): ], ) - except RuntimeError as e: + except RuntimeError: # Virtual devices must be set before GPUs have been initialized raise RuntimeError( "Error while setting up tensorflow environment. " From 9008960f510b99c3460f0ffb32da1456a9bdc454 Mon Sep 17 00:00:00 2001 From: Daksh Date: Wed, 12 Feb 2020 16:17:40 +0100 Subject: [PATCH 06/29] add types --- rasa/utils/tensorflow/environment.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/rasa/utils/tensorflow/environment.py b/rasa/utils/tensorflow/environment.py index 2afe9ba056c8..eb3401a23a23 100644 --- a/rasa/utils/tensorflow/environment.py +++ b/rasa/utils/tensorflow/environment.py @@ -1,5 +1,6 @@ import logging import os +from typing import Text import tensorflow as tf from rasa.constants import ( @@ -11,7 +12,7 @@ logger = logging.getLogger(__name__) -def setup_gpu_environment(gpu_memory_config): +def setup_gpu_environment(gpu_memory_config: Text) -> None: if gpu_memory_config: @@ -52,7 +53,9 @@ def setup_gpu_environment(gpu_memory_config): ) -def setup_cpu_environment(inter_op_parallel_threads, intra_op_parallel_threads): +def setup_cpu_environment( + inter_op_parallel_threads: Text, intra_op_parallel_threads: Text +) -> None: if inter_op_parallel_threads: tf.config.threading.set_inter_op_parallelism_threads( From 5ec8d3edff27e21c9529659310e7da4de7b9157f Mon Sep 17 00:00:00 2001 From: Daksh Date: Wed, 12 Feb 2020 16:51:43 +0100 Subject: [PATCH 07/29] remove old tests --- rasa/utils/tensorflow/environment.py | 2 +- tests/core/test_policies.py | 73 ---------------------------- 2 files changed, 1 insertion(+), 74 deletions(-) diff --git a/rasa/utils/tensorflow/environment.py b/rasa/utils/tensorflow/environment.py index eb3401a23a23..e015a5e3aec3 100644 --- a/rasa/utils/tensorflow/environment.py +++ b/rasa/utils/tensorflow/environment.py @@ -41,7 +41,7 @@ def setup_gpu_environment(gpu_memory_config: Text) -> None: ) except RuntimeError: - # Virtual devices must be set before GPUs have been initialized + # Add a helper explanation where the error comes from raise RuntimeError( "Error while setting up tensorflow environment. " "Virtual devices must be set before GPUs have been initialized" diff --git a/tests/core/test_policies.py b/tests/core/test_policies.py index d39a6e741a77..aaffba0c3669 100644 --- a/tests/core/test_policies.py +++ b/tests/core/test_policies.py @@ -52,35 +52,6 @@ from tests.core.utilities import get_tracker, read_dialogue_file, user_uttered -def tf_defaults(): - return { - "tf_config": { - "device_count": {"CPU": 4}, - # tell tf.Session to use CPU limit, if you have - # more CPU, you can increase this value appropriately - "inter_op_parallelism_threads": 0, - # the number of threads in the thread pool available - # for each process for blocking operation nodes set to 0 - # to allow the system to select the appropriate value. - "intra_op_parallelism_threads": 0, # tells the degree of thread - # parallelism of the tf.Session operation. - # the smaller the value, the less reuse the thread will have - # and the more likely it will use more CPU cores. - # if the value is 0, - # tensorflow will automatically select an appropriate value. - "gpu_options": {"allow_growth": True} - # if set True, will try to allocate - # as much GPU memory as possible to support running - } - } - - -def session_config(): - import tensorflow as tf - - return tf.ConfigProto(**tf_defaults()["tf_config"]) - - async def train_trackers(domain, augmentation_factor=20): return await training.load_data( DEFAULT_STORIES_FILE, domain, augmentation_factor=augmentation_factor @@ -185,18 +156,6 @@ def test_persist_and_load_empty_policy(self, tmpdir): loaded = empty_policy.__class__.load(tmpdir.strpath) assert loaded is not None - # TODO test tf config - # def test_tf_config(self, trained_policy, tmpdir): - # if hasattr(trained_policy, "session"): - # import tensorflow as tf - # - # # noinspection PyProtectedMember - # assert trained_policy.session._config == tf.Session()._config - # trained_policy.persist(tmpdir.strpath) - # loaded = trained_policy.__class__.load(tmpdir.strpath) - # # noinspection PyProtectedMember - # assert loaded.session._config == tf.Session()._config - @staticmethod def _get_next_action(policy, events, domain): tracker = get_tracker(events) @@ -212,22 +171,6 @@ def create_policy(self, featurizer, priority): return p -class TestKerasPolicyWithTfConfig(PolicyTestCollection): - def create_policy(self, featurizer, priority): - p = KerasPolicy(featurizer, priority, **tf_defaults()) - return p - - # TODO fix and test tf config - @pytest.mark.skip(reason="We need to fix tf.config!") - def test_tf_config(self, trained_policy, tmpdir): - # noinspection PyProtectedMember - assert trained_policy.session._config == session_config() - trained_policy.persist(tmpdir.strpath) - loaded = trained_policy.__class__.load(tmpdir.strpath) - # noinspection PyProtectedMember - assert loaded.session._config == session_config() - - class TestSklearnPolicy(PolicyTestCollection): def create_policy(self, featurizer, priority, **kwargs): p = SklearnPolicy(featurizer, priority, **kwargs) @@ -529,22 +472,6 @@ def create_policy(self, featurizer, priority): return p -class TestTEDPolicyWithTfConfig(TestTEDPolicy): - def create_policy(self, featurizer, priority): - p = TEDPolicy(featurizer=featurizer, priority=priority, **tf_defaults()) - return p - - # TODO test tf config - @pytest.mark.skip(reason="Fix tf config.") - def test_tf_config(self, trained_policy, tmpdir): - # noinspection PyProtectedMember - assert trained_policy.session._config == session_config() - trained_policy.persist(tmpdir.strpath) - loaded = trained_policy.__class__.load(tmpdir.strpath) - # noinspection PyProtectedMember - assert loaded.session._config == session_config() - - class TestMemoizationPolicy(PolicyTestCollection): def create_policy(self, featurizer, priority): max_history = None From 142327ad97336b457242f462ad2e438c153a97f9 Mon Sep 17 00:00:00 2001 From: Daksh Date: Wed, 12 Feb 2020 18:12:21 +0100 Subject: [PATCH 08/29] added docs --- docs/api/tensorflow_usage.rst | 58 +++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 59 insertions(+) create mode 100644 docs/api/tensorflow_usage.rst diff --git a/docs/api/tensorflow_usage.rst b/docs/api/tensorflow_usage.rst new file mode 100644 index 000000000000..8ecc0473a677 --- /dev/null +++ b/docs/api/tensorflow_usage.rst @@ -0,0 +1,58 @@ +:desc: Find out how to configure your environment for efficient usage of tensorflow inside Rasa + +.. _tensorflow_usage: + +Setting up Tensorflow Runtime +============================= + +Tensorflow allows setting the runtime environment via +`TF Config submodule `_. Rasa supports a smaller subset of these +configuration options and makes appropriate calls to the `tf.config` submodule. +This smaller subset comprises of configurations that developers frequently use with Rasa. +All configuration options are specified using environment variables as shown in subsequent sections. + + +Optimize CPU Performance +------------------------ + +Parallelize one operation +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Set ``TF_INTRA_OP_PARALLELISM_THREADS`` as an environment variable to specify maximum number of threads that can be used +to parallelize the execution of one operation. If left unspecified, this value defaults to 0 which means tensorflow should +pick an appropriate value depending on the system configuration. + + +Parallelize multiple operations +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Set ``TF_INTER_OP_PARALLELISM_THREADS`` as an environment variable to specify maximum number of threads that can be used +to parallelize the execution of multiple **non-blocking** operations. If left unspecified, this value defaults to 0 +which means tensorflow should pick an appropriate value depending on the system configuration. + + +Optimize GPU Performance +------------------------ + +Limiting GPU memory growth +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Tensorflow by default blocks all the available GPU memory for the running process. This can be limiting if you are running +multiple tensorflow processes and want to distribute memory across them. To prevent this, +set an environment variable ``TF_FORCE_GPU_ALLOW_GROWTH`` to ``True``. + + +Restricting absolute GPU memory available +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Often, a developer wants to limit the absolute amount of GPU memory that can be used by a process. + +For example, you may have two visible GPUs(GPU:0 and GPU:1) and you want to allocate 1024 MB from first GPU and 2048 MB from second GPU. +You can do so by setting an environment variable as ``TF_GPU_MEMORY_ALLOC="0:1024, 1:2048"``. + +Another scenario can be where you have access to 2 GPUs(GPU:0 and GPU:1) but you would like to use only second GPU for +Rasa process. +``TF_GPU_MEMORY_ALLOC="1:2048"`` would make 2048 MB of memory from GPU 1 available for the Rasa process + + + diff --git a/docs/index.rst b/docs/index.rst index 150317568b7a..1074a8dcf694 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -93,6 +93,7 @@ Understand messages, hold conversations, and connect to messaging channels and A api/lock-stores api/training-data-importers api/core-featurization + api/tensorflow_usage migration-guide changelog From ddae7aee2f98ff1d1453fa04b12637eaafcf9307 Mon Sep 17 00:00:00 2001 From: Daksh Date: Thu, 13 Feb 2020 15:26:49 +0100 Subject: [PATCH 09/29] refactored tests --- rasa/utils/tensorflow/environment.py | 50 +++++++++++++++++----------- tests/utils/test_tf_environment.py | 20 +++++++++-- 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/rasa/utils/tensorflow/environment.py b/rasa/utils/tensorflow/environment.py index e015a5e3aec3..e39a290dd0b5 100644 --- a/rasa/utils/tensorflow/environment.py +++ b/rasa/utils/tensorflow/environment.py @@ -12,17 +12,14 @@ logger = logging.getLogger(__name__) -def setup_gpu_environment(gpu_memory_config: Text) -> None: +def setup_gpu_environment() -> None: + + gpu_memory_config = os.getenv(ENV_GPU_CONFIG, None) if gpu_memory_config: # Parse GPU config - # gpu_config is of format "gpu_id_1:gpu_id_1_memory, gpu_id_2: gpu_id_2_memory" - # Parse it and store in a dictionary - parsed_gpu_config = { - instance.split(":")[0].strip(): int(instance.split(":")[1].strip()) - for instance in gpu_memory_config.split(",") - } + parsed_gpu_config = parse_gpu_config(gpu_memory_config) physical_gpus = tf.config.list_physical_devices("GPU") @@ -32,7 +29,7 @@ def setup_gpu_environment(gpu_memory_config: Text) -> None: for gpu_id, gpu_id_memory in parsed_gpu_config.items(): try: tf.config.experimental.set_virtual_device_configuration( - physical_gpus[int(gpu_id)], + physical_gpus[gpu_id], [ tf.config.experimental.VirtualDeviceConfiguration( memory_limit=gpu_id_memory @@ -49,13 +46,33 @@ def setup_gpu_environment(gpu_memory_config: Text) -> None: else: logger.info( - "You have an environment variable GPU_MEMORY_ALLOC set but no GPUs were detected to configure" + f"You have an environment variable '{ENV_GPU_CONFIG}' set but no GPUs were detected to configure" ) -def setup_cpu_environment( - inter_op_parallel_threads: Text, intra_op_parallel_threads: Text -) -> None: +def parse_gpu_config(gpu_memory_config: Text): + + # gpu_config is of format "gpu_id_1:gpu_id_1_memory, gpu_id_2: gpu_id_2_memory" + # Parse it and store in a dictionary + parsed_gpu_config = {} + + try: + for instance in gpu_memory_config.split(","): + instance_gpu_id, instance_gpu_mem = instance.split(":") + instance_gpu_id = int(instance_gpu_id) + instance_gpu_mem = int(instance_gpu_mem) + + parsed_gpu_config[instance_gpu_id] = instance_gpu_mem + except ValueError as e: + raise e + + return parsed_gpu_config + + +def setup_cpu_environment() -> None: + + inter_op_parallel_threads = os.getenv(ENV_CPU_INTER_OP_CONFIG, None) + intra_op_parallel_threads = os.getenv(ENV_CPU_INTRA_OP_CONFIG, None) if inter_op_parallel_threads: tf.config.threading.set_inter_op_parallelism_threads( @@ -70,10 +87,5 @@ def setup_cpu_environment( def setup_tf_environment(): - # Get all env variables - gpu_memory_config = os.getenv(ENV_GPU_CONFIG, None) - inter_op_parallel_threads = os.getenv(ENV_CPU_INTER_OP_CONFIG, None) - intra_op_parallel_threads = os.getenv(ENV_CPU_INTRA_OP_CONFIG, None) - - setup_gpu_environment(gpu_memory_config) - setup_cpu_environment(inter_op_parallel_threads, intra_op_parallel_threads) + setup_cpu_environment() + setup_gpu_environment() diff --git a/tests/utils/test_tf_environment.py b/tests/utils/test_tf_environment.py index 14c0d2ba2e98..ae2525f7b4bd 100644 --- a/tests/utils/test_tf_environment.py +++ b/tests/utils/test_tf_environment.py @@ -1,13 +1,27 @@ +import pytest import tensorflow as tf from rasa.utils.tensorflow.environment import setup_cpu_environment +from rasa.utils.tensorflow.environment import parse_gpu_config +from _pytest.monkeypatch import MonkeyPatch +from rasa.constants import ENV_CPU_INTER_OP_CONFIG, ENV_CPU_INTRA_OP_CONFIG def test_tf_cpu_environment_setting(): - inter_op_threads = "2" - intra_op_threads = "3" + monkeypatch = MonkeyPatch() + monkeypatch.setenv(ENV_CPU_INTRA_OP_CONFIG, "3") + monkeypatch.setenv(ENV_CPU_INTER_OP_CONFIG, "2") - setup_cpu_environment(inter_op_threads, intra_op_threads) + setup_cpu_environment() assert tf.config.threading.get_inter_op_parallelism_threads() == 2 assert tf.config.threading.get_intra_op_parallelism_threads() == 3 + + +@pytest.mark.parametrize( + "gpu_config_string, parsed_gpu_config", + [("0: 1024", {0: 1024}), ("0:1024, 1:2048", {0: 1024, 1: 2048})], +) +def test_parsed_gpu_config(gpu_config_string, parsed_gpu_config): + + assert parse_gpu_config(gpu_config_string) == parsed_gpu_config From c87593303e5778cfb1bea358e5e6c3e1ef167948 Mon Sep 17 00:00:00 2001 From: Daksh Date: Thu, 13 Feb 2020 16:09:48 +0100 Subject: [PATCH 10/29] WIP --- rasa/utils/tensorflow/environment.py | 17 +++++++++++++---- tests/utils/test_tf_environment.py | 16 +++++++++++----- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/rasa/utils/tensorflow/environment.py b/rasa/utils/tensorflow/environment.py index e39a290dd0b5..6ee35dea6bde 100644 --- a/rasa/utils/tensorflow/environment.py +++ b/rasa/utils/tensorflow/environment.py @@ -1,6 +1,6 @@ import logging import os -from typing import Text +from typing import Text, Tuple import tensorflow as tf from rasa.constants import ( @@ -63,13 +63,16 @@ def parse_gpu_config(gpu_memory_config: Text): instance_gpu_mem = int(instance_gpu_mem) parsed_gpu_config[instance_gpu_id] = instance_gpu_mem - except ValueError as e: - raise e + except ValueError: + # Add a helper explanation + raise ValueError( + f"Error parsing GPU configuration. Please cross-check the format of '{ENV_GPU_CONFIG}'" + ) return parsed_gpu_config -def setup_cpu_environment() -> None: +def setup_cpu_environment() -> Tuple[int, int]: inter_op_parallel_threads = os.getenv(ENV_CPU_INTER_OP_CONFIG, None) intra_op_parallel_threads = os.getenv(ENV_CPU_INTRA_OP_CONFIG, None) @@ -84,6 +87,12 @@ def setup_cpu_environment() -> None: int(intra_op_parallel_threads.strip()) ) + # Returning the actual values as a confirmation. Helps with tests too. + return ( + tf.config.threading.get_inter_op_parallelism_threads(), + tf.config.threading.get_intra_op_parallelism_threads(), + ) + def setup_tf_environment(): diff --git a/tests/utils/test_tf_environment.py b/tests/utils/test_tf_environment.py index ae2525f7b4bd..d23fee5abc32 100644 --- a/tests/utils/test_tf_environment.py +++ b/tests/utils/test_tf_environment.py @@ -4,16 +4,22 @@ from rasa.utils.tensorflow.environment import parse_gpu_config from _pytest.monkeypatch import MonkeyPatch from rasa.constants import ENV_CPU_INTER_OP_CONFIG, ENV_CPU_INTRA_OP_CONFIG +from typing import Text +import multiprocessing - -def test_tf_cpu_environment_setting(): +# @pytest.fixture() +def tf_cpu_environment_setter(inter_op_config: Text, intra_op_config: Text): monkeypatch = MonkeyPatch() - monkeypatch.setenv(ENV_CPU_INTRA_OP_CONFIG, "3") - monkeypatch.setenv(ENV_CPU_INTER_OP_CONFIG, "2") + monkeypatch.setenv(ENV_CPU_INTRA_OP_CONFIG, intra_op_config) + monkeypatch.setenv(ENV_CPU_INTER_OP_CONFIG, inter_op_config) + + return setup_cpu_environment() - setup_cpu_environment() +def test_tf_cpu_environment_setting(): + + child_process = multiprocessing.Process(target=tf_cpu_environment_setter, args=()) assert tf.config.threading.get_inter_op_parallelism_threads() == 2 assert tf.config.threading.get_intra_op_parallelism_threads() == 3 From 02950235f0540eb5b61768804c1729d2361e3a07 Mon Sep 17 00:00:00 2001 From: Daksh Date: Thu, 13 Feb 2020 16:16:38 +0100 Subject: [PATCH 11/29] check --- rasa/utils/tensorflow/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/utils/tensorflow/models.py b/rasa/utils/tensorflow/models.py index 77e67fddb739..d4b624e1e00f 100644 --- a/rasa/utils/tensorflow/models.py +++ b/rasa/utils/tensorflow/models.py @@ -218,7 +218,7 @@ def _get_tf_functions( logger.debug(f"Building tensorflow {phase} graph...") # allows increasing batch size - tf_dataset_function = tf.function(func=dataset_function) + tf_dataset_function = dataset_function # tf.function(func=dataset_function) init_dataset = tf_dataset_function(tf.ones((), tf.int32)) From 2843fe0388ad11e660a95f57092fa4023ba870e3 Mon Sep 17 00:00:00 2001 From: Daksh Date: Thu, 13 Feb 2020 16:40:59 +0100 Subject: [PATCH 12/29] fix test for cpu testing --- rasa/utils/tensorflow/models.py | 2 +- tests/utils/test_tf_environment.py | 36 ++++++++++++++++++++---------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/rasa/utils/tensorflow/models.py b/rasa/utils/tensorflow/models.py index d4b624e1e00f..77e67fddb739 100644 --- a/rasa/utils/tensorflow/models.py +++ b/rasa/utils/tensorflow/models.py @@ -218,7 +218,7 @@ def _get_tf_functions( logger.debug(f"Building tensorflow {phase} graph...") # allows increasing batch size - tf_dataset_function = dataset_function # tf.function(func=dataset_function) + tf_dataset_function = tf.function(func=dataset_function) init_dataset = tf_dataset_function(tf.ones((), tf.int32)) diff --git a/tests/utils/test_tf_environment.py b/tests/utils/test_tf_environment.py index d23fee5abc32..e93e5a979476 100644 --- a/tests/utils/test_tf_environment.py +++ b/tests/utils/test_tf_environment.py @@ -1,33 +1,45 @@ import pytest -import tensorflow as tf +from _pytest.monkeypatch import MonkeyPatch +from typing import Text, Dict +import multiprocessing from rasa.utils.tensorflow.environment import setup_cpu_environment from rasa.utils.tensorflow.environment import parse_gpu_config -from _pytest.monkeypatch import MonkeyPatch from rasa.constants import ENV_CPU_INTER_OP_CONFIG, ENV_CPU_INTRA_OP_CONFIG -from typing import Text -import multiprocessing -# @pytest.fixture() -def tf_cpu_environment_setter(inter_op_config: Text, intra_op_config: Text): + +def tf_cpu_setter( + inter_op_config: Text, intra_op_config: Text, shared_context_output: Dict[Text, int] +): monkeypatch = MonkeyPatch() monkeypatch.setenv(ENV_CPU_INTRA_OP_CONFIG, intra_op_config) monkeypatch.setenv(ENV_CPU_INTER_OP_CONFIG, inter_op_config) - return setup_cpu_environment() + set_inter_op_val, set_intra_op_val = setup_cpu_environment() + + shared_context_output[ENV_CPU_INTER_OP_CONFIG] = set_inter_op_val + shared_context_output[ENV_CPU_INTRA_OP_CONFIG] = set_intra_op_val + + +def test_tf_cpu_setting(): + manager = multiprocessing.Manager() + shared_context_output = manager.dict() -def test_tf_cpu_environment_setting(): + child_process = multiprocessing.get_context("spawn").Process( + target=tf_cpu_setter, args=("3", "2", shared_context_output) + ) + child_process.start() + child_process.join() - child_process = multiprocessing.Process(target=tf_cpu_environment_setter, args=()) - assert tf.config.threading.get_inter_op_parallelism_threads() == 2 - assert tf.config.threading.get_intra_op_parallelism_threads() == 3 + assert shared_context_output[ENV_CPU_INTER_OP_CONFIG] == 3 + assert shared_context_output[ENV_CPU_INTRA_OP_CONFIG] == 2 @pytest.mark.parametrize( "gpu_config_string, parsed_gpu_config", [("0: 1024", {0: 1024}), ("0:1024, 1:2048", {0: 1024, 1: 2048})], ) -def test_parsed_gpu_config(gpu_config_string, parsed_gpu_config): +def test_gpu_config_parser(gpu_config_string, parsed_gpu_config): assert parse_gpu_config(gpu_config_string) == parsed_gpu_config From ef527c001fdb948810f3d7c7937176132cb9c4c6 Mon Sep 17 00:00:00 2001 From: Daksh Date: Thu, 13 Feb 2020 17:38:49 +0100 Subject: [PATCH 13/29] address comments --- changelog/5230.feature.rst | 5 +-- docs/api/tensorflow_usage.rst | 51 +++++++++++++--------------- rasa/__main__.py | 5 ++- rasa/utils/tensorflow/environment.py | 50 ++++++++++++++++----------- 4 files changed, 60 insertions(+), 51 deletions(-) diff --git a/changelog/5230.feature.rst b/changelog/5230.feature.rst index 5d7599047663..c22e6f5f161a 100644 --- a/changelog/5230.feature.rst +++ b/changelog/5230.feature.rst @@ -1,6 +1,7 @@ -Refactor how GPU and CPU environments are configured for tensorflow 2.0 +Refactor how GPU and CPU environments are configured for TensorFlow 2.0 -Environment variables to set and description is shown in the example below: +Please refer to the `documentation `_ to understand +which environment variables to set in what scenarios. A couple of examples are shown below as well: .. code-block:: python diff --git a/docs/api/tensorflow_usage.rst b/docs/api/tensorflow_usage.rst index 8ecc0473a677..cb1dd5b9fba6 100644 --- a/docs/api/tensorflow_usage.rst +++ b/docs/api/tensorflow_usage.rst @@ -1,44 +1,43 @@ -:desc: Find out how to configure your environment for efficient usage of tensorflow inside Rasa +:desc: Find out how to configure your environment for efficient usage of TensorFlow inside Rasa OSS .. _tensorflow_usage: -Setting up Tensorflow Runtime -============================= +Setting up the TensorFlow Runtime +================================= -Tensorflow allows setting the runtime environment via -`TF Config submodule `_. Rasa supports a smaller subset of these -configuration options and makes appropriate calls to the `tf.config` submodule. -This smaller subset comprises of configurations that developers frequently use with Rasa. +TensorFlow allows setting the runtime environment via +`TF Config submodule `_. Rasa OSS supports a smaller subset of these +configuration options and makes appropriate calls to the ``tf.config`` submodule. +This smaller subset comprises of configurations that developers frequently use with Rasa OSS. All configuration options are specified using environment variables as shown in subsequent sections. +Optimizing CPU Performance +-------------------------- -Optimize CPU Performance ------------------------- - -Parallelize one operation -^^^^^^^^^^^^^^^^^^^^^^^^^ +Parallelizing one operation +^^^^^^^^^^^^^^^^^^^^^^^^^^^ Set ``TF_INTRA_OP_PARALLELISM_THREADS`` as an environment variable to specify maximum number of threads that can be used -to parallelize the execution of one operation. If left unspecified, this value defaults to 0 which means tensorflow should +to parallelize the execution of one operation. If left unspecified, this value defaults to ``0`` which means TensorFlow should pick an appropriate value depending on the system configuration. -Parallelize multiple operations -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Parallelizing multiple operations +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Set ``TF_INTER_OP_PARALLELISM_THREADS`` as an environment variable to specify maximum number of threads that can be used -to parallelize the execution of multiple **non-blocking** operations. If left unspecified, this value defaults to 0 -which means tensorflow should pick an appropriate value depending on the system configuration. +to parallelize the execution of multiple **non-blocking** operations. If left unspecified, this value defaults to ``0`` +which means TensorFlow should pick an appropriate value depending on the system configuration. -Optimize GPU Performance ------------------------- +Optimizing GPU Performance +-------------------------- Limiting GPU memory growth ^^^^^^^^^^^^^^^^^^^^^^^^^^ -Tensorflow by default blocks all the available GPU memory for the running process. This can be limiting if you are running -multiple tensorflow processes and want to distribute memory across them. To prevent this, +TensorFlow by default blocks all the available GPU memory for the running process. This can be limiting if you are running +multiple TensorFlow processes and want to distribute memory across them. To prevent this, set an environment variable ``TF_FORCE_GPU_ALLOW_GROWTH`` to ``True``. @@ -47,12 +46,10 @@ Restricting absolute GPU memory available Often, a developer wants to limit the absolute amount of GPU memory that can be used by a process. -For example, you may have two visible GPUs(GPU:0 and GPU:1) and you want to allocate 1024 MB from first GPU and 2048 MB from second GPU. +For example, you may have two visible GPUs(``GPU:0`` and ``GPU:1``) and you want to allocate 1024 MB from the first GPU +and 2048 MB from the second GPU. You can do so by setting an environment variable as ``TF_GPU_MEMORY_ALLOC="0:1024, 1:2048"``. -Another scenario can be where you have access to 2 GPUs(GPU:0 and GPU:1) but you would like to use only second GPU for -Rasa process. +Another scenario can be where you have access to 2 GPUs(``GPU:0`` and ``GPU:1``) but you would like to use only second +GPU for the Rasa OSS process. ``TF_GPU_MEMORY_ALLOC="1:2048"`` would make 2048 MB of memory from GPU 1 available for the Rasa process - - - diff --git a/rasa/__main__.py b/rasa/__main__.py index 482807f1f261..b3c17edae862 100644 --- a/rasa/__main__.py +++ b/rasa/__main__.py @@ -8,7 +8,7 @@ from rasa.cli.arguments.default_arguments import add_logging_options from rasa.cli.utils import parse_last_positional_argument_as_model_path from rasa.utils.common import set_log_level -from rasa.utils.tensorflow.environment import setup_tf_environment +import rasa.utils.tensorflow.environment as tf_env logger = logging.getLogger(__name__) @@ -69,8 +69,7 @@ def main() -> None: ) set_log_level(log_level) - # Set tensorflow environment - setup_tf_environment() + tf_env.setup_tf_environment() # insert current path in syspath so custom modules are found sys.path.insert(1, os.getcwd()) diff --git a/rasa/utils/tensorflow/environment.py b/rasa/utils/tensorflow/environment.py index 6ee35dea6bde..84ea0bbdab5b 100644 --- a/rasa/utils/tensorflow/environment.py +++ b/rasa/utils/tensorflow/environment.py @@ -1,37 +1,33 @@ import logging import os from typing import Text, Tuple - -import tensorflow as tf +import warnings from rasa.constants import ( ENV_GPU_CONFIG, ENV_CPU_INTER_OP_CONFIG, ENV_CPU_INTRA_OP_CONFIG, ) +from tensorflow import config as tf_config logger = logging.getLogger(__name__) def setup_gpu_environment() -> None: + """Set configuration for a GPU environment based on the environment variable set""" gpu_memory_config = os.getenv(ENV_GPU_CONFIG, None) - if gpu_memory_config: - - # Parse GPU config parsed_gpu_config = parse_gpu_config(gpu_memory_config) - - physical_gpus = tf.config.list_physical_devices("GPU") + physical_gpus = tf_config.list_physical_devices("GPU") # Logic taken from https://www.tensorflow.org/guide/gpu if physical_gpus: - for gpu_id, gpu_id_memory in parsed_gpu_config.items(): try: - tf.config.experimental.set_virtual_device_configuration( + tf_config.experimental.set_virtual_device_configuration( physical_gpus[gpu_id], [ - tf.config.experimental.VirtualDeviceConfiguration( + tf_config.experimental.VirtualDeviceConfiguration( memory_limit=gpu_id_memory ) ], @@ -45,12 +41,13 @@ def setup_gpu_environment() -> None: ) else: - logger.info( + warnings.warn( f"You have an environment variable '{ENV_GPU_CONFIG}' set but no GPUs were detected to configure" ) def parse_gpu_config(gpu_memory_config: Text): + """Parse GPU configuration variable from a string to a dict""" # gpu_config is of format "gpu_id_1:gpu_id_1_memory, gpu_id_2: gpu_id_2_memory" # Parse it and store in a dictionary @@ -73,24 +70,39 @@ def parse_gpu_config(gpu_memory_config: Text): def setup_cpu_environment() -> Tuple[int, int]: + """Set configuration for the CPU environment based on the environment variable set""" inter_op_parallel_threads = os.getenv(ENV_CPU_INTER_OP_CONFIG, None) intra_op_parallel_threads = os.getenv(ENV_CPU_INTRA_OP_CONFIG, None) if inter_op_parallel_threads: - tf.config.threading.set_inter_op_parallelism_threads( - int(inter_op_parallel_threads.strip()) - ) + + try: + inter_op_parallel_threads = int(inter_op_parallel_threads.strip()) + except ValueError: + raise ValueError( + f"Error parsing the environment variable '{ENV_CPU_INTER_OP_CONFIG}'. Please " + f"cross-check the value" + ) + + tf_config.threading.set_inter_op_parallelism_threads(inter_op_parallel_threads) if intra_op_parallel_threads: - tf.config.threading.set_intra_op_parallelism_threads( - int(intra_op_parallel_threads.strip()) - ) + + try: + intra_op_parallel_threads = int(intra_op_parallel_threads.strip()) + except ValueError: + raise ValueError( + f"Error parsing the environment variable '{ENV_CPU_INTRA_OP_CONFIG}'. Please " + f"cross-check the value" + ) + + tf_config.threading.set_intra_op_parallelism_threads(intra_op_parallel_threads) # Returning the actual values as a confirmation. Helps with tests too. return ( - tf.config.threading.get_inter_op_parallelism_threads(), - tf.config.threading.get_intra_op_parallelism_threads(), + tf_config.threading.get_inter_op_parallelism_threads(), + tf_config.threading.get_intra_op_parallelism_threads(), ) From 879bd0db250213a443382962a12e9b77d295332d Mon Sep 17 00:00:00 2001 From: Daksh Date: Thu, 13 Feb 2020 17:42:34 +0100 Subject: [PATCH 14/29] fix type annotations --- rasa/utils/tensorflow/environment.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rasa/utils/tensorflow/environment.py b/rasa/utils/tensorflow/environment.py index 84ea0bbdab5b..46745f25d61a 100644 --- a/rasa/utils/tensorflow/environment.py +++ b/rasa/utils/tensorflow/environment.py @@ -1,6 +1,6 @@ import logging import os -from typing import Text, Tuple +from typing import Text, Tuple, Dict import warnings from rasa.constants import ( ENV_GPU_CONFIG, @@ -15,7 +15,7 @@ def setup_gpu_environment() -> None: """Set configuration for a GPU environment based on the environment variable set""" - gpu_memory_config = os.getenv(ENV_GPU_CONFIG, None) + gpu_memory_config = os.getenv(ENV_GPU_CONFIG) if gpu_memory_config: parsed_gpu_config = parse_gpu_config(gpu_memory_config) physical_gpus = tf_config.list_physical_devices("GPU") @@ -46,7 +46,7 @@ def setup_gpu_environment() -> None: ) -def parse_gpu_config(gpu_memory_config: Text): +def parse_gpu_config(gpu_memory_config: Text) -> Dict[int, int]: """Parse GPU configuration variable from a string to a dict""" # gpu_config is of format "gpu_id_1:gpu_id_1_memory, gpu_id_2: gpu_id_2_memory" @@ -72,8 +72,8 @@ def parse_gpu_config(gpu_memory_config: Text): def setup_cpu_environment() -> Tuple[int, int]: """Set configuration for the CPU environment based on the environment variable set""" - inter_op_parallel_threads = os.getenv(ENV_CPU_INTER_OP_CONFIG, None) - intra_op_parallel_threads = os.getenv(ENV_CPU_INTRA_OP_CONFIG, None) + inter_op_parallel_threads = os.getenv(ENV_CPU_INTER_OP_CONFIG) + intra_op_parallel_threads = os.getenv(ENV_CPU_INTRA_OP_CONFIG) if inter_op_parallel_threads: @@ -106,7 +106,7 @@ def setup_cpu_environment() -> Tuple[int, int]: ) -def setup_tf_environment(): +def setup_tf_environment() -> None: setup_cpu_environment() setup_gpu_environment() From 0fec427357a174daa1360251536442d37f166f3d Mon Sep 17 00:00:00 2001 From: Daksh Date: Thu, 13 Feb 2020 17:54:50 +0100 Subject: [PATCH 15/29] fix imports --- rasa/utils/tensorflow/environment.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rasa/utils/tensorflow/environment.py b/rasa/utils/tensorflow/environment.py index 46745f25d61a..fe200ce64dd1 100644 --- a/rasa/utils/tensorflow/environment.py +++ b/rasa/utils/tensorflow/environment.py @@ -7,7 +7,6 @@ ENV_CPU_INTER_OP_CONFIG, ENV_CPU_INTRA_OP_CONFIG, ) -from tensorflow import config as tf_config logger = logging.getLogger(__name__) @@ -15,6 +14,8 @@ def setup_gpu_environment() -> None: """Set configuration for a GPU environment based on the environment variable set""" + from tensorflow import config as tf_config + gpu_memory_config = os.getenv(ENV_GPU_CONFIG) if gpu_memory_config: parsed_gpu_config = parse_gpu_config(gpu_memory_config) @@ -72,6 +73,8 @@ def parse_gpu_config(gpu_memory_config: Text) -> Dict[int, int]: def setup_cpu_environment() -> Tuple[int, int]: """Set configuration for the CPU environment based on the environment variable set""" + from tensorflow import config as tf_config + inter_op_parallel_threads = os.getenv(ENV_CPU_INTER_OP_CONFIG) intra_op_parallel_threads = os.getenv(ENV_CPU_INTRA_OP_CONFIG) From 5a88cd50201f080c1cc617d0546c0be0b0968b71 Mon Sep 17 00:00:00 2001 From: Daksh Date: Fri, 14 Feb 2020 12:06:47 +0100 Subject: [PATCH 16/29] test just one pytest file on travis --- Makefile | 2 +- tests/utils/test_tf_environment.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6f2aefe7128e..92997ee369ff 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,7 @@ prepare-tests-files: test: clean # OMP_NUM_THREADS can improve overral performance using one thread by process (on tensorflow), avoiding overload - OMP_NUM_THREADS=1 pytest tests -n $(JOBS) --cov rasa + OMP_NUM_THREADS=1 pytest tests/utils/test_tf_environment.py -n $(JOBS) --cov rasa doctest: clean cd docs && make doctest diff --git a/tests/utils/test_tf_environment.py b/tests/utils/test_tf_environment.py index e93e5a979476..f4dc24961662 100644 --- a/tests/utils/test_tf_environment.py +++ b/tests/utils/test_tf_environment.py @@ -23,7 +23,7 @@ def tf_cpu_setter( def test_tf_cpu_setting(): - manager = multiprocessing.Manager() + manager = multiprocessing.get_context("spawn").Manager() shared_context_output = manager.dict() child_process = multiprocessing.get_context("spawn").Process( From 6f2821834a54edfbaddeb10766e1213aac40363c Mon Sep 17 00:00:00 2001 From: Daksh Date: Fri, 14 Feb 2020 13:13:50 +0100 Subject: [PATCH 17/29] added all tests to pytest now --- Makefile | 2 +- docs/api/tensorflow_usage.rst | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 92997ee369ff..6f2aefe7128e 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,7 @@ prepare-tests-files: test: clean # OMP_NUM_THREADS can improve overral performance using one thread by process (on tensorflow), avoiding overload - OMP_NUM_THREADS=1 pytest tests/utils/test_tf_environment.py -n $(JOBS) --cov rasa + OMP_NUM_THREADS=1 pytest tests -n $(JOBS) --cov rasa doctest: clean cd docs && make doctest diff --git a/docs/api/tensorflow_usage.rst b/docs/api/tensorflow_usage.rst index cb1dd5b9fba6..c2eddcfcf0a3 100644 --- a/docs/api/tensorflow_usage.rst +++ b/docs/api/tensorflow_usage.rst @@ -1,4 +1,4 @@ -:desc: Find out how to configure your environment for efficient usage of TensorFlow inside Rasa OSS +:desc: Find out how to configure your environment for efficient usage of TensorFlow inside Rasa Open Source .. _tensorflow_usage: @@ -6,9 +6,9 @@ Setting up the TensorFlow Runtime ================================= TensorFlow allows setting the runtime environment via -`TF Config submodule `_. Rasa OSS supports a smaller subset of these +`TF Config submodule `_. Rasa Open Source supports a smaller subset of these configuration options and makes appropriate calls to the ``tf.config`` submodule. -This smaller subset comprises of configurations that developers frequently use with Rasa OSS. +This smaller subset comprises of configurations that developers frequently use with Rasa Open Source. All configuration options are specified using environment variables as shown in subsequent sections. Optimizing CPU Performance @@ -50,6 +50,6 @@ For example, you may have two visible GPUs(``GPU:0`` and ``GPU:1``) and you want and 2048 MB from the second GPU. You can do so by setting an environment variable as ``TF_GPU_MEMORY_ALLOC="0:1024, 1:2048"``. -Another scenario can be where you have access to 2 GPUs(``GPU:0`` and ``GPU:1``) but you would like to use only second -GPU for the Rasa OSS process. -``TF_GPU_MEMORY_ALLOC="1:2048"`` would make 2048 MB of memory from GPU 1 available for the Rasa process +Another scenario can be where you have access to 2 GPUs(``GPU:0`` and ``GPU:1``) but you would like to use only the second +GPU. +``TF_GPU_MEMORY_ALLOC="1:2048"`` would make 2048 MB of memory availble from GPU 1. From af6f03723504e57ff057c96161fdf84d0e54a9f5 Mon Sep 17 00:00:00 2001 From: Daksh Date: Fri, 14 Feb 2020 15:26:17 +0100 Subject: [PATCH 18/29] trying to fix the test. Multiprocessing :( --- tests/utils/test_tf_environment.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/utils/test_tf_environment.py b/tests/utils/test_tf_environment.py index f4dc24961662..c0b8d88ab6ed 100644 --- a/tests/utils/test_tf_environment.py +++ b/tests/utils/test_tf_environment.py @@ -23,10 +23,9 @@ def tf_cpu_setter( def test_tf_cpu_setting(): - manager = multiprocessing.get_context("spawn").Manager() - shared_context_output = manager.dict() + shared_context_output = multiprocessing.Manager().dict() - child_process = multiprocessing.get_context("spawn").Process( + child_process = multiprocessing.Process( target=tf_cpu_setter, args=("3", "2", shared_context_output) ) child_process.start() From 98c43b25e586ed8193b898e700a195fb91942cc3 Mon Sep 17 00:00:00 2001 From: Daksh Date: Mon, 17 Feb 2020 11:39:45 +0100 Subject: [PATCH 19/29] remove cpu env set test --- rasa/utils/tensorflow/environment.py | 50 +++++++++++++++++----------- tests/utils/test_tf_environment.py | 28 ---------------- 2 files changed, 30 insertions(+), 48 deletions(-) diff --git a/rasa/utils/tensorflow/environment.py b/rasa/utils/tensorflow/environment.py index fe200ce64dd1..a8366b2f0b0a 100644 --- a/rasa/utils/tensorflow/environment.py +++ b/rasa/utils/tensorflow/environment.py @@ -1,7 +1,7 @@ import logging import os from typing import Text, Tuple, Dict -import warnings +import rasa.utils.common as rasa_utils from rasa.constants import ( ENV_GPU_CONFIG, ENV_CPU_INTER_OP_CONFIG, @@ -14,39 +14,49 @@ def setup_gpu_environment() -> None: """Set configuration for a GPU environment based on the environment variable set""" - from tensorflow import config as tf_config - gpu_memory_config = os.getenv(ENV_GPU_CONFIG) if gpu_memory_config: + + # Import from tensorflow only if necessary(environment variable was set) + from tensorflow import config as tf_config + parsed_gpu_config = parse_gpu_config(gpu_memory_config) physical_gpus = tf_config.list_physical_devices("GPU") # Logic taken from https://www.tensorflow.org/guide/gpu if physical_gpus: for gpu_id, gpu_id_memory in parsed_gpu_config.items(): - try: - tf_config.experimental.set_virtual_device_configuration( - physical_gpus[gpu_id], - [ - tf_config.experimental.VirtualDeviceConfiguration( - memory_limit=gpu_id_memory - ) - ], - ) - - except RuntimeError: - # Add a helper explanation where the error comes from - raise RuntimeError( - "Error while setting up tensorflow environment. " - "Virtual devices must be set before GPUs have been initialized" - ) + + allocate_gpu_memory(physical_gpus[gpu_id], gpu_id_memory) else: - warnings.warn( + rasa_utils.raise_warning( f"You have an environment variable '{ENV_GPU_CONFIG}' set but no GPUs were detected to configure" ) +def allocate_gpu_memory(gpu_instance, logical_memory: int) -> None: + + from tensorflow import config as tf_config + + try: + tf_config.experimental.set_virtual_device_configuration( + gpu_instance, + [ + tf_config.experimental.VirtualDeviceConfiguration( + memory_limit=logical_memory + ) + ], + ) + + except RuntimeError: + # Add a helper explanation where the error comes from + raise RuntimeError( + "Error while setting up tensorflow environment. " + "Virtual devices must be set before GPUs have been initialized" + ) + + def parse_gpu_config(gpu_memory_config: Text) -> Dict[int, int]: """Parse GPU configuration variable from a string to a dict""" diff --git a/tests/utils/test_tf_environment.py b/tests/utils/test_tf_environment.py index c0b8d88ab6ed..2f8024ae6f52 100644 --- a/tests/utils/test_tf_environment.py +++ b/tests/utils/test_tf_environment.py @@ -7,34 +7,6 @@ from rasa.constants import ENV_CPU_INTER_OP_CONFIG, ENV_CPU_INTRA_OP_CONFIG -def tf_cpu_setter( - inter_op_config: Text, intra_op_config: Text, shared_context_output: Dict[Text, int] -): - - monkeypatch = MonkeyPatch() - monkeypatch.setenv(ENV_CPU_INTRA_OP_CONFIG, intra_op_config) - monkeypatch.setenv(ENV_CPU_INTER_OP_CONFIG, inter_op_config) - - set_inter_op_val, set_intra_op_val = setup_cpu_environment() - - shared_context_output[ENV_CPU_INTER_OP_CONFIG] = set_inter_op_val - shared_context_output[ENV_CPU_INTRA_OP_CONFIG] = set_intra_op_val - - -def test_tf_cpu_setting(): - - shared_context_output = multiprocessing.Manager().dict() - - child_process = multiprocessing.Process( - target=tf_cpu_setter, args=("3", "2", shared_context_output) - ) - child_process.start() - child_process.join() - - assert shared_context_output[ENV_CPU_INTER_OP_CONFIG] == 3 - assert shared_context_output[ENV_CPU_INTRA_OP_CONFIG] == 2 - - @pytest.mark.parametrize( "gpu_config_string, parsed_gpu_config", [("0: 1024", {0: 1024}), ("0:1024, 1:2048", {0: 1024, 1: 2048})], From d970ae9844b60caec19856d125d76bba65a106e8 Mon Sep 17 00:00:00 2001 From: Daksh Date: Mon, 17 Feb 2020 11:40:23 +0100 Subject: [PATCH 20/29] remove unused imports --- tests/utils/test_tf_environment.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/utils/test_tf_environment.py b/tests/utils/test_tf_environment.py index 2f8024ae6f52..f44eed243f7c 100644 --- a/tests/utils/test_tf_environment.py +++ b/tests/utils/test_tf_environment.py @@ -1,10 +1,5 @@ import pytest -from _pytest.monkeypatch import MonkeyPatch -from typing import Text, Dict -import multiprocessing -from rasa.utils.tensorflow.environment import setup_cpu_environment from rasa.utils.tensorflow.environment import parse_gpu_config -from rasa.constants import ENV_CPU_INTER_OP_CONFIG, ENV_CPU_INTRA_OP_CONFIG @pytest.mark.parametrize( From a96159b0d6be6d0098f068ea852f63eec373b090 Mon Sep 17 00:00:00 2001 From: Daksh Date: Mon, 17 Feb 2020 11:55:59 +0100 Subject: [PATCH 21/29] refactor function --- rasa/utils/tensorflow/environment.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/rasa/utils/tensorflow/environment.py b/rasa/utils/tensorflow/environment.py index a8366b2f0b0a..52016da934fd 100644 --- a/rasa/utils/tensorflow/environment.py +++ b/rasa/utils/tensorflow/environment.py @@ -1,6 +1,11 @@ import logging import os -from typing import Text, Tuple, Dict +from typing import Text, Dict +import typing + +if typing.TYPE_CHECKING: + from tensorflow import config as tf_config + import rasa.utils.common as rasa_utils from rasa.constants import ( ENV_GPU_CONFIG, @@ -35,7 +40,9 @@ def setup_gpu_environment() -> None: ) -def allocate_gpu_memory(gpu_instance, logical_memory: int) -> None: +def allocate_gpu_memory( + gpu_instance: "tf_config.PhysicalDevice", logical_memory: int +) -> None: from tensorflow import config as tf_config @@ -80,14 +87,17 @@ def parse_gpu_config(gpu_memory_config: Text) -> Dict[int, int]: return parsed_gpu_config -def setup_cpu_environment() -> Tuple[int, int]: +def setup_cpu_environment() -> None: """Set configuration for the CPU environment based on the environment variable set""" - from tensorflow import config as tf_config - inter_op_parallel_threads = os.getenv(ENV_CPU_INTER_OP_CONFIG) intra_op_parallel_threads = os.getenv(ENV_CPU_INTRA_OP_CONFIG) + if not inter_op_parallel_threads and not intra_op_parallel_threads: + return + + from tensorflow import config as tf_config + if inter_op_parallel_threads: try: @@ -112,12 +122,6 @@ def setup_cpu_environment() -> Tuple[int, int]: tf_config.threading.set_intra_op_parallelism_threads(intra_op_parallel_threads) - # Returning the actual values as a confirmation. Helps with tests too. - return ( - tf_config.threading.get_inter_op_parallelism_threads(), - tf_config.threading.get_intra_op_parallelism_threads(), - ) - def setup_tf_environment() -> None: From 6146bf96589537a294b996a92d496ff6ddc59b61 Mon Sep 17 00:00:00 2001 From: Daksh Varshneya Date: Mon, 17 Feb 2020 13:44:38 +0100 Subject: [PATCH 22/29] Apply suggestions from code review Co-Authored-By: Tobias Wochinger --- changelog/5230.feature.rst | 5 ++--- docs/api/tensorflow_usage.rst | 10 +++++----- rasa/utils/tensorflow/environment.py | 10 +++------- tests/utils/test_tf_environment.py | 1 - 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/changelog/5230.feature.rst b/changelog/5230.feature.rst index c22e6f5f161a..5aeae8295c2e 100644 --- a/changelog/5230.feature.rst +++ b/changelog/5230.feature.rst @@ -8,9 +8,8 @@ which environment variables to set in what scenarios. A couple of examples are s # This specifies to use 1024 MB of memory from GPU with logical ID 0 and 2048 MB of memory from GPU with logical ID 1 TF_GPU_MEMORY_ALLOC="0:1024, 1:2048" - # Specifies that atmost 3 CPU threads can be used to parallelize multiple non-blocking operations + # Specifies that at most 3 CPU threads can be used to parallelize multiple non-blocking operations TF_INTER_OP_PARALLELISM_THREADS="3" - # Specifies that atmost 2 CPU threads can be used to parallelize a particular operation. + # Specifies that at most 2 CPU threads can be used to parallelize a particular operation. TF_INTRA_OP_PARALLELISM_THREADS="2" - diff --git a/docs/api/tensorflow_usage.rst b/docs/api/tensorflow_usage.rst index c2eddcfcf0a3..40d9076067c8 100644 --- a/docs/api/tensorflow_usage.rst +++ b/docs/api/tensorflow_usage.rst @@ -14,7 +14,7 @@ All configuration options are specified using environment variables as shown in Optimizing CPU Performance -------------------------- -Parallelizing one operation +Parallelizing One Operation ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Set ``TF_INTRA_OP_PARALLELISM_THREADS`` as an environment variable to specify maximum number of threads that can be used @@ -22,7 +22,7 @@ to parallelize the execution of one operation. If left unspecified, this value d pick an appropriate value depending on the system configuration. -Parallelizing multiple operations +Parallelizing Multiple Operations ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Set ``TF_INTER_OP_PARALLELISM_THREADS`` as an environment variable to specify maximum number of threads that can be used @@ -33,7 +33,7 @@ which means TensorFlow should pick an appropriate value depending on the system Optimizing GPU Performance -------------------------- -Limiting GPU memory growth +Limiting GPU Memory Growth ^^^^^^^^^^^^^^^^^^^^^^^^^^ TensorFlow by default blocks all the available GPU memory for the running process. This can be limiting if you are running @@ -41,7 +41,7 @@ multiple TensorFlow processes and want to distribute memory across them. To prev set an environment variable ``TF_FORCE_GPU_ALLOW_GROWTH`` to ``True``. -Restricting absolute GPU memory available +Restricting Absolute GPU Memory Available ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Often, a developer wants to limit the absolute amount of GPU memory that can be used by a process. @@ -52,4 +52,4 @@ You can do so by setting an environment variable as ``TF_GPU_MEMORY_ALLOC="0:102 Another scenario can be where you have access to 2 GPUs(``GPU:0`` and ``GPU:1``) but you would like to use only the second GPU. -``TF_GPU_MEMORY_ALLOC="1:2048"`` would make 2048 MB of memory availble from GPU 1. +``TF_GPU_MEMORY_ALLOC="1:2048"`` would make 2048 MB of memory available from GPU 1. diff --git a/rasa/utils/tensorflow/environment.py b/rasa/utils/tensorflow/environment.py index 52016da934fd..b49981f43dc4 100644 --- a/rasa/utils/tensorflow/environment.py +++ b/rasa/utils/tensorflow/environment.py @@ -17,7 +17,7 @@ def setup_gpu_environment() -> None: - """Set configuration for a GPU environment based on the environment variable set""" + """Set configuration for TensorFlow GPU environment based on the environment variable set.""" gpu_memory_config = os.getenv(ENV_GPU_CONFIG) if gpu_memory_config: @@ -31,7 +31,6 @@ def setup_gpu_environment() -> None: # Logic taken from https://www.tensorflow.org/guide/gpu if physical_gpus: for gpu_id, gpu_id_memory in parsed_gpu_config.items(): - allocate_gpu_memory(physical_gpus[gpu_id], gpu_id_memory) else: @@ -81,7 +80,7 @@ def parse_gpu_config(gpu_memory_config: Text) -> Dict[int, int]: except ValueError: # Add a helper explanation raise ValueError( - f"Error parsing GPU configuration. Please cross-check the format of '{ENV_GPU_CONFIG}'" + f"Error parsing GPU configuration. Please cross-check the format of '{ENV_GPU_CONFIG}'." ) return parsed_gpu_config @@ -99,19 +98,17 @@ def setup_cpu_environment() -> None: from tensorflow import config as tf_config if inter_op_parallel_threads: - try: inter_op_parallel_threads = int(inter_op_parallel_threads.strip()) except ValueError: raise ValueError( f"Error parsing the environment variable '{ENV_CPU_INTER_OP_CONFIG}'. Please " - f"cross-check the value" + f"cross-check the value." ) tf_config.threading.set_inter_op_parallelism_threads(inter_op_parallel_threads) if intra_op_parallel_threads: - try: intra_op_parallel_threads = int(intra_op_parallel_threads.strip()) except ValueError: @@ -124,6 +121,5 @@ def setup_cpu_environment() -> None: def setup_tf_environment() -> None: - setup_cpu_environment() setup_gpu_environment() diff --git a/tests/utils/test_tf_environment.py b/tests/utils/test_tf_environment.py index f44eed243f7c..7366e66ac690 100644 --- a/tests/utils/test_tf_environment.py +++ b/tests/utils/test_tf_environment.py @@ -7,5 +7,4 @@ [("0: 1024", {0: 1024}), ("0:1024, 1:2048", {0: 1024, 1: 2048})], ) def test_gpu_config_parser(gpu_config_string, parsed_gpu_config): - assert parse_gpu_config(gpu_config_string) == parsed_gpu_config From 99a3fe2d8df63a30d8d3dbef8976764110c7a348 Mon Sep 17 00:00:00 2001 From: Daksh Varshneya Date: Mon, 17 Feb 2020 13:46:09 +0100 Subject: [PATCH 23/29] Apply suggestions from code review Co-Authored-By: Tobias Wochinger --- docs/api/tensorflow_usage.rst | 3 --- rasa/utils/tensorflow/environment.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/api/tensorflow_usage.rst b/docs/api/tensorflow_usage.rst index 40d9076067c8..ef6786b294b3 100644 --- a/docs/api/tensorflow_usage.rst +++ b/docs/api/tensorflow_usage.rst @@ -21,7 +21,6 @@ Set ``TF_INTRA_OP_PARALLELISM_THREADS`` as an environment variable to specify ma to parallelize the execution of one operation. If left unspecified, this value defaults to ``0`` which means TensorFlow should pick an appropriate value depending on the system configuration. - Parallelizing Multiple Operations ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -29,7 +28,6 @@ Set ``TF_INTER_OP_PARALLELISM_THREADS`` as an environment variable to specify ma to parallelize the execution of multiple **non-blocking** operations. If left unspecified, this value defaults to ``0`` which means TensorFlow should pick an appropriate value depending on the system configuration. - Optimizing GPU Performance -------------------------- @@ -40,7 +38,6 @@ TensorFlow by default blocks all the available GPU memory for the running proces multiple TensorFlow processes and want to distribute memory across them. To prevent this, set an environment variable ``TF_FORCE_GPU_ALLOW_GROWTH`` to ``True``. - Restricting Absolute GPU Memory Available ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/rasa/utils/tensorflow/environment.py b/rasa/utils/tensorflow/environment.py index b49981f43dc4..3de779030b65 100644 --- a/rasa/utils/tensorflow/environment.py +++ b/rasa/utils/tensorflow/environment.py @@ -114,7 +114,7 @@ def setup_cpu_environment() -> None: except ValueError: raise ValueError( f"Error parsing the environment variable '{ENV_CPU_INTRA_OP_CONFIG}'. Please " - f"cross-check the value" + f"cross-check the value." ) tf_config.threading.set_intra_op_parallelism_threads(intra_op_parallel_threads) From 14b6ea57e783a165a9fe8b48b76bc17ffc57ea4c Mon Sep 17 00:00:00 2001 From: Daksh Date: Mon, 17 Feb 2020 14:14:28 +0100 Subject: [PATCH 24/29] changes from code comments --- rasa/utils/tensorflow/environment.py | 68 +++++++++++-------- tests/utils/tensorflow/__init__.py | 0 .../{ => tensorflow}/test_tf_environment.py | 4 +- 3 files changed, 41 insertions(+), 31 deletions(-) create mode 100644 tests/utils/tensorflow/__init__.py rename tests/utils/{ => tensorflow}/test_tf_environment.py (62%) diff --git a/rasa/utils/tensorflow/environment.py b/rasa/utils/tensorflow/environment.py index 3de779030b65..b5371caa9d14 100644 --- a/rasa/utils/tensorflow/environment.py +++ b/rasa/utils/tensorflow/environment.py @@ -2,10 +2,6 @@ import os from typing import Text, Dict import typing - -if typing.TYPE_CHECKING: - from tensorflow import config as tf_config - import rasa.utils.common as rasa_utils from rasa.constants import ( ENV_GPU_CONFIG, @@ -13,35 +9,47 @@ ENV_CPU_INTRA_OP_CONFIG, ) +if typing.TYPE_CHECKING: + from tensorflow import config as tf_config + logger = logging.getLogger(__name__) -def setup_gpu_environment() -> None: +def _setup_gpu_environment() -> None: """Set configuration for TensorFlow GPU environment based on the environment variable set.""" gpu_memory_config = os.getenv(ENV_GPU_CONFIG) - if gpu_memory_config: - # Import from tensorflow only if necessary(environment variable was set) - from tensorflow import config as tf_config + if not gpu_memory_config: + return - parsed_gpu_config = parse_gpu_config(gpu_memory_config) - physical_gpus = tf_config.list_physical_devices("GPU") + # Import from tensorflow only if necessary(environment variable was set) + from tensorflow import config as tf_config - # Logic taken from https://www.tensorflow.org/guide/gpu - if physical_gpus: - for gpu_id, gpu_id_memory in parsed_gpu_config.items(): - allocate_gpu_memory(physical_gpus[gpu_id], gpu_id_memory) + parsed_gpu_config = _parse_gpu_config(gpu_memory_config) + physical_gpus = tf_config.list_physical_devices("GPU") - else: - rasa_utils.raise_warning( - f"You have an environment variable '{ENV_GPU_CONFIG}' set but no GPUs were detected to configure" - ) + # Logic taken from https://www.tensorflow.org/guide/gpu + if physical_gpus: + for gpu_id, gpu_id_memory in parsed_gpu_config.items(): + _allocate_gpu_memory(physical_gpus[gpu_id], gpu_id_memory) + + else: + rasa_utils.raise_warning( + f"You have an environment variable '{ENV_GPU_CONFIG}' set but no GPUs were detected to configure." + ) -def allocate_gpu_memory( +def _allocate_gpu_memory( gpu_instance: "tf_config.PhysicalDevice", logical_memory: int ) -> None: + """ + Create a new logical device out of the received GPU instance with specified amount of logical memory. + + Args: + gpu_instance: PhysicalDevice instance of a GPU device. + logical_memory: Absolute amount of memory to be allocated to the new logical device. + """ from tensorflow import config as tf_config @@ -56,15 +64,15 @@ def allocate_gpu_memory( ) except RuntimeError: - # Add a helper explanation where the error comes from + # Helper explanation of where the error comes from raise RuntimeError( "Error while setting up tensorflow environment. " - "Virtual devices must be set before GPUs have been initialized" + "Virtual devices must be set before GPUs have been initialized." ) -def parse_gpu_config(gpu_memory_config: Text) -> Dict[int, int]: - """Parse GPU configuration variable from a string to a dict""" +def _parse_gpu_config(gpu_memory_config: Text) -> Dict[int, int]: + """Parse GPU configuration variable from a string to a dict.""" # gpu_config is of format "gpu_id_1:gpu_id_1_memory, gpu_id_2: gpu_id_2_memory" # Parse it and store in a dictionary @@ -78,16 +86,17 @@ def parse_gpu_config(gpu_memory_config: Text) -> Dict[int, int]: parsed_gpu_config[instance_gpu_id] = instance_gpu_mem except ValueError: - # Add a helper explanation + # Helper explanation of where the error comes from raise ValueError( - f"Error parsing GPU configuration. Please cross-check the format of '{ENV_GPU_CONFIG}'." + f"Error parsing GPU configuration. Please cross-check the format of '{ENV_GPU_CONFIG}' " + f"at https://rasa.com/docs/rasa/api/tensorflow_usage.html#restricting-absolute-gpu-memory-available ." ) return parsed_gpu_config -def setup_cpu_environment() -> None: - """Set configuration for the CPU environment based on the environment variable set""" +def _setup_cpu_environment() -> None: + """Set configuration for the CPU environment based on the environment variable set.""" inter_op_parallel_threads = os.getenv(ENV_CPU_INTER_OP_CONFIG) intra_op_parallel_threads = os.getenv(ENV_CPU_INTRA_OP_CONFIG) @@ -121,5 +130,6 @@ def setup_cpu_environment() -> None: def setup_tf_environment() -> None: - setup_cpu_environment() - setup_gpu_environment() + """Setup CPU and GPU related environment settings for TensorFlow.""" + _setup_cpu_environment() + _setup_gpu_environment() diff --git a/tests/utils/tensorflow/__init__.py b/tests/utils/tensorflow/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/utils/test_tf_environment.py b/tests/utils/tensorflow/test_tf_environment.py similarity index 62% rename from tests/utils/test_tf_environment.py rename to tests/utils/tensorflow/test_tf_environment.py index 7366e66ac690..3fbcd480ace6 100644 --- a/tests/utils/test_tf_environment.py +++ b/tests/utils/tensorflow/test_tf_environment.py @@ -1,5 +1,5 @@ import pytest -from rasa.utils.tensorflow.environment import parse_gpu_config +from rasa.utils.tensorflow.environment import _parse_gpu_config @pytest.mark.parametrize( @@ -7,4 +7,4 @@ [("0: 1024", {0: 1024}), ("0:1024, 1:2048", {0: 1024, 1: 2048})], ) def test_gpu_config_parser(gpu_config_string, parsed_gpu_config): - assert parse_gpu_config(gpu_config_string) == parsed_gpu_config + assert _parse_gpu_config(gpu_config_string) == parsed_gpu_config From 98d54a9300d6a3181d933fd7dfa46b33bdbccfae Mon Sep 17 00:00:00 2001 From: Daksh Date: Mon, 17 Feb 2020 14:19:51 +0100 Subject: [PATCH 25/29] add docstring --- rasa/utils/tensorflow/environment.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/rasa/utils/tensorflow/environment.py b/rasa/utils/tensorflow/environment.py index b5371caa9d14..0fc52c07fdb7 100644 --- a/rasa/utils/tensorflow/environment.py +++ b/rasa/utils/tensorflow/environment.py @@ -43,8 +43,7 @@ def _setup_gpu_environment() -> None: def _allocate_gpu_memory( gpu_instance: "tf_config.PhysicalDevice", logical_memory: int ) -> None: - """ - Create a new logical device out of the received GPU instance with specified amount of logical memory. + """Create a new logical device out of the received GPU instance with specified amount of logical memory. Args: gpu_instance: PhysicalDevice instance of a GPU device. @@ -72,7 +71,14 @@ def _allocate_gpu_memory( def _parse_gpu_config(gpu_memory_config: Text) -> Dict[int, int]: - """Parse GPU configuration variable from a string to a dict.""" + """Parse GPU configuration variable from a string to a dict. + + Args: + gpu_memory_config: String containing the configuration for GPU usage. + + Returns: + Parsed configuration as a dictionary with GPU IDs as keys and requested memory as the value. + """ # gpu_config is of format "gpu_id_1:gpu_id_1_memory, gpu_id_2: gpu_id_2_memory" # Parse it and store in a dictionary From 69bd8f2c10527e802861d185d26f5b300080939b Mon Sep 17 00:00:00 2001 From: Daksh Varshneya Date: Mon, 17 Feb 2020 15:00:59 +0100 Subject: [PATCH 26/29] Update rasa/utils/tensorflow/environment.py Co-Authored-By: Tobias Wochinger --- rasa/utils/tensorflow/environment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rasa/utils/tensorflow/environment.py b/rasa/utils/tensorflow/environment.py index 0fc52c07fdb7..2cfbdd70e8d1 100644 --- a/rasa/utils/tensorflow/environment.py +++ b/rasa/utils/tensorflow/environment.py @@ -137,5 +137,6 @@ def _setup_cpu_environment() -> None: def setup_tf_environment() -> None: """Setup CPU and GPU related environment settings for TensorFlow.""" + _setup_cpu_environment() _setup_gpu_environment() From eb4fce6dcd7d4770676dedbb6e0cb3a308f87821 Mon Sep 17 00:00:00 2001 From: Daksh Varshneya Date: Mon, 17 Feb 2020 18:49:27 +0100 Subject: [PATCH 27/29] Apply suggestions from code review Co-Authored-By: Tobias Wochinger --- rasa/utils/tensorflow/environment.py | 2 +- tests/utils/tensorflow/test_tf_environment.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rasa/utils/tensorflow/environment.py b/rasa/utils/tensorflow/environment.py index 2cfbdd70e8d1..a3f86e0110bf 100644 --- a/rasa/utils/tensorflow/environment.py +++ b/rasa/utils/tensorflow/environment.py @@ -23,7 +23,7 @@ def _setup_gpu_environment() -> None: if not gpu_memory_config: return - # Import from tensorflow only if necessary(environment variable was set) + # Import from tensorflow only if necessary (environment variable was set) from tensorflow import config as tf_config parsed_gpu_config = _parse_gpu_config(gpu_memory_config) diff --git a/tests/utils/tensorflow/test_tf_environment.py b/tests/utils/tensorflow/test_tf_environment.py index 3fbcd480ace6..f8bd7d6916c2 100644 --- a/tests/utils/tensorflow/test_tf_environment.py +++ b/tests/utils/tensorflow/test_tf_environment.py @@ -6,5 +6,5 @@ "gpu_config_string, parsed_gpu_config", [("0: 1024", {0: 1024}), ("0:1024, 1:2048", {0: 1024, 1: 2048})], ) -def test_gpu_config_parser(gpu_config_string, parsed_gpu_config): +def test_gpu_config_parser(gpu_config_string: Text, parsed_gpu_config: Dict[int, int]): assert _parse_gpu_config(gpu_config_string) == parsed_gpu_config From 895035940e08c3956ee75cc9f6da0741975fb02a Mon Sep 17 00:00:00 2001 From: Daksh Date: Mon, 17 Feb 2020 18:53:05 +0100 Subject: [PATCH 28/29] shorten docstring --- rasa/utils/tensorflow/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/utils/tensorflow/environment.py b/rasa/utils/tensorflow/environment.py index a3f86e0110bf..cc8977f38e1b 100644 --- a/rasa/utils/tensorflow/environment.py +++ b/rasa/utils/tensorflow/environment.py @@ -43,7 +43,7 @@ def _setup_gpu_environment() -> None: def _allocate_gpu_memory( gpu_instance: "tf_config.PhysicalDevice", logical_memory: int ) -> None: - """Create a new logical device out of the received GPU instance with specified amount of logical memory. + """Create a new logical device for the requested amount of memory. Args: gpu_instance: PhysicalDevice instance of a GPU device. From f168ce9e0b21fda0ccb62803c6ab7e397f2e5436 Mon Sep 17 00:00:00 2001 From: Daksh Date: Mon, 17 Feb 2020 21:41:14 +0100 Subject: [PATCH 29/29] fix import --- tests/utils/tensorflow/test_tf_environment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/utils/tensorflow/test_tf_environment.py b/tests/utils/tensorflow/test_tf_environment.py index f8bd7d6916c2..3f30570d7975 100644 --- a/tests/utils/tensorflow/test_tf_environment.py +++ b/tests/utils/tensorflow/test_tf_environment.py @@ -1,4 +1,5 @@ import pytest +from typing import Text, Dict from rasa.utils.tensorflow.environment import _parse_gpu_config