From 272112503b59bfa6772a8ad99320124ef9fa9abc Mon Sep 17 00:00:00 2001 From: Matthew Ellis Date: Wed, 24 May 2023 15:14:59 -0700 Subject: [PATCH] Remove deprecated code and associated tests. --- doc/changelog.rst | 4 + smartsim/constants.py | 54 ------ smartsim/database/__init__.py | 9 +- smartsim/database/orchestrator.py | 290 ------------------------------ smartsim/tf/__init__.py | 62 ------- smartsim/tf/utils.py | 78 -------- tests/test_deprecations.py | 66 ------- 7 files changed, 5 insertions(+), 558 deletions(-) delete mode 100644 smartsim/constants.py delete mode 100644 smartsim/tf/__init__.py delete mode 100644 smartsim/tf/utils.py delete mode 100644 tests/test_deprecations.py diff --git a/doc/changelog.rst b/doc/changelog.rst index ec0529be7..ab3884257 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -25,11 +25,14 @@ Description A full list of changes and detailed notes can be found below: +- Remove deprecated code - Update Fortran tutorials for SmartRedis - Add support for multiple network interface binding in Orchestrator and Colocated DBs Detailed notes +- Deprecated launcher-specific orchestrators, constants, and ML utilities +were removed. (PR289_) - Update the Github Actions runner image from `macos-10.15`` to `macos-12``. The former began deprecation in May 2022 and was finally removed in May 2023 (PR285_) - The Fortran tutorials had not been fully updated to show how to handle return/error @@ -37,6 +40,7 @@ codes. These have now all been updated (PR284_) - Orchestrator and Colocated DB now accept a list of interfaces to bind to. The argument name is still `interface` for backward compatibility reasons. (PR281_) +.. _PR289: https://github.com/CrayLabs/SmartSim/pull/289 .. _PR285: https://github.com/CrayLabs/SmartSim/pull/285 .. _PR284: https://github.com/CrayLabs/SmartSim/pull/284 .. _PR281: https://github.com/CrayLabs/SmartSim/pull/281 diff --git a/smartsim/constants.py b/smartsim/constants.py deleted file mode 100644 index 63f6faebb..000000000 --- a/smartsim/constants.py +++ /dev/null @@ -1,54 +0,0 @@ -# BSD 2-Clause License -# -# Copyright (c) 2021-2023, Hewlett Packard Enterprise -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# Constants for SmartSim - -from warnings import simplefilter, warn - -dep_msg = "This is a deprecated module. Please use smartsim.status instead.\n" -dep_msg += "This module will be removed in the next release." - -simplefilter("once", DeprecationWarning) -warn(dep_msg, DeprecationWarning, stacklevel=2) - - -# Statuses that are applied to jobs -STATUS_RUNNING = "Running" -STATUS_COMPLETED = "Completed" -STATUS_CANCELLED = "Cancelled" -STATUS_FAILED = "Failed" -STATUS_NEW = "New" -STATUS_PAUSED = "Paused" - -# SmartSim status mapping -SMARTSIM_STATUS = { - "Running": STATUS_RUNNING, - "Paused": STATUS_PAUSED, - "Completed": STATUS_COMPLETED, - "Cancelled": STATUS_CANCELLED, - "Failed": STATUS_FAILED, - "New": STATUS_NEW, -} diff --git a/smartsim/database/__init__.py b/smartsim/database/__init__.py index 0a40f0412..f16cf7703 100644 --- a/smartsim/database/__init__.py +++ b/smartsim/database/__init__.py @@ -24,11 +24,4 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# deprecated classes -from .orchestrator import ( - CobaltOrchestrator, - LSFOrchestrator, - Orchestrator, - PBSOrchestrator, - SlurmOrchestrator, -) +from .orchestrator import Orchestrator diff --git a/smartsim/database/orchestrator.py b/smartsim/database/orchestrator.py index cc893f250..257284904 100644 --- a/smartsim/database/orchestrator.py +++ b/smartsim/database/orchestrator.py @@ -849,293 +849,3 @@ def _fill_reserved(self): "n", "nnodes", ] - - -# -# Deprecated Orchestrator Classes -# -# Same functionality incorporated into the Orchestrator base class -# - - -class CobaltOrchestrator(Orchestrator): - def __init__( - self, - port=6379, - db_nodes=1, - batch=True, - hosts=None, - run_command="aprun", - interface="ipogif0", - account=None, - queue=None, - time=None, - single_cmd=True, - **kwargs, - ): - """Initialize an Orchestrator reference for Cobalt based systems - - The orchestrator launches as a batch by default. If batch=False, - at launch, the orchestrator will look for an interactive - allocation to launch on. - - The Cobalt orchestrator does not support multiple databases per node. - - :param port: TCP/IP port, defaults to 6379 - :type port: int - :param db_nodes: number of database shards, defaults to 1 - :type db_nodes: int, optional - :param batch: Run as a batch workload, defaults to True - :type batch: bool, optional - :param hosts: specify hosts to launch on, defaults to None. Optional if not launching with OpenMPI - :type hosts: list[str] - :param run_command: specify launch binary. Options are ``mpirun`` and ``aprun``, defaults to ``aprun``. - :type run_command: str, optional - :param interface: network interface to use, defaults to "ipogif0" - :type interface: str, optional - :param account: account to run batch on - :type account: str, optional - :param queue: queue to launch batch in - :type queue: str, optional - :param time: walltime for batch 'HH:MM:SS' format - :type time: str, optional - """ - simplefilter("once", DeprecationWarning) - msg = "CobaltOrchestrator(...) is deprecated and will be removed in a future release.\n" - msg += "Please update your code to use Orchestrator(launcher='cobalt', ...)." - warn(msg, DeprecationWarning, stacklevel=2) - super().__init__( - port, - interface, - db_nodes=db_nodes, - batch=batch, - run_command=run_command, - single_cmd=single_cmd, - launcher="cobalt", - hosts=hosts, - account=account, - queue=queue, - time=time, - **kwargs, - ) - - -class LSFOrchestrator(Orchestrator): - def __init__( - self, - port=6379, - db_nodes=1, - cpus_per_shard=4, - gpus_per_shard=0, - batch=True, - hosts=None, - project=None, - time=None, - interface="ib0", - single_cmd=True, - **kwargs, - ): - """Initialize an Orchestrator reference for LSF based systems - - The orchestrator launches as a batch by default. If - batch=False, at launch, the orchestrator will look for an interactive - allocation to launch on. - - The LSFOrchestrator port provided will be incremented if multiple - databases per host are launched (``db_per_host>1``). - - Each database shard is assigned a resource set with cpus and gpus - allocated contiguously on the host: - it is the user's responsibility to check if - enough resources are available on each host. - - A list of hosts to launch the database on can be specified - these addresses must correspond to - those of the first ``db_nodes//db_per_host`` compute nodes - in the allocation: for example, for 8 ``db_nodes`` and 2 ``db_per_host`` - the ``host_list`` must contain the addresses of hosts 1, 2, 3, and 4. - - ``LSFOrchestrator`` is launched with only one ``jsrun`` command - as launch binary, and an Explicit Resource File (ERF) which is - automatically generated. The orchestrator is always launched on the - first ``db_nodes//db_per_host`` compute nodes in the allocation. - - :param port: TCP/IP port - :type port: int - :param db_nodes: number of database shards, defaults to 1 - :type db_nodes: int, optional - :param cpus_per_shard: cpus to allocate per shard, defaults to 4 - :type cpus_per_shard: int, optional - :param gpus_per_shard: gpus to allocate per shard, defaults to 0 - :type gpus_per_shard: int, optional - :param batch: Run as a batch workload, defaults to True - :type batch: bool, optional - :param hosts: specify hosts to launch on - :type hosts: list[str], optional - :param project: project to run batch on - :type project: str, optional - :param time: walltime for batch 'HH:MM' format - :type time: str, optional - :param interface: network interface to use - :type interface: str - """ - simplefilter("once", DeprecationWarning) - msg = "LSFOrchestrator(...) is deprecated and will be removed in a future release.\n" - msg += "Please update your code to use Orchestrator(launcher='lsf', ...)." - warn(msg, DeprecationWarning, stacklevel=2) - if single_cmd != True: - raise SSUnsupportedError( - "LSFOrchestrator can only be run with single_cmd=True (MPMD)." - ) - - super().__init__( - port, - interface, - db_nodes=db_nodes, - batch=batch, - run_command="jsrun", - launcher="lsf", - project=project, - hosts=hosts, - time=time, - cpus_per_shard=cpus_per_shard, - gpus_per_shard=gpus_per_shard, - **kwargs, - ) - - -class SlurmOrchestrator(Orchestrator): - def __init__( - self, - port=6379, - db_nodes=1, - batch=True, - hosts=None, - run_command="srun", - account=None, - time=None, - alloc=None, - db_per_host=1, - interface="ipogif0", - single_cmd=False, - **kwargs, - ): - """Initialize an Orchestrator reference for Slurm based systems - - The orchestrator launches as a batch by default. The Slurm orchestrator - can also be given an allocation to run on. If no allocation is provided, - and batch=False, at launch, the orchestrator will look for an interactive - allocation to launch on. - - The SlurmOrchestrator port provided will be incremented if multiple - databases per node are launched. - - SlurmOrchestrator supports launching with both ``srun`` and ``mpirun`` - as launch binaries. If mpirun is used, the hosts parameter should be - populated with length equal to that of the ``db_nodes`` argument. - - :param port: TCP/IP port - :type port: int - :param db_nodes: number of database shards, defaults to 1 - :type db_nodes: int, optional - :param batch: Run as a batch workload, defaults to True - :type batch: bool, optional - :param hosts: specify hosts to launch on - :type hosts: list[str] - :param run_command: specify launch binary. Options are "mpirun" and "srun", defaults to "srun" - :type run_command: str, optional - :param account: account to run batch on - :type account: str, optional - :param time: walltime for batch 'HH:MM:SS' format - :type time: str, optional - :param alloc: allocation to launch on, defaults to None - :type alloc: str, optional - :param db_per_host: number of database shards per system host (MPMD), defaults to 1 - :type db_per_host: int, optional - :param single_cmd: run all shards with one (MPMD) command, defaults to True - :type single_cmd: bool - """ - simplefilter("once", DeprecationWarning) - msg = "SlurmOrchestrator(...) is deprecated and will be removed in a future release.\n" - msg += "Please update your code to use Orchestrator(launcher='slurm', ...)." - warn(msg, DeprecationWarning, stacklevel=2) - super().__init__( - port, - interface, - db_nodes=db_nodes, - batch=batch, - run_command=run_command, - alloc=alloc, - db_per_host=db_per_host, - single_cmd=single_cmd, - launcher="slurm", - account=account, - hosts=hosts, - time=time, - **kwargs, - ) - - -class PBSOrchestrator(Orchestrator): - def __init__( - self, - port=6379, - db_nodes=1, - batch=True, - hosts=None, - run_command="aprun", - interface="ipogif0", - account=None, - time=None, - queue=None, - single_cmd=True, - **kwargs, - ): - """Initialize an Orchestrator reference for PBSPro based systems - - The ``PBSOrchestrator`` launches as a batch by default. If batch=False, - at launch, the ``PBSOrchestrator`` will look for an interactive - allocation to launch on. - - The PBS orchestrator does not support multiple databases per node. - - If ``mpirun`` is specifed as the ``run_command``, then the ``hosts`` - argument is required. - - :param port: TCP/IP port - :type port: int - :param db_nodes: number of compute nodes to span accross, defaults to 1 - :type db_nodes: int, optional - :param batch: run as a batch workload, defaults to True - :type batch: bool, optional - :param hosts: specify hosts to launch on, defaults to None - :type hosts: list[str] - :param run_command: specify launch binary. Options are ``mpirun`` and ``aprun``, defaults to "aprun" - :type run_command: str, optional - :param interface: network interface to use, defaults to "ipogif0" - :type interface: str, optional - :param account: account to run batch on - :type account: str, optional - :param time: walltime for batch 'HH:MM:SS' format - :type time: str, optional - :param queue: queue to launch batch in - :type queue: str, optional - """ - simplefilter("once", DeprecationWarning) - msg = "PBSOrchestrator(...) is deprecated and will be removed in a future release.\n" - msg += "Please update your code to use Orchestrator(launcher='pbs', ...)." - warn(msg, DeprecationWarning, stacklevel=2) - super().__init__( - port, - interface, - db_nodes=db_nodes, - batch=batch, - run_command=run_command, - single_cmd=single_cmd, - launcher="pbs", - hosts=hosts, - account=account, - queue=queue, - time=time, - **kwargs, - ) diff --git a/smartsim/tf/__init__.py b/smartsim/tf/__init__.py deleted file mode 100644 index a603912fe..000000000 --- a/smartsim/tf/__init__.py +++ /dev/null @@ -1,62 +0,0 @@ -# BSD 2-Clause License -# -# Copyright (c) 2021-2023, Hewlett Packard Enterprise -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import warnings - -msg = "smartsim.tf has been deprecated in favor of smartsim.ml.tf and\n" -msg += "will be removed in a future release. Please update your import statements." -warnings.warn( - msg, - DeprecationWarning, - stacklevel=2, -) - -from .._core._install.buildenv import Version_, Versioner -from ..error import SmartSimError -from ..log import get_logger - -logger = get_logger(__name__) - -vers = Versioner() -TF_VERSION = vers.TENSORFLOW - -try: - import tensorflow as tf - - installed_tf = Version_(tf.__version__) - assert installed_tf >= "2.4.0" - -except ImportError: # pragma: no cover - raise ModuleNotFoundError( - f"TensorFlow {TF_VERSION} is not installed. Please install it to use smartsim.tf" - ) from None -except AssertionError: # pragma: no cover - raise SmartSimError( - f"TensorFlow >= {TF_VERSION} is required for smartsim.tf, you have {tf.__version__}" - ) from None - - -from .utils import freeze_model diff --git a/smartsim/tf/utils.py b/smartsim/tf/utils.py deleted file mode 100644 index 2652e23da..000000000 --- a/smartsim/tf/utils.py +++ /dev/null @@ -1,78 +0,0 @@ -# BSD 2-Clause License -# -# Copyright (c) 2021-2023, Hewlett Packard Enterprise -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from pathlib import Path - -import tensorflow as tf -from tensorflow.python.framework.convert_to_constants import ( - convert_variables_to_constants_v2, -) - - -def freeze_model(model, output_dir, file_name): - """Freeze a Keras or TensorFlow Graph - - to use a Keras or TensorFlow model in SmartSim, the model - must be frozen and the inputs and outputs provided to the - smartredis.client.set_model_from_file() method. - - This utiliy function provides everything users need to take - a trained model and put it inside an ``orchestrator`` instance - - :param model: TensorFlow or Keras model - :type model: tf.Module - :param output_dir: output dir to save model file to - :type output_dir: str - :param file_name: name of model file to create - :type file_name: str - :return: path to model file, model input layer names, model output layer names - :rtype: str, list[str], list[str] - """ - # TODO figure out why layer names don't match up to - # specified name in Model init. - - if not file_name.endswith(".pb"): - file_name = file_name + ".pb" - - full_model = tf.function(lambda x: model(x)) - full_model = full_model.get_concrete_function( - tf.TensorSpec(model.inputs[0].shape, model.inputs[0].dtype) - ) - - frozen_func = convert_variables_to_constants_v2(full_model) - frozen_func.graph.as_graph_def() - - input_names = [x.name.split(":")[0] for x in frozen_func.inputs] - output_names = [x.name.split(":")[0] for x in frozen_func.outputs] - - tf.io.write_graph( - graph_or_graph_def=frozen_func.graph, - logdir=output_dir, - name=file_name, - as_text=False, - ) - model_file_path = str(Path(output_dir, file_name).resolve()) - return model_file_path, input_names, output_names diff --git a/tests/test_deprecations.py b/tests/test_deprecations.py deleted file mode 100644 index a296c4181..000000000 --- a/tests/test_deprecations.py +++ /dev/null @@ -1,66 +0,0 @@ -# BSD 2-Clause License -# -# Copyright (c) 2021-2023, Hewlett Packard Enterprise -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -import pytest - -from smartsim.database import ( - CobaltOrchestrator, - LSFOrchestrator, - PBSOrchestrator, - SlurmOrchestrator, -) - -tf_available = True -try: - import tensorflow -except ImportError: - tf_available = False - - -def test_deprecated_orchestrators(wlmutils): - with pytest.deprecated_call(): - _ = SlurmOrchestrator(interface=wlmutils.get_test_interface()) - - with pytest.deprecated_call(): - _ = LSFOrchestrator(interface=wlmutils.get_test_interface()) - - with pytest.deprecated_call(): - _ = CobaltOrchestrator(interface=wlmutils.get_test_interface()) - - with pytest.deprecated_call(): - _ = PBSOrchestrator(interface=wlmutils.get_test_interface()) - - -@pytest.mark.skipif(not tf_available, reason="Requires TF to run") -def test_deprecated_tf(): - with pytest.deprecated_call(): - from smartsim.tf import freeze_model - - -def test_deprecated_constants(): - with pytest.deprecated_call(): - from smartsim import constants