block.py

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# coding: utf-8
# pylint: disable= arguments-differ, too-many-lines, reimported
"""Base container class for all neural network models."""
__all__ = ['Block', 'HybridBlock', 'SymbolBlock']

import enum
import ctypes
import copy
import warnings
import weakref
from collections import OrderedDict, defaultdict
import contextlib
import contextvars

import re
import json
import numpy as np

from ..base import mx_real_t, MXNetError, NDArrayHandle, SymbolHandle, py_str, check_call, _LIB
from .. import symbol, ndarray, initializer, autograd, _deferred_compute as dc, name as _name, \
    profiler as _profiler, device as _device
from ..symbol.numpy import _symbol as np_symbol
from ..symbol import Symbol, fromjson
from ..ndarray import NDArray, get_dtype_name
from .parameter import Parameter, DeferredInitializationError
from .utils import _indent, _brief_print_list, HookHandle, shape_is_known
from .utils import _check_same_symbol_type, _check_all_np_ndarrays, _check_block_input_np_ndarrays
from .. import numpy_extension as _mx_npx
from .. import numpy as _mx_np, ndarray as nd
from .. util import is_np_array, np_shape, np_array, wrap_ctx_to_device_func


_naming_counter = contextvars.ContextVar('namecounter')
_prefix = contextvars.ContextVar('prefix', default='')


@contextlib.contextmanager
def _block_scope(block):
    """Append the classname of the current Block to the symbolic and memory profiler name scopes."""
    name = type(block).__name__.lower()
    counter = _naming_counter.get(None)
    if counter is not None:
        count = counter.get(name, 0)
        counter[name] = count + 1
        name = f'{name}{count}'
    counter_token = _naming_counter.set({})
    prefix_token = _prefix.set(_prefix.get() + name + '_')
    with _name.Prefix(_prefix.get()):
        with _profiler.scope(name + ':'):
            yield
    _naming_counter.reset(counter_token)
    _prefix.reset(prefix_token)


def _gather_type_device_info(args):
    """Analyze the elements inside the nested args object and find:
        - If there exists ndarray
        - If there exists symbol
        - All devices appearing in args

    Parameters
    ----------
    args : list or NDArray or Symbol
        Could be a nested architecture.

    Returns
    -------
    has_symbol : bool
        Whether the elements in args contains symbols
    has_ndarray : bool
        Whether the elements in args contains ndarrays
    device_set : set of mxnet.device.Device
        Contains all possible devices of the inner ndarrays in args. Can be empty if there is no
        ndarray inside args.
    first_device : mxnet.device.Device or None
        Device of the first appeared NDArray (for backward-compatibility)
    """
    if isinstance(args, NDArray):
        return False, True, {args.device}, args.device
    elif isinstance(args, Symbol):
        return True, False, set(), None
    elif isinstance(args, (list, tuple)):
        has_symbol = False
        has_ndarray = False
        device_set = set()
        first_device = None
        for ele in args:
            ele_has_sym, ele_has_nd, ele_device_set, ele_first_device =\
                _gather_type_device_info(ele)
            has_symbol = has_symbol or ele_has_sym
            has_ndarray = has_ndarray or ele_has_nd
            if first_device is None and ele_first_device is not None:
                first_device = ele_first_device
            device_set = device_set | ele_device_set
            if has_symbol and has_ndarray:
                break
        return has_symbol, has_ndarray, device_set, first_device
    else:
        return False, False, set(), None


def _flatten(args, inout_str):
    """Parse the arguments into a flattened list + an additional format array.
    The format array stores the structure of the original arguments to help reconstruct the inputs.

    Parameters
    ----------
    args : NDArray, Symbol, or (nested) list of Symbol or NDArray
        We allow None inside the args.
    inout_str : str
        The name of the HybridBlock

    Returns
    -------
    flat : list of Symbol or NDArray
        The flatten version of the input args.
    fmts : (nested) list of ints
        Stores the format information of the original structured args.
    """
    if isinstance(args, NDArray):
        return [args], int(0)
    if isinstance(args, Symbol):
        length = len(args.list_outputs())
        length = length if length > 1 else 0
        return [args], int(length)
    if args is None:
        return [None], int(-1)

    if not isinstance(args, (list, tuple)):
        raise ValueError("When hybridized, the input of HybridBlock {}"
                         " must be (nested) list of Symbol"
                         " or NDArray, "
                         "but got {} of type {}".format(inout_str, str(args), str(type(args))))
    flat = []
    fmts = []
    for i in args:
        arg, fmt = _flatten(i, inout_str)
        flat.extend(arg)
        fmts.append(fmt)
    return flat, fmts


def _regroup(args, fmt):
    """Reconstruct the structured arguments based on the flattened version.

    Parameters
    ----------
    args : NDArray, Symbol, or (nested) list of Symbol or NDArray
        We allow None inside the args.
    fmt : (nested) list of ints
        Stores the format information of the original structured args.

    Returns
    -------
    ret : NDArray, Symbol, or (nested) list of Symbol or NDArray

    """
    def _merger(args, fmt):
        """Recursive call to merge the arguments"""
        if isinstance(fmt, int):
            if fmt < -1:
                raise ValueError("Unsupported encoded format {}.".format(fmt))
            if fmt == 0:
                return args[0], args[1:]
            if fmt == -1:
                if args[0] is not None:
                    raise ValueError('We do not support passing types that are not None'
                                     ' when the initial HybridBlock has received NoneType and'
                                     ' has been hybridized.'
                                     ' Received arg = {}, fmt = {}.'.format(args[0], fmt))
                return None, args[1:]
            else:
                return args[:fmt], args[fmt:]

        if not isinstance(args, (list, tuple)):
            raise ValueError("When hybridized, the output of HybridBlock must be (nested)"
                             " list of Symbol or NDArray, "
                             "but got {} of type {}".format(args, type(args)))
        ret = []
        for i in fmt:
            res, args = _merger(args, i)
            ret.append(res)
        return ret, args
    return _merger(args, fmt)[0]


class Block:
    """Base class for all neural network layers and models. Your models should
    subclass this class.

    :py:class:`Block` can be nested recursively in a tree structure. You can create and
    assign child :py:class:`Block` as regular attributes::

        import mxnet as mx
        from mxnet.gluon import Block, nn

        class Model(Block):
            def __init__(self, **kwargs):
                super(Model, self).__init__(**kwargs)
                self.dense0 = nn.Dense(20)
                self.dense1 = nn.Dense(20)

            def forward(self, x):
                x = mx.npx.relu(self.dense0(x))
                return mx.npx.relu(self.dense1(x))

        model = Model()
        model.initialize(device=mx.cpu(0))
        model(mx.np.zeros((10, 10), device=mx.cpu(0)))


    Child :py:class:`Block` assigned this way will be registered and :py:meth:`collect_params`
    will collect their Parameters recursively. You can also manually register
    child blocks with :py:meth:`register_child`.

    """
    def __init__(self):
        self._children = OrderedDict()
        self._reg_params = {}
        self._forward_hooks = OrderedDict()
        self._forward_pre_hooks = OrderedDict()

    def __repr__(self):
        s = '{name}(\n{modstr}\n)'
        modstr = '\n'.join(['  ({key}): {block}'.format(key=key,
                                                        block=_indent(block.__repr__(), 2))
                            for key, block in self.__dict__.items() if isinstance(block, Block)])
        return s.format(name=self.__class__.__name__, modstr=modstr)

    def __setattr__(self, name, value):
        """Registers parameters."""

        if hasattr(self, name):
            existing = getattr(self, name)
            if isinstance(existing, (Parameter, Block)) and not isinstance(value, type(existing)):
                raise TypeError('Changing attribute type for {name} from {type1} to {type2}' \
                                'is not allowed.'.format(
                                    name=name, type1=type(existing), type2=type(value)))

        if isinstance(value, Block):
            self.register_child(value, name)
        elif isinstance(value, Parameter):
            self._reg_params[name] = value

        super(Block, self).__setattr__(name, value)

    def _check_container_with_block(self):
        children = set(self._children.values())
        def _find_unregistered_block_in_container(data):
            # Find whether a nested container structure contains Blocks
            if isinstance(data, (list, tuple)):
                for ele in data:
                    if _find_unregistered_block_in_container(ele):
                        return True
                return False
            elif isinstance(data, dict):
                for _, v in data.items():
                    if _find_unregistered_block_in_container(v):
                        return True
                return False
            elif isinstance(data, Block):
                return not data in (c() for c in children)
            else:
                return False
        for k, v in self.__dict__.items():
            if isinstance(v, (list, tuple, dict)) and not (k.startswith('__') or k == '_children'):
                if _find_unregistered_block_in_container(v):
                    warnings.warn('"{name}" is an unregistered container with Blocks. '
                                  'Note that Blocks inside the list, tuple or dict will not be '
                                  'registered automatically. Make sure to register them using '
                                  'register_child() or switching to '
                                  'nn.Sequential/nn.HybridSequential instead. '
                                  .format(name=self.__class__.__name__ + "." + k), stacklevel=3)

    def _alias(self):
        return self.__class__.__name__.lower()

    @property
    def params(self):
        """Returns this :py:class:`Block`'s parameter dictionary (does not include its
        children's parameters)."""
        return self._reg_params

    def collect_params(self, select=None):
        """Returns a :py:class:`Dict` containing this :py:class:`Block` and all of its
        children's Parameters(default), also can returns the select :py:class:`Dict`
        which match some given regular expressions.

        For example, collect the specified parameters in ['conv1.weight', 'conv1.bias', 'fc.weight',
        'fc.bias']::

            model.collect_params('conv1.weight|conv1.bias|fc.weight|fc.bias')

        or collect all parameters whose names end with 'weight' or 'bias', this can be done
        using regular expressions::

            model.collect_params('.*weight|.*bias')

        Parameters
        ----------
        select : str
            regular expressions

        Returns
        -------
        The selected :py:class:`Dict`
        """
        # We need to check here because blocks inside containers are not supported.
        self._check_container_with_block()
        return self._collect_params_with_prefix(select=select)

    def _collect_params_with_prefix(self, prefix='', select=None):
        if prefix:
            prefix += '.'
        if select is None:
            ret = {prefix + key : val for key, val in self._reg_params.items()}
        else:
            pattern = re.compile(select)
            ret = {prefix + key : val for key, val in self._reg_params.items() if pattern.match(prefix + key)}

        for name, child in self._children.items():
            ret.update(child()._collect_params_with_prefix(prefix + name, select))
        return ret

    def save_parameters(self, filename, deduplicate=False):
        """Save parameters to file.

        Saved parameters can only be loaded with `load_parameters`. Note that this
        method only saves parameters, not model structure. If you want to save
        model structures, please use :py:meth:`HybridBlock.export`.

        Parameters
        ----------
        filename : str
            Path to file.
        deduplicate : bool, default False
            If True, save shared parameters only once. Otherwise, if a Block
            contains multiple sub-blocks that share parameters, each of the
            shared parameters will be separately saved for every sub-block.

        References
        ----------
        `Saving and Loading Gluon Models \
        <https://mxnet.apache.org/api/python/docs/tutorials/packages/gluon/blocks/save_load_params.html>`_
        """
        params = self._collect_params_with_prefix()

        if deduplicate:
            # Shared parameters are stored only a single time as of MXNet 1.6.
            # Shared parameters are registered under multiple prefixes returned by
            # _collect_params_with_prefix. We select a single one and only store
            # it. In load_parameters it is sufficient for a shared parameter to
            # only set it for a single prefix.
            reverse_params = {v: k for k, v in params.items()}
            params = {v: k for k, v in reverse_params.items()}

        arg_dict = {key: val._reduce() for key, val in params.items()}
        if is_np_array():
            _mx_npx.savez(filename, **arg_dict)
        else:
            ndarray.save(filename, arg_dict)

    @wrap_ctx_to_device_func
    def load_parameters(self, filename, device=None, allow_missing=False,
                        ignore_extra=False, cast_dtype=False, dtype_source='current'):
        """Load parameters from file previously saved by `save_parameters`.

        Parameters
        ----------
        filename : str
            Path to parameter file.
        device : Device or list of Device, default cpu()
            Device(s) to initialize loaded parameters on.
        allow_missing : bool, default False
            Whether to silently skip loading parameters not represents in the file.
        ignore_extra : bool, default False
            Whether to silently ignore parameters from the file that are not
            present in this Block.
        cast_dtype : bool, default False
            Cast the data type of the NDArray loaded from the checkpoint to the dtype
            provided by the Parameter if any.
        dtype_source : str, default 'current'
            must be in {'current', 'saved'}
            Only valid if cast_dtype=True, specify the source of the dtype for casting
            the parameters
        References
        ----------
        `Saving and Loading Gluon Models \
        <https://mxnet.apache.org/api/python/docs/tutorials/packages/gluon/blocks/save_load_params.html>`_
        """
        if is_np_array():
            # failure may happen when loading parameters saved as NDArrays within
            # NumPy semantics. Check the failure type and recover from it if it happens.
            try:
                loaded = _mx_npx.load(filename)
            except MXNetError as e:
                err_msg = str(e)
                if 'is_np_shape' in err_msg:
                    # Loading failure due to parameters saved without numpy semantics.
                    # Temporarily disable numpy semantics and load parameters. After it's
                    # done, resume the numpy semantics. This is fine because the cases
                    # numpy ndarray covers is a superset of the legacy ndarray's.
                    with np_array(False):
                        with np_shape(False):
                            loaded_nds = ndarray.load(filename)
                    assert isinstance(loaded_nds, dict),\
                        'expecting a dict type, got {}'.format(str(type(loaded_nds)))
                    loaded = {k: loaded_nds[k].as_np_ndarray() for k in loaded_nds}
                else:
                    raise ValueError(err_msg)
        else:
            loaded = ndarray.load(filename)

        if not loaded:
            return
        full_dict = {'params': loaded, 'filename': filename}
        self.load_dict(full_dict, device, allow_missing, ignore_extra, cast_dtype, dtype_source)

    def load_dict(self, param_dict, device=None, allow_missing=False,
                  ignore_extra=False, cast_dtype=False, dtype_source="current"):
        """Load parameters from dict

        Parameters
        ----------
        param_dict : dict
            Dictionary containing model parameters
        device : Device, optional
            Device context on which the memory is allocated. Default is
            `mxnet.device.current_device()`.
        allow_missing : bool, default False
            Whether to silently skip loading parameters not represented in the file.
        ignore_extra : bool, default False
            Whether to silently ignore parameters from the file that are not
            present in this dict.
        cast_dtype : bool, default False
            Cast the data type of the NDArray loaded from the checkpoint to the dtype
            provided by the Parameter if any
        dtype_source : str, default 'current'
            must be in {'current', 'saved'}
            Only valid if cast_dtype=True, specify the source of the dtype for casting
            the parameters
        """
        if isinstance(param_dict.get('filename'), str):
            # pass from load_parameters
            filename = param_dict['filename']
            param_dict = param_dict['params']
        else:
            filename = None
        params = self.collect_params()
        error_str = f"file: {filename}" if filename else "param_dict"
        loaded = {k[4:] if k.startswith('arg:') or k.startswith('aux:') else k: v \
                  for k, v in param_dict.items()}

        if not allow_missing:
            params_inv = defaultdict(list)
            for k, v in params.items():
                params_inv[v].append(k)

            for name, param in params.items():
                assert any(p in loaded for p in params_inv[param]), \
                f"Parameter '{name}' is missing in '{error_str}', which contains parameters: {_brief_print_list(loaded.keys())}. " \
                    "Set allow_missing=True to ignore missing parameters."

        if device is None:
            device = _device.current_device()
        for name in loaded:
            if not ignore_extra and name not in params:
                raise ValueError(
                    f"Parameter '{name}' loaded from '{error_str}' is not present in Dict, " \
                    f"which contains parameters {_brief_print_list(params.keys())}. Set ignore_extra=True to ignore. ")
            if name in params:
                param = loaded[name]
                if isinstance(param, np.ndarray):
                    param = _mx_np.array(param) if is_np_array() else nd.array(param)
                params[name]._load_init(param, device, cast_dtype=cast_dtype, dtype_source=dtype_source)

    def register_child(self, block, name=None):
        """Registers block as a child of self. :py:class:`Block` s assigned to self as
        attributes will be registered automatically."""
        if name is None:
            name = str(len(self._children))
        self._children[name] = weakref.ref(block)

    def register_forward_pre_hook(self, hook):
        r"""Registers a forward pre-hook on the block.

        The hook function is called immediately before :func:`forward`.
        It should not modify the input or output.

        Parameters
        ----------
        hook : callable
            The forward hook function of form `hook(block, input) -> None`.

        Returns
        -------
        :class:`mxnet.gluon.utils.HookHandle`
        """
        handle = HookHandle()
        handle.attach(self._forward_pre_hooks, hook)
        return handle

    def register_forward_hook(self, hook):
        r"""Registers a forward hook on the block.

        The hook function is called immediately after :func:`forward`.
        It should not modify the input or output.

        Parameters
        ----------
        hook : callable
            The forward hook function of form `hook(block, input, output) -> None`.

        Returns
        -------
        :class:`mxnet.gluon.utils.HookHandle`
        """
        handle = HookHandle()
        handle.attach(self._forward_hooks, hook)
        return handle

    def apply(self, fn):
        r"""Applies ``fn`` recursively to every child block as well as self.

        Parameters
        ----------
        fn : callable
            Function to be applied to each submodule, of form `fn(block)`.

        Returns
        -------
        this block
        """
        for cld in self._children.values():
            cld().apply(fn)
        fn(self)
        return self

    @wrap_ctx_to_device_func
    def initialize(self, init=initializer.Uniform(), device=None, verbose=False,
                   force_reinit=False):
        """Initializes :py:class:`Parameter` s of this :py:class:`Block` and its children.

        Parameters
        ----------
        init : Initializer
            Global default Initializer to be used when :py:meth:`Parameter.init` is ``None``.
            Otherwise, :py:meth:`Parameter.init` takes precedence.
        device : Device or list of Device
            Keeps a copy of Parameters on one or many device(s).
        verbose : bool, default False
            Whether to verbosely print out details on initialization.
        force_reinit : bool, default False
            Whether to force re-initialization if parameter is already initialized.
        """
        params = self.collect_params()
        if verbose:
            init.set_verbosity(verbose=verbose)
        for v in params.values():
            v.initialize(None, device, init, force_reinit=force_reinit)

    def save(self, prefix):
        """Save the model architecture and parameters to load again later

        Saves the model architecture as a nested dictionary where each Block
        in the model is a dictionary and its children are sub-dictionaries.

        Each Block is uniquely identified by Block class name and a unique ID.
        We save each Block's parameter UUID to restore later in order to match
        the saved parameters.

        Recursively traverses a Block's children in order (since its an
        OrderedDict) and uses the unique ID to denote that specific Block.

        Assumes that the model is created in an identical order every time.
        If the model is not able to be recreated deterministically do not
        use this set of APIs to save/load your model.

        For HybridBlocks, the cached_graph is saved (Symbol & inputs) if
        it has already been hybridized.

        Parameters
        ----------
        prefix : str
            The prefix to use in filenames for saving this model:
            <prefix>-model.json and <prefix>-model.params
        """
        # create empty model structure
        model = {}
        def _save_cached_graphs(blk, structure, index=0):
            # create new entry for this block
            mdl = {}
            # encode unique name based on block type and ID
            name = type(blk).__name__.lower()
            structure[name+str(index)] = mdl
            index += 1
            if isinstance(blk, HybridBlock):
                if blk._cached_graph:
                    # save in/out formats
                    mdl['in_format'] = blk._in_format
                    mdl['out_format'] = blk._out_format
                    # save cached graph & input symbols
                    syms, out = blk._cached_graph
                    mdl_syms = []
                    for sym in syms:
                        mdl_syms.append(sym.tojson())
                    mdl['inputs'] = mdl_syms
                    mdl['symbol'] = out.tojson()
                    mdl['hybridized'] = True
                else:
                    mdl['hybridized'] = False
            # save param uuids
            pmap = {}
            mdl['params'] = pmap
            pnames = list(blk.params.keys())
            for p in pnames:
                param = blk.params[p]
                pmap[p] = param._uuid
            # recursively save children
            for child in blk._children.values():
                index = _save_cached_graphs(child(), mdl, index)
            # return latest index (ie. block count)
            return index

        # save top-level block
        _save_cached_graphs(self, model)
        # save model
        with open(prefix+'-model.json', 'w') as fp:
            json.dump(model, fp)
        # save params
        self.save_parameters('MyModel-model.params')

    def load(self, prefix):
        """Load a model saved using the `save` API

        Reconfigures a model using the saved configuration. This function
        does not regenerate the model architecture. It resets each Block's
        parameter UUIDs as they were when saved in order to match the names of the
        saved parameters.

        This function assumes the Blocks in the model were created in the same
        order they were when the model was saved. This is because each Block is
        uniquely identified by Block class name and a unique ID in order (since
        its an OrderedDict) and uses the unique ID to denote that specific Block.

        Assumes that the model is created in an identical order every time.
        If the model is not able to be recreated deterministically do not
        use this set of APIs to save/load your model.

        For HybridBlocks, the cached_graph (Symbol & inputs) and settings are
        restored if it had been hybridized before saving.

        Parameters
        ----------
        prefix : str
            The prefix to use in filenames for loading this model:
            <prefix>-model.json and <prefix>-model.params
        """
        # load model json from file
        with open(prefix+'-model.json') as fp:
            model = json.load(fp)

        def _load_cached_graphs(blk, structure, index=0):
            # get block name
            name = type(blk).__name__.lower()
            # lookup previous encoded name based on block type and ID
            mdl = structure[name+str(index)]
            index += 1
            if isinstance(blk, HybridBlock):
                if mdl['hybridized']:
                    # restore in/out formats
                    blk._in_format = mdl['in_format']
                    blk._out_format = mdl['out_format']
                    # get saved symbol
                    out = fromjson(mdl['symbol'])
                    syms = []
                    # recreate inputs for this symbol
                    for inp in mdl['inputs']:
                        syms.append(fromjson(inp))
                    # reset cached_graph and active status
                    blk._cached_graph = (syms, out)
                    blk._active = True
            # reload param uuids
            pmap = mdl['params']
            for p, uuid in pmap.items():
                param = blk.params[p]
                param._uuid = uuid
            # recursively reload children
            for child in blk._children.values():
                index = _load_cached_graphs(child(), mdl, index)
            # return latest index (ie. block count)
            return index

        # load top-level block
        _load_cached_graphs(self, model)
        # load params
        self.load_parameters('MyModel-model.params')

    def hybridize(self, active=True, **kwargs):
        """ Please refer description of HybridBlock hybridize().
        """
        for cld in self._children.values():
            cld().hybridize(active, **kwargs)

    def cast(self, dtype):
        """Cast this Block to use another data type.

        Parameters
        ----------
        dtype : str or numpy.dtype
            The new data type.
        """
        for child in self._children.values():
            child().cast(dtype)
        for _, param in self.params.items():
            param.cast(dtype)

    def zero_grad(self):
        """Sets all Parameters' gradient buffer to 0."""
        # collect gradient arrays for each device
        arrays = defaultdict(list)
        params = self.collect_params()
        for p in params.values():
            if p.grad_req == 'null' or p._grad is None:
                continue
            for g in p.list_grad():
                if g.stype == 'row_sparse':
                    ndarray.zeros_like(g, out=g)
                else:
                    if is_np_array():
                        arrays[g.device].append(g.as_nd_ndarray())
                    else:
                        arrays[g.device].append(g)

        if len(arrays) == 0:
            return

        for arr in arrays.values():
            ndarray.reset_arrays(*arr, num_arrays=len(arr))

    def reset_device(self, device):
        """Re-assign all Parameters to other devices.

        Parameters
        ----------
        device : Device or list of Device, default :py:meth:`device.current_device()`.
            Assign Parameter to given device. If device is a list of Device, a
            copy will be made for each device.
        """
        params = self.collect_params()
        for i in params.values():
            i.reset_device(device)

    def reset_ctx(self, ctx):
        """This function has been deprecated. Please refer to ``Block.reset_device``."""
        warnings.warn('Block.reset_ctx has been renamed to'
                      ' Block.reset_device', DeprecationWarning)
        self.reset_device(ctx)

    def setattr(self, name, value):
        """Set an attribute to a new value for all Parameters.

        For example, set grad_req to null if you don't need gradient w.r.t a
        model's Parameters::

            model.setattr('grad_req', 'null')

        or change the learning rate multiplier::

            model.setattr('lr_mult', 0.5)

        Parameters
        ----------
        name : str
            Name of the attribute.
        value : valid type for attribute name
            The new value for the attribute.
        """
        params = self.collect_params()
        for i in params.values():
            setattr(i, name, value)

    def share_parameters(self, shared):
        """Share parameters recursively inside the model.

        For example, if you want ``dense1`` to share ``dense0``'s weights, you can do::

            dense0 = nn.Dense(20)
            dense1 = nn.Dense(20)
            dense1.share_parameters(dense0.collect_params())

        which equals to
            dense1.weight = dense0.weight
            dense1.bias = dense0.bias

        Note that unlike the `load_parameters` or `load_dict` functions,
        `share_parameters` results in the `Parameter` object being shared (or
        tied) between the models, whereas `load_parameters` or `load_dict` only
        set the value of the data dictionary of a model. If you call
        `load_parameters` or `load_dict` after `share_parameters`, the loaded
        value will be reflected in all networks that use the shared (or tied)
        `Parameter` object.

        Parameters
        ----------
        shared : Dict
            Dict of the shared parameters.

        Returns
        -------
        this block
        """
        if shared is None:
            return self
        if not isinstance(shared, (dict, OrderedDict)):
            raise ValueError("'shared' should be in type of Dict. Get type {}!".format(type(shared)))
        shared_set = set(shared.keys())
        self._shared_parameters(shared, shared_set)
        if len(shared_set) > 0:
            for name in shared_set:
                warnings.warn("Parameter name {} is not in the current model!".format(name))
        return self

    def _shared_parameters(self, shared, shared_set, prefix=""):
        if prefix:
            prefix += '.'
        for name in self._reg_params:
            key = prefix + name
            if shared.get(key) is not None:
                setattr(self, name, shared[key])
                shared_set.remove(key)
        for name, child in self._children.items():
            child()._shared_parameters(shared, shared_set, prefix + name)

    def __call__(self, *args):
        """Calls forward. Only accepts positional arguments."""
        for hook in self._forward_pre_hooks.values():
            hook(self, args)

        out = self.forward(*args)

        for hook in self._forward_hooks.values():
            hook(self, args, out)
        if _mx_npx.is_np_array():
            _check_all_np_ndarrays(out)
        return out

    def forward(self, *args):
        """Overrides to implement forward computation using :py:class:`NDArray`. Only
        accepts positional arguments.

        Parameters
        ----------
        *args : list of NDArray
            Input tensors.
        """
        # pylint: disable= invalid-name
        raise NotImplementedError

    def register_op_hook(self, callback, monitor_all=False):
        """Install callback monitor.

        Parameters
        ----------
        callback : function
            Function called to inspect the values of the intermediate outputs
            of blocks after hybridization. It takes 3 parameters:
            name of the tensor being inspected (str)
            name of the operator producing or consuming that tensor (str)
            tensor being inspected (NDArray).
        monitor_all : bool, default False
            If True, monitor both input and output, otherwise monitor output only.
        """
        for cld in self._children.values():
            cld().register_op_hook(callback, monitor_all)

    def summary(self, *inputs):
        """Print the summary of the model's output and parameters.

        The network must have been initialized, and must not have been hybridized.

        Parameters
        ----------
        inputs : object
            Any input that the model supports. For any tensor in the input, only
            :class:`mxnet.ndarray.NDArray` is supported.
        """
        summary = OrderedDict()
        seen = set()
        hooks = []

        def _get_shape_str(args):
            def flatten(args):
                if not isinstance(args, (list, tuple)):
                    return [args], int(0)
                flat = []
                fmts = []
                for i in args:
                    arg, fmt = flatten(i)
                    flat.extend(arg)
                    fmts.append(fmt)
                return flat, fmts

            def regroup(args, fmt):
                if isinstance(fmt, int):
                    if fmt == 0:
                        return args[0], args[1:]
                    return args[:fmt], args[fmt:]
                ret = []
                for i in fmt:
                    res, args = regroup(args, i)
                    ret.append(res)
                return ret, args

            flat_args, fmts = flatten(args)
            flat_arg_shapes = [x.shape if isinstance(x, ndarray.NDArray) else x
                               for x in flat_args]
            shapes = regroup(flat_arg_shapes, fmts)[0]
            if isinstance(shapes, list):
                shape_str = str(shapes)[1:-1]
            else:
                shape_str = str(shapes)
            return shape_str.replace('L', '')

        def _register_summary_hook(block):
            assert not isinstance(block, HybridBlock) or not block._active, \
                    '"{}" must not be hybridized to print summary.'.format(type(block).__name__)
            def _summary_hook(block, _, outputs):
                class_name = block.__class__.__name__
                block_idx = len(summary) - 1

                m_key = f'{class_name}-{block_idx+1}'
                summary[m_key] = OrderedDict()
                summary[m_key]['output_shape'] = _get_shape_str(outputs)

                params = 0
                summary[m_key]['trainable'] = 0
                summary[m_key]['shared'] = 0
                for p in block.params.values():
                    params += p.data().size
                    summary[m_key]['trainable'] += 0 if p.grad_req == 'null' else p.data().size
                    if p in seen:
                        summary[m_key]['shared'] += p.data().size
                    else:
                        seen.add(p)
                summary[m_key]['n_params'] = params

            from .nn.basic_layers import Sequential, HybridSequential
            if not isinstance(block, (Sequential, HybridSequential)):
                hooks.append(block.register_forward_hook(_summary_hook))

        summary['Input'] = OrderedDict()
        summary['Input']['output_shape'] = _get_shape_str(inputs)
        summary['Input']['n_params'] = 0
        summary['Input']['trainable'] = 0
        summary['Input']['shared'] = 0

        try:
            self.apply(_register_summary_hook)
            self(*inputs)

            line_format = '{:>20}  {:>42} {:>15}'
            print('-'*80)
            print(line_format.format('Layer (type)', 'Output Shape', 'Param #'))
            print('='*80)
            total_params = 0
            trainable_params = 0
            shared_params = 0
            for layer in summary:
                print(line_format.format(layer,
                                         str(summary[layer]['output_shape']),
                                         summary[layer]['n_params']))
                total_params += summary[layer]['n_params']
                trainable_params += summary[layer]['trainable']
                shared_params += summary[layer]['shared']
            print('='*80)
            print('Parameters in forward computation graph, duplicate included')
            print('   Total params: ' + str(total_params))
            print('   Trainable params: ' + str(trainable_params))
            print('   Non-trainable params: ' + str(total_params - trainable_params))
            print('Shared params in forward computation graph: ' + str(shared_params))
            print('Unique parameters in model: ' + str(total_params - shared_params))
            print('-'*80)
        finally:
            for h in hooks:
                h.detach()


class HybridBlock(Block):
    """`HybridBlock` supports forwarding with both Symbol and NDArray.

    `HybridBlock` is similar to `Block`, with a few differences::

        import mxnet as mx
        from mxnet.gluon import HybridBlock, nn

        class Model(HybridBlock):
            def __init__(self, **kwargs):
                super(Model, self).__init__(**kwargs)
                self.dense0 = nn.Dense(20)
                self.dense1 = nn.Dense(20)

            def forward(self, x):
                x = mx.npx.relu(self.dense0(x))
                return mx.npx.relu(self.dense1(x))

        model = Model()
        model.initialize(device=mx.cpu(0))
        model.hybridize()
        model(mx.np.zeros((10, 10), device=mx.cpu(0)))

    Forward computation in :py:class:`HybridBlock` must be static to work with :py:class:`Symbol` s,
    i.e. you cannot call :py:meth:`NDArray.asnumpy`, :py:attr:`NDArray.shape`,
    :py:attr:`NDArray.dtype`, `NDArray` indexing (`x[i]`) etc on tensors.
    Also, you cannot use branching or loop logic that bases on non-constant
    expressions like random numbers or intermediate results, since they change
    the graph structure for each iteration.

    Before activating with :py:meth:`hybridize()`, :py:class:`HybridBlock` works just like normal
    :py:class:`Block`. After activation, :py:class:`HybridBlock` will create a symbolic graph
    representing the forward computation and cache it. On subsequent forwards,
    the cached graph will be used instead of :py:meth:`forward`.

    Please see references for detailed tutorial.

    References
    ----------
        `Hybridize - A Hybrid of Imperative and Symbolic Programming
        <https://mxnet.apache.org/versions/master/api/python/docs/tutorials/packages/gluon/blocks/hybridize.html>`_
    """
    class OptConstraint:
        class Flag(enum.Flag):
            DisableAMP = enum.auto()

        def __init__(self, flag) -> None:
            self.flag = flag
            self.enter_state = None

        def __enter__(self):
            self.enter_state = HybridBlock.OptConstraint.Flag(get_optimization_constraints())
            target_state = self.enter_state | self.flag
            set_optimization_constraints(target_state)

        def __exit__(self, ptype, value, trace):
            set_optimization_constraints(self.enter_state)

        @staticmethod
        def disable_all():
            opt_flag = HybridBlock.OptConstraint.Flag()
            for flag in HybridBlock.OptConstraint.Flag:
                opt_flag |= flag

        @staticmethod
        def disable_amp():
            return HybridBlock.OptConstraint(HybridBlock.OptConstraint.Flag.DisableAMP)

    def __init__(self):
        super(HybridBlock, self).__init__()
        assert hasattr(self, "hybrid_forward") is False, (
            "'forward' instead of 'hybrid_forward' interface needs to be used starting from Gluon2.0."
            "Please follow MXNet2.0 Migration Guide to use new APIs.")
        self._cached_graph = ()
        self._cached_op = None
        self._out_format = None
        self._in_format = None
        self._called_infer_shape_already = False
        self._active = False
        self._flags = []
        self._callback = None
        self._monitor_all = False
        self._backend = None
        self._backend_opts = {}
        self._partition_if_dynamic = True
        self._first_forward = True

    def __setattr__(self, name, value):
        """Registers parameters."""
        super(HybridBlock, self).__setattr__(name, value)
        if isinstance(value, HybridBlock):
            if self._active:
                warnings.warn("Currently the model has been hybridized. Automatically deactivate the hybridization \
                               when changing the children blocks.")
                self._active = False
            self._clear_cached_op()

    @staticmethod
    def generate_arg_names(arg_num):
        return ['data'] if arg_num == 1 else ['data{}'.format(i) for i in range(arg_num)]

    def _get_graph(self, *args):
        if not self._cached_graph:
            flatten_args, self._in_format = _flatten(args, "input")
            flatten_args = [ele.detach() if ele is not None else None for ele in flatten_args]
            real_args = [ele for ele in flatten_args if ele is not None]
            if len(real_args) == 0:
                raise ValueError('All args are None and we do not support such a case.'
                                 ' Received args={}'.format(args))
            arg_names = HybridBlock.generate_arg_names(len(real_args))
            symbol_inputs = [
                symbol.var(name).as_np_ndarray()
                if isinstance(arg, _mx_np.ndarray) else symbol.var(name)
                for arg, name in zip(real_args, arg_names)
            ]
            dc.set_variable(real_args, symbol_inputs)
            args = _regroup(flatten_args, self._in_format)
            with autograd.pause(), dc.context():
                out = super().__call__(*args)
            flatten_out, self._out_format = _flatten(out, "output")
            symbol_outputs = dc.get_symbol(flatten_out, sym_cls=type(symbol_inputs[0]))
            dc.clear(flatten_out)
            self._cached_graph = symbol_inputs, symbol_outputs
        return self._cached_graph

    def _build_cache(self, *args, update_graph=True):
        data, out = self._get_graph(*args)
        data_names = {data.name: i for i, data in enumerate(data)}
        params = {p.var().name: p for p in self.collect_params().values()}
        param_serialization_names = {p.var().name: n for n, p in self.collect_params().items()}
        param_names = set(params.keys())
        input_names = out.list_inputs()
        expected_names = set(input_names)
        for name in expected_names:
            assert name in param_names or name in data_names, \
                f"Unknown input to HybridBlock: {name}"

        used_data_names = [i for i in data_names if i in expected_names]
        if len(used_data_names) != len(data_names):
            unused = ', '.join([f'{i}-th' for name, i in data_names.items()
                                if name not in expected_names])
            warnings.warn(f"The {unused} input to HybridBlock is not used by "
                          "any computation. Is this intended?", stacklevel=4)

        used_param_names = [i for i in param_names if i in expected_names]
        if len(used_param_names) != len(param_names):
            unused = ', '.join(list(param_names - set(used_param_names)))
            warnings.warn(f"Parameter {unused} is not used by any computation. "
                          "Is this intended?", stacklevel=4)

        args, _ = _flatten(args, "input")
        try:
            for name in input_names:
                if name in params:
                    params[name].data()
        except DeferredInitializationError:
            self._deferred_infer_shape(*args)
            for name in input_names:
                if name in params:
                    params[name]._finish_deferred_init()

        arg_dict, aux_dict = dict(), dict()
        if self._backend:
            # set device for inputs
            _, _, device_set, _ = _gather_type_device_info(list(args))
            device = device_set.pop() if len(device_set) > 0 else None
            # get list of params in the order of out.list_arguments
            input_shapes = dict()
            for name in out.list_arguments():
                if name in data_names.keys() and data_names[name] < len(args):
                    if isinstance(args[data_names[name]], NDArray):
                        arg_dict[name] = args[data_names[name]]
                    elif (isinstance(args[data_names[name]], symbol.Symbol) and
                          '__shape__' in args[data_names[name]].list_attr()):
                        shape_str = args[data_names[name]].list_attr()['__shape__']
                        input_shapes[name] = tuple(map(int, shape_str.strip('()').split(',')))
                elif name in params:
                    arg_dict[name] = params[name].data()

            for name in out.list_auxiliary_states():
                if name in data_names.keys() and data_names[name] < len(args):
                    if isinstance(args[data_names[name]], NDArray):
                        aux_dict[name] = args[data_names[name]]
                    elif (isinstance(args[data_names[name]], symbol.Symbol) and
                          '__shape__' in args[data_names[name]].list_attr()):
                        shape_str = args[data_names[name]].list_attr()['__shape__']
                        input_shapes[name] = tuple(map(int, shape_str.strip('()').split(',')))
                elif name in params:
                    aux_dict[name] = params[name].data()

            # Partition the graph
            out = out.optimize_for(self._backend, arg_dict, aux_dict, device, input_shapes, **self._backend_opts)

            #update cached graph with partitioned graph
            if update_graph:
                self._cached_graph = data, out

        input_names = out.list_inputs()
        data_indices = []
        param_indices = []

        # In the default case, _cached_ops_args contains all the parameters from params (the sets are identical)
        # In the case of Partition API optimized graph _cached_ops_args might contain some parameters from params,
        # might contain some new parameters created during optimization and added to `arg_dict/aux_dict`,
        # and might not contain some parameters that were deleted during optimization.
        self._cached_op_args = []
        for i, name in enumerate(input_names):
            triple = None
            if name in data_names:
                data_indices.append(i)
                triple = (True, name, data_names[name])
            else:
                param_indices.append(i)
                if name in params:
                    param = params[name]
                    serialization_name = param_serialization_names[name]  # HybridBlock.export
                else:
                    # The param is missing from the original params dictionary, which means the param must have
                    # been added by the Partition API backend
                    if name in arg_dict or name:
                        param_data = arg_dict[name]
                    elif name in aux_dict:
                        param_data = aux_dict[name]
                    else:
                        raise RuntimeError('A parameter was added to the graph during optimization but it was not '
                                           'added to the parameter dicts.\n'
                                           'Please check the backend.')

                    param = Parameter(name, dtype=param_data.dtype)
                    param._var_name = name
                    serialization_name = name  # HybridBlock.export
                    param._load_init(param_data, param_data.device)
                triple = (False, serialization_name, param)

            self._cached_op_args.append(triple)

        for i in range(len(self._flags) - 1, -1, -1):
            kv = self._flags[i]
            if kv[0] in ['data_indices', 'param_indices']:
                self._flags.remove(kv)
        self._flags = [('data_indices', data_indices), ('param_indices', param_indices)] + self._flags
        self._cached_op = ndarray.CachedOp(out, self._flags)

    def _deferred_infer_shape(self, *args):
        try:
            self.infer_shape(*args)
        except Exception as e:
            error_msg = "Deferred initialization failed because shape"\
                        " cannot be inferred. {}".format(e)
            raise ValueError(error_msg)

    def _call_cached_op(self, *args):
        if self._cached_op is None:
            self._build_cache(*args)

        if self._first_forward and self._partition_if_dynamic:
            self._first_forward = False
            # partition static shape ops if the graph contains any dynamic shape op
            _, out = self._cached_graph
            is_dynamic = out.has_dynamic_shape_op()
            if is_dynamic:
                self._backend = 'static_shape'
                self._backend_opts = {k : v for k, v in self._flags}
                self._build_cache(*args, update_graph=False)

        assert self._cached_op, "Gluon failed to build the cache. " \
                                "This should never happen. " \
                                "Please submit an issue on Github" \
                                " https://github.com/apache/mxnet."
        if self._callback:
            self._cached_op._register_op_hook(self._callback, self._monitor_all)
            if len(self._flags) >= 2 and (self._flags[1] or self._flags[0]):
                warnings.warn("register_op_hook is experimental when static_alloc=True / static_shape=True "
                              " and may not work correctly")

        args, fmt = _flatten(args, "input")
        if fmt != self._in_format:
            # Do not raise in the case that the fmt or stored_fmt ends with None and
            # We are relying on the default values.
            if len(self._in_format) > len(fmt):
                valid = all([self._in_format[i] == -1
                             for i in range(len(fmt), len(self._in_format))])
                valid = valid and (fmt == self._in_format[:len(fmt)])
            elif len(self._in_format) < len(fmt):
                valid = all([fmt[i] == -1
                             for i in range(len(self._in_format), len(fmt))])
                valid = valid and (fmt[:len(self._in_format)] == self._in_format)
            else:
                valid = False
            if not valid:
                raise ValueError("The argument structure of HybridBlock does not match"
                                 " the cached version. Stored format = {}, input format = {}"
                                 .format(fmt, self._in_format))

        args_without_none = [ele for ele in args if ele is not None]
        cargs = [args_without_none[i] if is_arg else i.data()
                 for is_arg, name, i in self._cached_op_args]
        out = self._cached_op(*cargs)
        if isinstance(out, NDArray):
            out = [out]
        return _regroup(out, self._out_format)

    def optimize_for(self, x, *args, backend=None, clear=False,
                     partition_if_dynamic=True,
                     static_alloc=False,
                     static_shape=False,
                     inline_limit=2,
                     forward_bulk_size=None,
                     backward_bulk_size=None,
                     **kwargs):
        """Partitions the current HybridBlock and optimizes it for a given backend
        without executing a forward pass. Modifies the HybridBlock in-place.

        Immediately partitions a HybridBlock using the specified backend. Combines
        the work done in the hybridize API with part of the work done in the forward
        pass without calling the CachedOp. Can be used in place of hybridize,
        afterwards `export` can be called or inference can be run. See README.md in
        example/extensions/lib_subgraph/README.md for more details.

        Examples
        --------
        # partition and then export to file
        block.optimize_for(x, backend='myPart')
        block.export('partitioned')

        # partition and then run inference
        block.optimize_for(x, backend='myPart')
        block(x)

        Parameters
        ----------
        x : NDArray
            first input to model
        *args : NDArray
            other inputs to model
        backend : str
            The name of backend, as registered in `SubgraphBackendRegistry`, default None
        backend_opts : dict of user-specified options to pass to the backend for partitioning, optional
            Passed on to `PrePartition` and `PostPartition` functions of `SubgraphProperty`
        clear : bool, default False
            clears any previous optimizations
        partition_if_dynamic : bool, default False
            whether to partition the graph when dynamic shape op exists
        static_alloc : bool, default False
            Statically allocate memory to improve speed. Memory usage may increase.
        static_shape : bool, default False
            Optimize for invariant input shapes between iterations. Must also
            set static_alloc to True. Change of input shapes is still allowed
            but slower.
        inline_limit : optional int, default 2
            Maximum number of operators that can be inlined.
        forward_bulk_size : optional int, default None
            Segment size of bulk execution during forward pass.
        backward_bulk_size : optional int, default None
            Segment size of bulk execution during backward pass.
        **kwargs: The backend options, optional
            Passed on to `PrePartition` and `PostPartition` functions of `SubgraphProperty`
        """
        self._backend = backend
        if len(kwargs) > 0:
            self._backend_opts = kwargs

        if clear or not self._active:
            self.hybridize(True, partition_if_dynamic, static_alloc, static_shape,
                           inline_limit, forward_bulk_size, backward_bulk_size)

        # do part of forward API call
        has_symbol, has_ndarray, device_set, _ = _gather_type_device_info([x] + list(args))
        if not has_symbol and not has_ndarray:
            raise ValueError('In HybridBlock, there must be one NDArray or one Symbol in the input.'
                             ' Please check the type of the args.\n')
        if len(device_set) > 1:
            raise ValueError('Found multiple devices in the input, '
                             'After hybridized, the HybridBlock only supports one input '
                             'device. You can print the ele.device in the '
                             'input arguments to inspect their devices. '
                             'Find all devices = {}'.format(device_set))

        self._build_cache(x, *args)
        assert self._cached_op, "Gluon failed to build the cache. " \
                                "This should never happen. " \
                                "Please submit an issue on Github" \
                                " https://github.com/apache/mxnet."
        # do not actually call the cached_op

        self._first_forward = True
        # clear the backend
        self._backend = None
        self._backend_opts = {}

    def _clear_cached_op(self):
        self._cached_graph = ()
        self._cached_op = None
        self._first_forward = True

    def register_child(self, block, name=None):
        if not isinstance(block, HybridBlock):
            raise ValueError(
                "Children of HybridBlock must also be HybridBlock, " \
                f"but {str(block)} has type {str(type(block))}. If you are using Sequential, " \
                "please try HybridSequential instead.")
        super(HybridBlock, self).register_child(block, name)
        if self._active:
            warnings.warn("Currently the model has been hybridized. Automatically deactivate the hybridization \
                           when adding new children block.")
            self._active = False
        self._clear_cached_op()

    def hybridize(self, active=True,
                  partition_if_dynamic=True,
                  static_alloc=False,
                  static_shape=False,
                  inline_limit=2,
                  forward_bulk_size=None,
                  backward_bulk_size=None):
        """Activates or deactivates :py:class:`HybridBlock` s recursively. Has no effect on
        non-hybrid children.

        Parameters
        ----------
        active : bool, default True
            Whether to turn hybrid on or off.
        partition_if_dynamic : bool, default False
            whether to partition the graph when dynamic shape op exists
        static_alloc : bool, default False
            Statically allocate memory to improve speed. Memory usage may increase.
        static_shape : bool, default False
            Optimize for invariant input shapes between iterations. Must also
            set static_alloc to True. Change of input shapes is still allowed
            but slower.
        inline_limit : optional int, default 2
            Maximum number of operators that can be inlined.
        forward_bulk_size : optional int, default None
            Segment size of bulk execution during forward pass.
        backward_bulk_size : optional int, default None
            Segment size of bulk execution during backward pass.
        """

        self._active = active
        self._partition_if_dynamic = partition_if_dynamic
        self._flags = [("static_alloc", static_alloc), ("static_shape", static_shape),
                       ("inline_limit", inline_limit)]
        if forward_bulk_size is not None:
            self._flags.append(("forward_bulk_size", forward_bulk_size))
        if backward_bulk_size is not None:
            self._flags.append(("backward_bulk_size", backward_bulk_size))
        self._clear_cached_op()
        if active and self._forward_hooks or self._forward_pre_hooks:
            warnings.warn('"{block}" is being hybridized while still having forward hook/pre-hook. '
                          'If "{block}" is a child of HybridBlock, the hooks will not take effect.'
                          .format(block=self))
        super(HybridBlock, self).hybridize(active,
                                           static_alloc=static_alloc,
                                           static_shape=static_shape,
                                           inline_limit=inline_limit,
                                           forward_bulk_size=forward_bulk_size,
                                           backward_bulk_size=backward_bulk_size)

    def cast(self, dtype):
        if self._active:
            warnings.warn("Currently the model has been hybridized. Automatically deactivate the hybridization \
                           when cast the block to use another data type.")
            self._active = False
        self._clear_cached_op()
        super(HybridBlock, self).cast(dtype)

    def _infer_attrs(self, infer_fn, attr, *args):
        """Generic infer attributes."""
        inputs, out = self._get_graph(*args)
        args, _ = _flatten(args, "input")
        args_without_none = [ele for ele in args if ele is not None]
        with warnings.catch_warnings(record=True) as w:
            arg_attrs, _, aux_attrs = getattr(out, infer_fn)(
                **{i.name: getattr(j, attr) for i, j in zip(inputs, args_without_none)})
            if arg_attrs is None:
                raise ValueError(w[0].message)
        sdict = {i: j for i, j in zip(out.list_arguments(), arg_attrs)}
        sdict.update({name : attr for name, attr in \
             zip(out.list_auxiliary_states(), aux_attrs)})
        for i in self.collect_params().values():
            setattr(i, attr, sdict[i.var().name])

    def infer_shape(self, *args):
        """Infers shape of Parameters from inputs."""
        # pylint: disable=unused-argument
        # In Gluon 2, users must implement infer_shape, if any deferred
        # initialized parameters are associated with the HybridBlock
        params = [p for p in self._reg_params.values() if not shape_is_known(p.shape)]
        if params:
            params_str = ", ".join("{} ({})".format(p.name, p.shape) for p in params)
            raise RuntimeError(
                "{name} has parameters with unknown shape. You need to either specify the shape "
                "in __init__ or implement {name}.infer_shape to set the parameter shapes "
                "based on the first input. Parameters with unknown shapes are {params}".format(
                    name=type(self).__name__, params=params_str))

    def infer_type(self, *args):
        """Infers data type of Parameters from inputs."""
        self._infer_attrs('infer_type', 'dtype', *args)

    def export(self, path, epoch=0, remove_amp_cast=True):
        """Export HybridBlock to json format that can be loaded by
        `gluon.SymbolBlock.imports` or the C++ interface.

        .. note:: When there are only one input, it will have name `data`. When there
                  Are more than one inputs, they will be named as `data0`, `data1`, etc.

        Parameters
        ----------
        path : str or None
            Path to save model. Two files `path-symbol.json` and `path-xxxx.params`
            will be created, where xxxx is the 4 digits epoch number.
            If None, do not export to file but return Python Symbol object and
            corresponding dictionary of parameters.
        epoch : int
            Epoch number of saved model.
        remove_amp_cast : bool, optional
            Whether to remove the amp_cast and amp_multicast operators, before saving the model.

        Returns
        -------
        symbol_filename : str
            Filename to which model symbols were saved, including `path` prefix.
        params_filename : str
            Filename to which model parameters were saved, including `path` prefix.
        """
        if not self._cached_graph:
            raise RuntimeError(
                "Please first call block.hybridize() and then run forward with "
                "this block at least once before calling export.")
        sym = copy.copy(self._cached_graph[1])

        # Deduplicate params (shared parameters use the same input symbol)
        reverse_params = {v: k for k, v in self.collect_params().items()}
        params = {v: k for k, v in reverse_params.items()}

        # In export we have global information on the structure of the graph
        # can rename the symbol inputs to human-readable, deterministic names.
        # That's not true in general, which is why internally random unique identifiers are used.
        rename_map = {param.var().name: name for name, param in params.items()}
        for var in sym.get_inputs():
            if var.name in rename_map:
                var._set_attr(name=rename_map[var.name])
        
        path_string = path if path is not None else ""
        sym_filename = f'{path_string}-symbol.json'
        if path is not None:
            sym.save(sym_filename, remove_amp_cast=remove_amp_cast)

        arg_names = set(sym.list_arguments())
        aux_names = set(sym.list_auxiliary_states())
        arg_dict = {}
        for is_arg, name, param in self._cached_op_args:
            if not is_arg:
                if name in arg_names:
                    arg_dict['arg:{}'.format(name)] = param._reduce()
                else:
                    if name not in aux_names:
                        warnings.warn('Parameter "{name}" is not found in the graph. '
                                      .format(name=name), stacklevel=3)
                    else:
                        arg_dict[f'aux:{name}'] = param._reduce()
        params_filename = f'{path_string}-{epoch:04d}.params'

        if path is not None:
            if is_np_array():
                _mx_npx.savez(params_filename, **arg_dict)
            else:
                ndarray.save(params_filename, arg_dict)
            return (sym_filename, params_filename if arg_dict else None)

        if remove_amp_cast:
            handle = SymbolHandle()
            check_call(_LIB.MXSymbolRemoveAmpCast(sym.handle, ctypes.byref(handle)))
            sym = type(sym)(handle)
        return sym, arg_dict

    def register_op_hook(self, callback, monitor_all=False):
        """Install op hook for block recursively.

        Parameters
        ----------
        callback : function
            Function called to inspect the values of the intermediate outputs
            of blocks after hybridization. It takes 3 parameters:
            name of the tensor being inspected (str)
            name of the operator producing or consuming that tensor (str)
            tensor being inspected (NDArray).
        monitor_all : bool, default False
            If True, monitor both input and output, otherwise monitor output only.
        """
        def c_callback(name, op_name, array):
            """wrapper for user callback"""
            array = ctypes.cast(array, NDArrayHandle)
            array = NDArray(array, writable=False)
            name = py_str(name)
            op_name = py_str(op_name)
            callback(name, op_name, array)

        self._callback = c_callback
        self._monitor_all = monitor_all
        for cld in self._children.values():
            cld()._callback = c_callback
            cld()._monitor_all = monitor_all

    def __call__(self, x, *args):
        _check_block_input_np_ndarrays([x, *args])
        assert self.forward is not HybridBlock.forward, (
            'Must define {name}.forward. '
            'Defining {name}.hybrid_forward is deprecated.'.format(name=type(self).__name__))

        _, has_ndarray, device_set, first_device = _gather_type_device_info([x] + list(args))
        if not has_ndarray:
            raise ValueError('In HybridBlock, there must be one NDArray in the input.'
                             ' Please check the type of the args.\n')
        if self._active and not dc.is_deferred_compute():
            # Do not call CachedOp if not hybridized or inside deferred compute mode.
            if len(device_set) > 1:
                raise ValueError('Find multiple devices in the input, '
                                 'After hybridized, the HybridBlock only supports one input '
                                 'device. You can print the ele.device in the '
                                 'input arguments to inspect their devices. '
                                 'Find all devices = {}'.format(device_set))

        if not self._called_infer_shape_already:
            self.infer_shape(x, *args)
            for p in self._reg_params.values():
                p._finish_deferred_init()
            self._called_infer_shape_already = True

        if not self._active:
            # Normal imperative computation of forward()
            return super().__call__(x, *args)

        if dc.is_deferred_compute():
            # Deferred compute is already enabled. This typically means that the current
            # HybridBlock is a child block of a HybridBlock that has been hybridized.
            return super().__call__(x, *args)

        with first_device:
            return self._call_cached_op(x, *args)

    def forward(self, x, *args):
        """Overrides the forward computation. Arguments must be
        :py:class:`mxnet.numpy.ndarray`."""

        raise NotImplementedError

    def reset_device(self, device):
        """Re-assign all Parameters to other devices. If the Block is hybridized, it will reset the _cached_op_args.

        Parameters
        ----------
        device : Device or list of Device, default :py:meth:`device.current_device()`.
            Assign Parameter to given device. If device is a list of Device, a
            copy will be made for each device.
        """
        params = self.collect_params()
        if self._cached_op:
            for p in self._cached_op_args:
                # resetting parameters creating by the partitioning backend
                if p.name not in params:
                    p.reset_device(device)
        for p in params.values():
            p.reset_device(device)

    def reset_ctx(self, ctx):
        """This function has been deprecated. Please refer to ``HybridBlock.reset_device``."""
        warnings.warn('HybridBlock.reset_ctx has been renamed to'
                      ' HybridBlock.reset_device', DeprecationWarning)
        self.reset_device(ctx)


class SymbolBlock(HybridBlock):
    """Construct block from symbol. This is useful for using pre-trained models
    as feature extractors. For example, you may want to extract the output
    from fc2 layer in AlexNet.

    Parameters
    ----------
    outputs : Symbol or list of Symbol
        The desired output for SymbolBlock.
    inputs : Symbol or list of Symbol
        The Variables in output's argument that should be used as inputs.
    params : dict
        Parameter dictionary for arguments and auxililary states of outputs
        that are not inputs.

    Examples
    --------
    >>> # To extract the feature from fc1 and fc2 layers of AlexNet:
    >>> alexnet = gluon.model_zoo.vision.alexnet(pretrained=True, device=mx.cpu())
    >>> inputs = mx.sym.var('data')
    >>> out = alexnet(inputs)
    >>> internals = out.get_internals()
    >>> print(internals.list_outputs())
    ['data', ..., 'features_9_act_fwd_output', ..., 'features_11_act_fwd_output', ...]
    >>> outputs = [internals['features_9_act_fwd_output'],
                   internals['features_11_act_fwd_output']]
    >>> # Create SymbolBlock that shares parameters with alexnet
    >>> feat_model = gluon.SymbolBlock(outputs, inputs, params=alexnet.collect_params())
    >>> x = mx.nd.random.normal(shape=(16, 3, 224, 224))
    >>> print(feat_model(x))
    """
    @staticmethod
    @wrap_ctx_to_device_func
    def imports(symbol_file, input_names, param_file=None, device=None, allow_missing=False,
                ignore_extra=False):
        """Import model previously saved by `gluon.HybridBlock.export`
        as a `gluon.SymbolBlock` for use in Gluon.

        Parameters
        ----------
        symbol_file : str
            Path to symbol file.
        input_names : list of str
            List of input variable names
        param_file : str, optional
            Path to parameter file.
        device : Device, default None
            The device to initialize `gluon.SymbolBlock` on.
        allow_missing : bool, default False
            Whether to silently skip loading parameters not represents in the file.
        ignore_extra : bool, default False
            Whether to silently ignore parameters from the file that are not
            present in this Block.

        Returns
        -------
        gluon.SymbolBlock
            `gluon.SymbolBlock` loaded from symbol and parameter files.

        Examples
        --------
        >>> net1 = gluon.model_zoo.vision.resnet18_v1(pretrained=True)
        >>> net1.hybridize()
        >>> x = mx.nd.random.normal(shape=(1, 3, 32, 32))
        >>> out1 = net1(x)
        >>> net1.export('net1', epoch=1)
        >>>
        >>> net2 = gluon.SymbolBlock.imports(
        ...     'net1-symbol.json', ['data'], 'net1-0001.params')
        >>> out2 = net2(x)
        """
        if is_np_array():
            sym = np_symbol.load(symbol_file)
        else:
            sym = symbol.load(symbol_file)
        if isinstance(input_names, str):
            input_names = [input_names]
        if param_file is None:
            # Get a valid type inference by using fp32
            inputs = [symbol.var(i, dtype=mx_real_t) for i in input_names]
        else:
            # Do not specify type, rely on saved params type instead
            inputs = [symbol.var(i).as_np_ndarray() if is_np_array() else symbol.var(i) for i in input_names]
        ret = SymbolBlock(sym, inputs)
        if param_file is not None:
            ret.load_parameters(param_file, device, allow_missing, ignore_extra, True, 'saved')
        return ret

    def __repr__(self):
        s = '{name}(\n{modstr}\n)'
        modstr = '\n'.join(['{block} : {numinputs} -> {numoutputs}'.format(block=self._cached_graph[1],
                                                                           numinputs=len(self._cached_graph[0]),
                                                                           numoutputs=len(self._cached_graph[1].
                                                                                          list_outputs()))])
        return s.format(name=self.__class__.__name__,
                        modstr=modstr)

    def __init__(self, outputs, inputs, params=None):
        super(SymbolBlock, self).__init__()

        if isinstance(inputs, symbol.Symbol) and len(inputs.list_outputs()) == 1:
            inputs = [inputs]
        if isinstance(outputs, (list, tuple)) and len(outputs) == 1:
            outputs = outputs[0]

        syms, self._in_format = _flatten(inputs, "input")
        out, self._out_format = _flatten(outputs, "output")
        input_names = set()
        for i in syms:
            assert len(i.get_internals().list_outputs()) == 1, \
                f"Input symbols must be variable, but {str(i)} is an output of operators"
            input_names.add(i.name)

        # check if any symbol is row_sparse
        row_sparse_storage = ndarray.ndarray._STORAGE_TYPE_STR_TO_ID['row_sparse']

        for i in out:
            for j in i.get_internals():
                assert(j.attr("__storage_type__") != str(row_sparse_storage)), \
                    f"SymbolBlock doesn't support Parameter '{j.name}' because its storage " \
                    "type is 'row_sparse'."
        if len(out) > 1:
            out = symbol.Group(out, _check_same_symbol_type(out))
        else:
            out = out[0]

        # Infer type of parameters. Without this, every parameter will be created with
        # default type i.e., fp32
        arg_params = out.list_arguments()
        aux_params = out.list_auxiliary_states()

        arg_types, aux_types = _infer_param_types(syms, out, arg_params, aux_params)

        if params is None:
            params = {}
        unused_params = set(params.keys()) - set(arg_params) - set(aux_params)
        if len(unused_params) > 0:
            raise ValueError('{} params are unused by the model.'.format(unused_params))
        self._reg_params = params

        for i, arg in enumerate(arg_params):
            if arg in self._reg_params:
                self._reg_params[arg]._check_and_setattr(allow_deferred_init=True, dtype=arg_types[i])
                if self._reg_params[arg]._var is None:
                    self._reg_params[arg]._var_name = arg
            elif arg not in input_names:
                self._reg_params[arg] = Parameter(name=arg, allow_deferred_init=True, dtype=arg_types[i])
                self._reg_params[arg]._var_name = arg
        for i, aux in enumerate(aux_params):
            if aux in self._reg_params:
                self._reg_params[aux]._check_and_setattr(grad_req='null', allow_deferred_init=True,
                                                         dtype=aux_types[i])
                if self._reg_params[aux]._var is None:
                    self._reg_params[aux]._var_name = aux
            elif aux not in input_names:
                self._reg_params[aux] = Parameter(name=aux, grad_req='null',
                                                  allow_deferred_init=True, dtype=aux_types[i])
                self._reg_params[aux]._var_name = aux

        self._cached_graph = syms, out

    def infer_shape(self, *args):
        """Infers shape of Parameters from inputs."""
        self._infer_attrs('infer_shape', 'shape', *args)

    def __call__(self, x, *args):
        """Calls forward. Only accepts positional arguments."""
        for hook in self._forward_pre_hooks.values():
            hook(self, [x, *args])

        out = self.forward(x, *args)

        for hook in self._forward_hooks.values():
            hook(self, [x, *args], out)

        return out

    def forward(self, x, *args):
        if dc.is_deferred_compute():
            raise RuntimeError('Calling a SymbolBlock from within HybridBlock '
                               'is not yet supported in Gluon 2.')

        if isinstance(x, NDArray):
            with x.device:
                return self._call_cached_op(x, *args)

        assert isinstance(x, Symbol), \
            "HybridBlock requires the first argument to forward be either " \
            f"Symbol or NDArray, but got {type(x)}"
        args, in_fmt = _flatten([x] + list(args), "input")
        assert in_fmt == self._in_format, "Invalid input format"
        ret = copy.copy(self._cached_graph[1])
        ret._compose(**{k.name: v for k, v in zip(self._cached_graph[0], args)})
        return _regroup(list(ret), self._out_format)

    def _clear_cached_op(self):
        tmp = self._cached_graph
        super(SymbolBlock, self)._clear_cached_op()
        self._cached_graph = tmp

    def cast(self, dtype):
        self._clear_cached_op()
        super(SymbolBlock, self).cast(dtype)
        if get_dtype_name(dtype) == 'float16':
            # correct BatchNorm types back to float32 due to its special requirement
            out = self._cached_graph[1]
            params_list = out.get_internals().list_inputs()
            for node in params_list:
                if node.endswith('running_var'):
                    prefix = node[:-11]
                    sibs = [prefix + t for t in ('running_mean', 'gamma', 'beta')]
                    is_bn = all(p in params_list for p in sibs)
                    if is_bn:
                        self.params.get(node).cast('float32')
                        for sib in sibs:
                            self.params.get(sib).cast('float32')
                if node.endswith('moving_var'):
                    # another convention used
                    prefix = node[:-10]
                    sibs = [prefix + t for t in ('moving_mean', 'gamma', 'beta')]
                    is_bn = all(p in params_list for p in sibs)
                    if is_bn:
                        self.params.get(node).cast('float32')
                        for sib in sibs:
                            self.params.get(sib).cast('float32')

def _infer_param_types(in_params, out_params, arg_params, aux_params, default_dtype=mx_real_t):
    """Utility function that helps in inferring DType of args and auxs params
    from given input param.

    Parameters
    ----------
    in_params: List of Symbol
        List of input symbol variables.
    out_params: Symbol
        Output symbol variable.
    arg_params: List of Str
        List of names of argument parametrs.
    aux_params: List of Str
        List of names of auxiliary parameters.
    default_dtype: numpy.dtype or str, default 'float32'
        Default data type for arg_params and aux_params, if unable to infer the type.

    Returns
    -------
    arg_types: List of numpy.dtype
        List of arg_params type. Order is same as arg_params.
        Defaults to 'float32', if unable to infer type.
    aux_types: List of numpy.dtype
        List of aux_params type. Order is same as aux_params.
        Defaults to 'float32', if unable to infer type.
    """
    arg_types = None
    aux_types = None

    # Get Input symbol details. This will be used to infer types of
    # other parameters.
    input_sym_names = [in_param.name for in_param in in_params]

    # Try to infer input types. If not successful, we will set default dtype.
    # If successful, we will try to infer other params in the graph.
    input_sym_arg_types = []
    can_infer_input_type = True
    for in_param in in_params:
        input_sym_arg_type = in_param.infer_type()[0]
        if not input_sym_arg_type or len(input_sym_arg_type) < 1:
            can_infer_input_type = False
            break
        else:
            input_sym_arg_types.append(in_param.infer_type()[0][0])

    # Try to infer types of other parameters.
    if can_infer_input_type:
        params = {k:v for k, v in zip(input_sym_names, input_sym_arg_types)}
        try:
            arg_types, _, aux_types = out_params.infer_type(**params)
        except MXNetError:
            # Cannot infer type with current input
            arg_types, aux_types = None, None

    if arg_types is None or len(arg_types) != len(arg_params):
        arg_types = []
        for _ in arg_params:
            arg_types.append(default_dtype)

    if aux_types is None or len(aux_types) != len(aux_params):
        aux_types = []
        for _ in aux_params:
            aux_types.append(default_dtype)

    return (arg_types, aux_types)


def set_optimization_constraints(state):
    prev_state = ctypes.c_uint()
    check_call(_LIB.MXSetOptimizationConstraints(ctypes.c_uint(state.value), ctypes.byref(prev_state)))
    return HybridBlock.OptConstraint.Flag(prev_state.value)


def get_optimization_constraints():
    curr = ctypes.c_uint()
    check_call(_LIB.MXGetOptimizationConstraints(ctypes.byref(curr)))
    return HybridBlock.OptConstraint.Flag(curr.value)