parameter.py

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# coding: utf-8
# pylint: disable=unnecessary-pass, too-many-lines
"""Neural network parameter."""

__all__ = ['DeferredInitializationError', 'Parameter', 'Constant',
           'tensor_types']


import uuid
import warnings
import weakref
import numpy as np

from ..base import mx_real_t, MXNetError
from .. import symbol, ndarray, initializer, device as _device, _deferred_compute as dc
from ..device import Device, cpu
from .. import autograd
from .utils import shape_is_known
from ..util import is_np_shape, is_np_array, wrap_ctx_to_device_func
from .. import numpy as _mx_np  # pylint: disable=reimported

# pylint: disable= invalid-name
tensor_types = (symbol.Symbol, ndarray.NDArray)
# pylint: enable= invalid-name

class DeferredInitializationError(MXNetError):
    """Error for unfinished deferred initialization."""
    pass

class Parameter(object):
    """A Container holding parameters (weights) of Blocks.

    :py:class:`Parameter` holds a copy of the parameter on each :py:class:`Device` after
    it is initialized with ``Parameter.initialize(...)``. If :py:attr:`grad_req` is
    not ``'null'``, it will also hold a gradient array on each :py:class:`Device`::

        device = mx.gpu(0)
        x = mx.np.zeros((16, 100), device=device)
        w = mx.gluon.Parameter('fc_weight', shape=(64, 100), init=mx.init.Xavier())
        b = mx.gluon.Parameter('fc_bias', shape=(64,), init=mx.init.Zero())
        w.initialize(device=device)
        b.initialize(device=device)
        out = mx.npx.fully_connected(x, w.data(device), b.data(device), num_hidden=64)

    Parameters
    ----------
    name : str, default 'weight'
        Name of this parameter. It decides the corresponding default initializer.
    grad_req : {'write', 'add', 'null'}, default 'write'
        Specifies how to update gradient to grad arrays.

        - ``'write'`` means everytime gradient is written to grad :py:class:`NDArray`.
        - ``'add'`` means everytime gradient is added to the grad :py:class:`NDArray`. You need
          to manually call ``zero_grad()`` to clear the gradient buffer before each
          iteration when using this option.
        - 'null' means gradient is not requested for this parameter. gradient arrays
          will not be allocated.
    shape : int or tuple of int, default None
        Shape of this parameter. By default shape is not specified. Parameter with
        unknown shape can be used for :py:class:`Symbol` API, but ``init`` will throw an error
        when using :py:class:`NDArray` API.
    dtype : numpy.dtype or str, default 'float32'
        Data type of this parameter. For example, ``numpy.float32`` or ``'float32'``.
    lr_mult : float, default 1.0
        Learning rate multiplier. Learning rate will be multiplied by lr_mult
        when updating this parameter with optimizer.
    wd_mult : float, default 1.0
        Weight decay multiplier (L2 regularizer coefficient). Works similar to lr_mult.
    init : Initializer, default None
        Initializer of this parameter. Will use the global initializer by default.
    stype: {'default', 'row_sparse', 'csr'}, defaults to 'default'.
        The storage type of the parameter.
    grad_stype: {'default', 'row_sparse', 'csr'}, defaults to 'default'.
        The storage type of the parameter's gradient.

    Attributes
    ----------
    grad_req : {'write', 'add', 'null'}
        This can be set before or after initialization. Setting ``grad_req`` to ``'null'``
        with ``x.grad_req = 'null'`` saves memory and computation when you don't
        need gradient w.r.t x.
    lr_mult : float
        Local learning rate multiplier for this Parameter. The actual learning rate
        is calculated with ``learning_rate * lr_mult``. You can set it with
        ``param.lr_mult = 2.0``
    wd_mult : float
        Local weight decay multiplier for this Parameter.
    """
    def __init__(self, name='weight', grad_req='write', shape=None, dtype=mx_real_t,
                 lr_mult=1.0, wd_mult=1.0, init=None, allow_deferred_init=False,
                 differentiable=True, stype='default', grad_stype='default'):
        self._var = None
        self._uuid = str(uuid.uuid4())
        self._var_name = None
        self._data = None
        self._grad = None
        self._device_list = None
        self._device_map = None
        self._trainer = None
        self._deferred_init = ()
        self._differentiable = differentiable
        self._allow_deferred_init = allow_deferred_init
        self._grad_req = None
        if isinstance(shape, int):
            shape = (shape,)
        self._shape = shape
        self._name = name
        self._dtype = dtype
        self.lr_mult = lr_mult
        self.wd_mult = wd_mult
        self.grad_req = grad_req
        self.init = init
        # sparse related storage type information
        valid_stypes = ['default', 'row_sparse', 'csr']
        assert grad_stype in valid_stypes, "grad_stype for Parameter must be " \
            f"one of 'default', 'row_sparse', or 'csr', but got '{grad_stype}'"
        assert stype in valid_stypes, "stype for Parameter must be " \
            f"one of 'default', 'row_sparse', or 'csr', but got '{stype}'"
        self._grad_stype = grad_stype
        self._stype = stype

    def __repr__(self):
        s = 'Parameter (shape={shape}, dtype={dtype})'
        return s.format(shape=self.shape, dtype=self.dtype)

    @property
    def grad_req(self):
        return self._grad_req

    @property
    def name(self):
        return self._name

    @grad_req.setter
    def grad_req(self, req):
        assert req in ['write', 'add', 'null'], \
            f"grad_req must be one of 'write', 'add', or 'null', but got '{req}'"
        if not self._differentiable:
            req = 'null'
        if self._grad_req == req:
            return
        self._grad_req = req
        if req == 'null' and self._grad is not None:
            self._grad = None
            self._data = [i.detach() for i in self._data]
        elif self._data is not None:
            self._init_grad()

    @property
    def dtype(self):
        """The type of the parameter.

        Setting the dtype value is equivalent to casting the value of the parameter
        """
        return self._dtype

    @dtype.setter
    def dtype(self, dtype):
        self.cast(dtype)

    @property
    def shape(self):
        """The shape of the parameter.

        By default, an unknown dimension size is 0. However, when the NumPy semantic
        is turned on, unknown dimension size is -1.
        """
        if self._shape is None:
            return None
        elif is_np_shape():
            # Parameters shouldn't be zero-size. If one of its dimension is 0,
            # it means the parameter isn't initialized. In the NumPy semantics,
            # the unknown dimension should be marked with -1.
            return tuple(i if i != 0 else -1 for i in self._shape)
        else:
            return self._shape

    @shape.setter
    def shape(self, new_shape):
        if self._shape is None:
            self._shape = new_shape
            return

        assert len(self._shape) == len(new_shape) and \
            all(j in (-1, 0, i) for i, j in zip(new_shape, self._shape)), \
            f"Expected shape {str(new_shape)} is incompatible with given shape {str(self._shape)} for Parameter {str(self.name)}." 
            # -1 means unknown dim size in np_shape mode

        self._shape = new_shape

    def _set_trainer(self, trainer):
        """ Set the trainer this parameter is associated with. """
        # trainer cannot be replaced for sparse params
        if self._stype != 'default' and self._trainer and trainer and self._trainer() is not trainer:
            raise RuntimeError(
                f"Failed to set the trainer for Parameter '{self.name}' because it was already set. " \
                f"More than one trainers for a {self._stype} Parameter is not supported.")
        if trainer is not None:
            self._trainer = weakref.ref(trainer)
        else:
            self._trainer = trainer

    def _check_and_get(self, arr_list, device):
        if arr_list is not None:
            if device is list:
                return arr_list
            if device is None:
                if len(arr_list) == 1:
                    return arr_list[0]
                else:
                    device = _device.current_device()
            device_list = self._device_map[device.device_typeid&1]
            if device.device_id < len(device_list):
                idx = device_list[device.device_id]
                if idx is not None:
                    return arr_list[idx]
            raise RuntimeError(
                f"Parameter '{self.name}' was not initialized on device {str(device)}. "
                f"It was only initialized on {str(self._device_list)}.")
        if self._deferred_init:
            raise DeferredInitializationError(
                f"Parameter '{self.name}' has not been initialized yet because initialization was " \
                "deferred. Actual initialization happens during the first forward pass. " \
                "Please pass one batch of data through the network before accessing Parameters. " \
                "You can also avoid deferred initialization by specifying in_units, " \
                "num_features, etc., for network layers.")
        raise RuntimeError(
            f"Parameter '{self.name}' has not been initialized. Note that " \
            "you should initialize parameters and create Trainer " \
            "with Block.collect_params() instead of Block.params " \
            "because the later does not include Parameters of " \
            "nested child Blocks")

    @wrap_ctx_to_device_func
    def _get_row_sparse(self, arr_list, device, row_id):
        """ Get row_sparse data from row_sparse parameters based on row_id. """
        # get row sparse params based on row ids
        if not isinstance(row_id, ndarray.NDArray):
            raise TypeError(f"row_id must have NDArray type, but {type(row_id)} is given")
        trainer = self._trainer() if self._trainer else None
        if not trainer:
            raise RuntimeError(f"Cannot get row_sparse data for Parameter '{self.name}' when no " \
                               "Trainer is created with it.")
        results = self._check_and_get(arr_list, device)

        # fetch row sparse params from the trainer
        trainer._row_sparse_pull(self, results, row_id)
        return results

    @wrap_ctx_to_device_func
    def _load_init(self, data, device, cast_dtype=False, dtype_source='current'):
        """
        (Re)initializes by loading from data.
        Parameters
        ----------
        data : NDArray
            The data to load
        device : Device or list of Device
            Device(s) initialize loaded parameters on.
        cast_dtype : bool, default False
            Cast the data type of the parameter
        dtype_source : str, default 'current'
            must be in {'current', 'saved'}
            Only valid if cast_dtype=True, specify the source of the dtype for casting
            the parameters
        """
        if cast_dtype:
            assert dtype_source in ['current', 'saved']
        if self.shape:
            unknown_dim_size = -1 if is_np_shape() else 0
            for self_dim, data_dim in zip(self.shape, data.shape):
                assert self_dim in (unknown_dim_size, data_dim), \
                    f"Failed loading Parameter '{self.name}' from saved params: " \
                    f"shape incompatible expected {str(self.shape)} vs saved {str(data.shape)}"
            self.shape = tuple(i if i != unknown_dim_size else j
                               for i, j in zip(self.shape, data.shape))
        if self.dtype:
            if cast_dtype and self.dtype != data.dtype:
                if dtype_source == 'current':
                    data = data.astype(self.dtype, copy=False)
                elif dtype_source == 'saved':
                    self.dtype = data.dtype
            else:
                assert self.dtype == data.dtype, \
                f"Failed loading Parameter '{self.name}' from saved params: " \
                f"dtype incompatible expected {str(self.dtype)} vs saved {str(data.dtype)}. " \
                "Set cast_dtype=True to cast the dtype of saved params."
        if self._stype != data.stype:
            data = data.tostype(self._stype)
        if isinstance(device, Device):
            device = [device]
        if self._data is None:
            if self._deferred_init:
                assert device is None or set(device) == set(self._deferred_init[1]), \
                    f"Failed to load Parameter '{self.name}' on {str(device)} because it was " \
                    f"previous initialized on {str(self.list_device())}."
                device = self._deferred_init[1]
            elif device is None:
                device = [cpu()]
            self._init_impl(data, device)
        else:
            assert device is None or set(device) == set(self.list_device()), \
                f"Failed to load Parameter '{self.name}' on {str(device)} because it was " \
                f"previous initialized on {str(self.list_device())}."
            self.set_data(data)
        self._deferred_init = ()

    def _finish_deferred_init(self):
        """Finishes deferred initialization."""
        if not self._deferred_init:
            return
        init, device, default_init, data = self._deferred_init
        self._deferred_init = ()

        assert shape_is_known(self.shape), \
            f"Cannot initialize Parameter '{self.name}' because it has " \
            f"invalid shape: {str(self.shape)}. Please specify in_units, " \
            "in_channels, etc for `Block`s."

        with autograd.pause(), dc.context(False):
            if data is None:
                if is_np_array():
                    kwargs = {'shape': self.shape, 'dtype': self.dtype, 'device': cpu()}
                    if self._stype != 'default':
                        raise ValueError("Currently stype {} is not supported in NumPy interface and Gluon2.0"
                                         .format(self._stype))
                    zeros_fn = _mx_np.zeros
                else:
                    kwargs = {'shape': self.shape, 'dtype': self.dtype, 'ctx': cpu()}
                    kwargs['stype'] = self._stype
                    zeros_fn = ndarray.zeros
                data = zeros_fn(**kwargs)
                initializer.create(default_init)(
                    initializer.InitDesc(self.name, {'__init__': init}), data)

            self._init_impl(data, device)

    def _init_impl(self, data, device_list):
        """Sets data and grad."""
        self._device_list = list(device_list)
        self._device_map = [[], []]
        for i, device in enumerate(self._device_list):
            dev_list = self._device_map[device.device_typeid&1]
            while len(dev_list) <= device.device_id:
                dev_list.append(None)
            dev_list[device.device_id] = i

        self._data = [data.copyto(device) for device in self._device_list]
        self._init_grad()

    def _init_grad(self):
        """Initialize grad buffers."""
        if self.grad_req == 'null':
            self._grad = None
            return

        if is_np_array():
            if self._grad_stype != 'default':
                raise ValueError("Currently stype {} is not supported in NumPy interface and Gluon2.0"
                                 .format(self._grad_stype))
            self._grad = [_mx_np.zeros(shape=i.shape, dtype=i.dtype, device=i.device)
                          for i in self._data]
        else:
            self._grad = [ndarray.zeros(shape=i.shape, dtype=i.dtype, ctx=i.context,
                                        stype=self._grad_stype) for i in self._data]

        autograd.mark_variables(self._check_and_get(self._data, list),
                                self._grad, self.grad_req)

    def _reduce(self):
        """Reduce data from multiple device to cpu."""
        device = cpu()
        if self._stype == 'default':
            block = self.list_data()
            if len(block) > 1:
                if is_np_array():
                    data = sum([w.copyto(device) for w in block]) / len(block)
                else:
                    data = ndarray.add_n(*(w.copyto(device) for w in block)) / len(block)
            else:
                data = self.data().copyto(device)
        else:
            # fetch all rows for 'row_sparse' param
            all_row_ids = ndarray.arange(0, self.shape[0], dtype='int64', ctx=device)
            data = ndarray.zeros(self.shape, stype='row_sparse', ctx=device)
            trainer = self._trainer() if self._trainer else None
            if not trainer:
                raise RuntimeError(f"Cannot reduce row_sparse data for Parameter '{self.name}' when no " \
                                   "Trainer is created with it.")
            trainer._row_sparse_pull(self, data, all_row_ids, full_idx=True)
        return data

    @wrap_ctx_to_device_func
    def initialize(self, init=None, device=None, default_init=initializer.Uniform(),
                   force_reinit=False):
        """Initializes parameter and gradient arrays. Only used for :py:class:`NDArray` API.

        Parameters
        ----------
        init : Initializer
            The initializer to use. Overrides :py:meth:`Parameter.init` and default_init.
        device : Device or list of Device, default :py:meth:`device.current_device()`.
            Assign Parameter to given device. If device is a list of Device, a
            copy will be made for each device.

            .. note::
                Copies are independent arrays. User is responsible for keeping
                their values consistent when updating.
                Normally :py:class:`gluon.Trainer` does this for you.

        default_init : Initializer
            Default initializer is used when both :py:func:`init`
            and :py:meth:`Parameter.init` are ``None``.
        force_reinit : bool, default False
            Whether to force re-initialization if parameter is already initialized.
        Examples
        --------
        >>> weight = mx.gluon.Parameter('weight', shape=(2, 2))
        >>> weight.initialize(device=mx.cpu(0))
        >>> weight.data()
        [[-0.01068833  0.01729892]
         [ 0.02042518 -0.01618656]]
        <NDArray 2x2 @cpu(0)>
        >>> weight.grad()
        [[ 0.  0.]
         [ 0.  0.]]
        <NDArray 2x2 @cpu(0)>
        >>> weight.initialize(device=[mx.gpu(0), mx.gpu(1)])
        >>> weight.data(mx.gpu(0))
        [[-0.00873779 -0.02834515]
         [ 0.05484822 -0.06206018]]
        <NDArray 2x2 @gpu(0)>
        >>> weight.data(mx.gpu(1))
        [[-0.00873779 -0.02834515]
         [ 0.05484822 -0.06206018]]
        <NDArray 2x2 @gpu(1)>
        """
        if self._data is not None and not force_reinit:
            warnings.warn(f"Parameter '{self.name}' is already initialized, ignoring. " \
                          "Set force_reinit=True to re-initialize.",
                          stacklevel=2)
            return
        self._data = self._grad = None
        if device is None:
            device = [_device.current_device()]
        if isinstance(device, Device):
            device = [device]
        if isinstance(self.init, initializer.RNNFused):
            self.init.set_initializer(init if init else default_init)
            init = default_init = self.init
        if init is None:
            init = default_init if self.init is None else self.init
        if not shape_is_known(self.shape):
            if self._allow_deferred_init:
                self._deferred_init = (init, device, default_init, None)
                return
            raise ValueError(f"Cannot initialize Parameter '{self.name}' because it has " \
                             f"invalid shape: {str(self.shape)}.")

        self._deferred_init = (init, device, default_init, None)
        self._finish_deferred_init()

    def reset_device(self, device):
        """Re-assign Parameter to other devices.

        Parameters
        ----------
        device : Device or list of Device, default ``device.current_device()``.
            Assign Parameter to given device. If device is a list of Device, a
            copy will be made for each device.
        """
        if device is None:
            device = [_device.current_device()]
        if isinstance(device, Device):
            device = [device]
        if self._data:
            data = self._reduce()
            with autograd.pause():
                self._init_impl(data, device)
        elif self._deferred_init:
            init, _, default_init, data = self._deferred_init
            self._deferred_init = (init, device, default_init, data)
        else:
            raise ValueError(f"Cannot reset device for Parameter '{self.name}' because it "
                             "has not been initialized.")

    def reset_ctx(self, ctx):
        """This function has been deprecated. Please refer to ``Parameter.reset_device``."""
        warnings.warn('Parameter.reset_ctx has been renamed to'
                      ' Parameter.reset_device', DeprecationWarning)
        self.reset_device(ctx)

    def set_data(self, data):
        """Sets this parameter's value on all devices."""
        self.shape = data.shape

        if self._data is None:
            assert self._deferred_init, \
                f"Parameter '{self.name}' has not been initialized"
            self._deferred_init = self._deferred_init[:3] + (data,)
            return

        # if update_on_kvstore, we need to make sure the copy stored in kvstore is in sync
        trainer = self._trainer() if self._trainer else None
        if trainer and trainer._kv_initialized and trainer._update_on_kvstore:
            if self not in trainer._params_to_init:
                trainer._reset_kvstore()

        for arr in self._check_and_get(self._data, list):
            arr[:] = data

    def row_sparse_data(self, row_id):
        """Returns a copy of the 'row_sparse' parameter on the same device as row_id's.
        The copy only retains rows whose ids occur in provided row ids.
        The parameter must have been initialized on this device before.

        Parameters
        ----------
        row_id: NDArray
            Row ids to retain for the 'row_sparse' parameter.

        Returns
        -------
        NDArray on row_id's device
        """
        if self._stype != 'row_sparse':
            raise RuntimeError(f"Cannot return a copy of Parameter {self.name} via row_sparse_data() " \
                               f"because its storage type is {self._stype}. Please use data() instead.")
        return self._get_row_sparse(self._data, row_id.device, row_id)

    def list_row_sparse_data(self, row_id):
        """Returns copies of the 'row_sparse' parameter on all devices, in the same order
        as creation. The copy only retains rows whose ids occur in provided row ids.
        The parameter must have been initialized before.

        Parameters
        ----------
        row_id: NDArray
            Row ids to retain for the 'row_sparse' parameter.

        Returns
        -------
        list of NDArrays
        """
        if self._stype != 'row_sparse':
            raise RuntimeError(f"Cannot return copies of Parameter '{self.name}' on all devices via " \
                               f"list_row_sparse_data() because its storage type is {self._stype}. Please " \
                               "use data() instead.")
        return self._get_row_sparse(self._data, list, row_id)

    @wrap_ctx_to_device_func
    def data(self, device=None):
        """Returns a copy of this parameter on one device. Must have been
        initialized on this device before. For sparse parameters, use
        :py:meth:`Parameter.row_sparse_data` instead.

        Parameters
        ----------
        device : Device
            Desired device.

        Returns
        -------
        NDArray on device
        """
        if self._stype != 'default':
            raise RuntimeError(f"Cannot return a copy of Parameter '{self.name}' on device {str(device)} via data() " \
                               f"because its storage type is {self._stype}. Please use row_sparse_data() instead.")
        data = self._check_and_get(self._data, device)
        dc.set_variable(data, self.var())
        return data

    def list_data(self):
        """Returns copies of this parameter on all devices, in the same order
        as creation. For sparse parameters, use :py:meth:`Parameter.list_row_sparse_data`
        instead.

        Returns
        -------
        list of NDArrays
        """
        if self._stype != 'default':
            raise RuntimeError(f"Cannot return copies of Parameter '{self.name}' on all devices via " \
                               f"list_data() because its storage type is {self._stype}. Please use " \
                               "row_sparse_data() instead.")
        return self._check_and_get(self._data, list)

    def grad(self, device=None):
        """Returns a gradient buffer for this parameter on one device.

        Parameters
        ----------
        device : Device
            Desired device.
        """
        if self._data is not None and self._grad is None:
            raise RuntimeError(
                f"Cannot get gradient array for Parameter '{self.name}' " \
                "because grad_req='null'")
        return self._check_and_get(self._grad, device)

    def list_grad(self):
        """Returns gradient buffers on all devices, in the same order
        as :py:meth:`values`."""
        if self._data is not None and self._grad is None:
            raise RuntimeError(
                f"Cannot get gradient array for Parameter '{self.name}' " \
                "because grad_req='null'")
        return self._check_and_get(self._grad, list)

    def list_ctx(self):
        """This function has been deprecated. Please refer to ``Parameter.list_device``."""
        warnings.warn('Parameter.list_ctx has been renamed to'
                      ' Parameter.list_device', DeprecationWarning)
        return self.list_device()

    def list_device(self):
        """Returns a list of devices this parameter is initialized on."""
        if self._data is None:
            if self._deferred_init:
                return self._deferred_init[1]
            raise RuntimeError(f"Parameter '{self.name}' has not been initialized")
        return self._device_list

    def zero_grad(self):
        """Sets gradient buffer on all devices to 0. No action is taken if
        parameter is uninitialized or doesn't require gradient."""
        if self._grad is None:
            return
        for i in self._grad:
            ndarray.zeros_like(i, out=i)

    def var(self):
        """Returns a symbol representing this parameter."""
        if self._var is None:
            if self._var_name is None:  # _var_name is set manually in SymbolBlock.import
                # The variable name is required by the storage profiler.
                self._var_name = self._uuid.replace('-', '_') + '_' + self._name
            self._var = symbol.var(self._var_name, shape=self.shape, dtype=self.dtype,
                                   lr_mult=self.lr_mult, wd_mult=self.wd_mult,
                                   init=self.init, stype=self._stype)
            if is_np_array():
                self._var = self._var.as_np_ndarray()
        return self._var

    def cast(self, dtype):
        """Cast data and gradient of this Parameter to a new data type.

        Parameters
        ----------
        dtype : str or numpy.dtype
            The new data type.
        """
        self._dtype = dtype
        self._var = None  # Clear Symbol Variable as it caches the dtype
        if self._data is None:
            return
        with autograd.pause():
            self._data = [i.astype(dtype) for i in self._data]
            if self._grad is None:
                return
            self._grad = [i.astype(dtype) for i in self._grad]
            autograd.mark_variables(self._data, self._grad, self.grad_req)

    def _check_and_setattr(self, **kwargs):
        """check and set attributes for parameter"""
        for k, v in kwargs.items():
            if hasattr(self, k) and getattr(self, k) is not None:
                existing = getattr(self, k)
                if k == 'shape' and len(v) == len(existing):
                    inferred_shape = []
                    matched = True
                    for dim1, dim2 in zip(v, existing):
                        if dim1 != dim2 and dim1 > 0 and dim2 > 0:
                            matched = False
                            break
                        elif dim1 == dim2:
                            inferred_shape.append(dim1)
                        elif dim1 in (0, -1):  # -1 means unknown dim size in np_shape mode
                            inferred_shape.append(dim2)
                        else:
                            inferred_shape.append(dim1)

                    if matched:
                        self._shape = tuple(inferred_shape)
                        continue
                elif k == 'dtype' and np.dtype(v) == np.dtype(existing):
                    continue

                assert v is None or v == existing, \
                    f"Cannot retrieve Parameter '{self.name}' because desired attribute " \
                    f"does not match with stored for attribute '{k}': " \
                    f"desired '{str(v)}' vs stored '{str(getattr(self, k))}'."
            else:
                setattr(self, k, v)

class Constant(Parameter):
    """A constant parameter for holding immutable tensors.
    `Constant`s are ignored by `autograd` and `Trainer`, thus their values
    will not change during training. But you can still update their values
    manually with the `set_data` method.

    `Constant` s can be created with either::

        const = mx.gluon.Constant([[1,2],[3,4]])

    or::

        class Block(gluon.Block):
            def __init__(self, **kwargs):
                super(Block, self).__init__(**kwargs)
                self.const = mx.gluon.Constant([[1,2],[3,4]])

    Parameters
    ----------
    value : array-like
        Initial value for the constant.
    """
    def __init__(self, value):
        if not isinstance(value, ndarray.NDArray):
            array_fn = _mx_np.array if is_np_array() else ndarray.array
            value = array_fn(value)
        self.value = value

        class Init(initializer.Initializer):
            def _init_weight(self, _, arr):
                value.copyto(arr)
        init_name = 'Constant_{}'.format(id(self))
        initializer.alias(init_name)(Init)

        super(Constant, self).__init__(
            name='const', grad_req='null', shape=value.shape, dtype=value.dtype,
            init=init_name)

    def __repr__(self):
        s = 'Constant (shape={shape}, dtype={dtype})'
        return s.format(shape=self.shape, dtype=self.dtype)

    @property
    def grad_req(self):
        return 'null'

    @grad_req.setter
    def grad_req(self, req):
        if req != 'null':
            warnings.warn('Constant parameter "{}" does not support '
                          'grad_req other than "null", and new value "{}" '
                          'is ignored.'.format(self.name, req))