From 9cbdad31b0c034889d2291793a4c1376776edd8e Mon Sep 17 00:00:00 2001 From: Kirill Suvorov Date: Fri, 10 Nov 2023 14:56:29 +0000 Subject: [PATCH] REFACTOR-#384: Fix docs building. Signed-off-by: Kirill Suvorov --- .../unidist/core/backends/mpi/core/common.rst | 6 +-- .../core/backends/mpi/core/controller.rst | 2 +- .../core/backends/mpi/core/monitor.rst | 4 +- docs/optimization_notes/index.rst | 9 +++++ .../core/backends/mpi/core/communication.py | 38 +++++++++---------- 5 files changed, 34 insertions(+), 25 deletions(-) diff --git a/docs/flow/unidist/core/backends/mpi/core/common.rst b/docs/flow/unidist/core/backends/mpi/core/common.rst index 233c38a6..b08e5a44 100644 --- a/docs/flow/unidist/core/backends/mpi/core/common.rst +++ b/docs/flow/unidist/core/backends/mpi/core/common.rst @@ -16,11 +16,11 @@ API .. autoclass:: unidist.core.backends.mpi.core.common.Operation :members: -.. autoclass:: unidist.core.backends.mpi.core.common.MasterDataID +.. autoclass:: unidist.core.backends.mpi.core.common.MpiDataID :members: .. autofunction:: unidist.core.backends.mpi.core.common.get_logger .. autofunction:: unidist.core.backends.mpi.core.common.unwrapped_data_ids_list -.. autofunction:: unidist.core.backends.mpi.core.common.master_data_ids_to_base -.. autofunction:: unidist.core.backends.mpi.core.common.unwrap_data_ids .. autofunction:: unidist.core.backends.mpi.core.common.materialize_data_ids +.. autofunction:: unidist.core.backends.mpi.core.common.check_mpich_version +.. autofunction:: unidist.core.backends.mpi.core.common.is_shared_memory_supported diff --git a/docs/flow/unidist/core/backends/mpi/core/controller.rst b/docs/flow/unidist/core/backends/mpi/core/controller.rst index 502e77aa..6be3ed93 100644 --- a/docs/flow/unidist/core/backends/mpi/core/controller.rst +++ b/docs/flow/unidist/core/backends/mpi/core/controller.rst @@ -30,7 +30,7 @@ Both of the functions block execution until read/write finishes. .. autofunction:: unidist.core.backends.mpi.core.controller.api.put :py:func:`~unidist.core.backends.mpi.core.controller.api.wait` carries out blocking of execution -until a requested number of :py:class:`~unidist.core.backends.mpi.core.common.MasterDataID` isn't ready. +until a requested number of :py:class:`~unidist.core.backends.mpi.core.common.MpiDataID` isn't ready. .. autofunction:: unidist.core.backends.mpi.core.controller.api.wait diff --git a/docs/flow/unidist/core/backends/mpi/core/monitor.rst b/docs/flow/unidist/core/backends/mpi/core/monitor.rst index 835413f9..4c9e0f8e 100644 --- a/docs/flow/unidist/core/backends/mpi/core/monitor.rst +++ b/docs/flow/unidist/core/backends/mpi/core/monitor.rst @@ -8,7 +8,7 @@ Monitor related functionality API """"""""""""""""""""""""""""""""" -Infinite :py:func:`~unidist.core.backends.mpi.core.monitor.monitor_loop` function tracks +Infinite :py:func:`~unidist.core.backends.mpi.core.monitor.loop.monitor_loop` function tracks MPI backend statistics: executed tasks counter. API @@ -16,4 +16,4 @@ API Cancel operation from :py:class:`~unidist.core.backends.mpi.core.common.Operations` class breaks the loop. -.. autofunction:: unidist.core.backends.mpi.core.monitor.monitor_loop +.. autofunction:: unidist.core.backends.mpi.core.monitor.loop.monitor_loop diff --git a/docs/optimization_notes/index.rst b/docs/optimization_notes/index.rst index 24726be8..54930b14 100644 --- a/docs/optimization_notes/index.rst +++ b/docs/optimization_notes/index.rst @@ -8,6 +8,15 @@ Optimization Notes This page contains optimization notes for every backend that can be applied to it to get ultimate performance. +.. toctree:: + :hidden: + + mpi + dask + ray + pymp + pyseq + - :doc:`MPI ` - :doc:`Dask ` - :doc:`Ray ` diff --git a/unidist/core/backends/mpi/core/communication.py b/unidist/core/backends/mpi/core/communication.py index 2ed24c86..ceb0e094 100755 --- a/unidist/core/backends/mpi/core/communication.py +++ b/unidist/core/backends/mpi/core/communication.py @@ -284,8 +284,8 @@ def mpi_send_operation(comm, op_type, dest_rank): Notes ----- * This blocking send is used when we have to wait for completion of the communication, - which is necessary for the pipeline to continue, or when the receiver is waiting for a result. - Otherwise, use non-blocking ``mpi_isend_operation``. + which is necessary for the pipeline to continue, or when the receiver is waiting for a result. + Otherwise, use non-blocking ``mpi_isend_operation``. * The special tag is used for this communication, namely, ``common.MPITag.OPERATION``. """ comm.send(op_type, dest=dest_rank, tag=common.MPITag.OPERATION) @@ -309,8 +309,8 @@ def mpi_send_object(comm, data, dest_rank, tag=common.MPITag.OBJECT): Notes ----- * This blocking send is used when we have to wait for completion of the communication, - which is necessary for the pipeline to continue, or when the receiver is waiting for a result. - Otherwise, use non-blocking ``mpi_isend_object``. + which is necessary for the pipeline to continue, or when the receiver is waiting for a result. + Otherwise, use non-blocking ``mpi_isend_object``. * The special tag is used for this communication, namely, ``common.MPITag.OBJECT``. """ comm.send(data, dest=dest_rank, tag=tag) @@ -361,7 +361,7 @@ def mpi_isend_object(comm, data, dest_rank): Notes ----- - * The special tag is used for this communication, namely, ``common.MPITag.OBJECT``. + The special tag is used for this communication, namely, ``common.MPITag.OBJECT``. """ return comm.isend(data, dest=dest_rank, tag=common.MPITag.OBJECT) @@ -475,10 +475,10 @@ def mpi_send_buffer(comm, buffer, dest_rank, data_type=MPI.CHAR, buffer_size=Non Notes ----- * This blocking send is used when we have to wait for completion of the communication, - which is necessary for the pipeline to continue, or when the receiver is waiting for a result. - Otherwise, use non-blocking ``mpi_isend_buffer``. + which is necessary for the pipeline to continue, or when the receiver is waiting for a result. + Otherwise, use non-blocking ``mpi_isend_buffer``. * The special tags are used for this communication, namely, - ``common.MPITag.OBJECT`` and ``common.MPITag.BUFFER``. + ``common.MPITag.OBJECT`` and ``common.MPITag.BUFFER``. """ if buffer_size: comm.send(buffer_size, dest=dest_rank, tag=common.MPITag.OBJECT) @@ -521,7 +521,7 @@ def mpi_isend_buffer(comm, buffer_size, buffer, dest_rank): Notes ----- - * The special tags are used for this communication, namely, + The special tags are used for this communication, namely, ``common.MPITag.OBJECT`` and ``common.MPITag.BUFFER``. """ requests = [] @@ -565,7 +565,7 @@ def mpi_recv_buffer(comm, source_rank, result_buffer=None): Notes ----- - * The special tags are used for this communication, namely, + The special tags are used for this communication, namely, ``common.MPITag.OBJECT`` and ``common.MPITag.BUFFER``. """ if result_buffer is None: @@ -677,7 +677,7 @@ def send_complex_data(comm, data, dest_rank, is_serialized=False): Notes ----- - * This blocking send is used when we have to wait for completion of the communication, + This blocking send is used when we have to wait for completion of the communication, which is necessary for the pipeline to continue, or when the receiver is waiting for a result. Otherwise, use non-blocking ``isend_complex_data``. """ @@ -732,7 +732,7 @@ def _isend_complex_data_impl(comm, s_data, raw_buffers, dest_rank, info_package) Notes ----- * The special tags are used for this communication, namely, - ``common.MPITag.OBJECT`` and ``common.MPITag.BUFFER``. + ``common.MPITag.OBJECT`` and ``common.MPITag.BUFFER``. """ handlers = [] # wrap to dict for sending and correct deserialization of the object by the recipient @@ -778,7 +778,7 @@ def isend_complex_data(comm, data, dest_rank, is_serialized=False): Notes ----- - * The special tags are used for this communication, namely, + The special tags are used for this communication, namely, ``common.MPITag.OBJECT`` and ``common.MPITag.BUFFER``. """ if is_serialized: @@ -829,7 +829,7 @@ def recv_complex_data(comm, source_rank, info_package): Notes ----- - * The special tags are used for this communication, namely, + The special tags are used for this communication, namely, ``common.MPITag.OBJECT`` and ``common.MPITag.BUFFER``. """ msgpack_buffer = bytearray(info_package["s_data_len"]) @@ -866,11 +866,11 @@ def send_simple_operation(comm, operation_type, operation_data, dest_rank): Notes ----- * This blocking send is used when we have to wait for completion of the communication, - which is necessary for the pipeline to continue, or when the receiver is waiting for a result. - Otherwise, use non-blocking ``isend_simple_operation``. + which is necessary for the pipeline to continue, or when the receiver is waiting for a result. + Otherwise, use non-blocking ``isend_simple_operation``. * Serialization of the data to be sent takes place just using ``pickle.dump`` in this case. * The special tags are used for this communication, namely, - ``common.MPITag.OPERATION`` and ``common.MPITag.OBJECT``. + ``common.MPITag.OPERATION`` and ``common.MPITag.OBJECT``. """ # Send operation type mpi_send_operation(comm, operation_type, dest_rank) @@ -902,7 +902,7 @@ def isend_simple_operation(comm, operation_type, operation_data, dest_rank): ----- * Serialization of the data to be sent takes place just using ``pickle.dump`` in this case. * The special tags are used for this communication, namely, - ``common.MPITag.OPERATION`` and ``common.MPITag.OBJECT``. + ``common.MPITag.OPERATION`` and ``common.MPITag.OBJECT``. """ # Send operation type handlers = [] @@ -948,7 +948,7 @@ def isend_complex_operation( Notes ----- - * The special tags are used for this communication, namely, + The special tags are used for this communication, namely, ``common.MPITag.OPERATION``, ``common.MPITag.OBJECT`` and ``common.MPITag.BUFFER``. """ # Send operation type