From a67069bac63c977fb1b91de781e695f959485a84 Mon Sep 17 00:00:00 2001 From: ooooo <3164076421@qq.com> Date: Tue, 2 Jul 2024 23:53:27 +0800 Subject: [PATCH 1/4] [Typing][A-100] Add type annotations for python/paddle/io/dataloader/worker.py --- python/paddle/io/dataloader/worker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/paddle/io/dataloader/worker.py b/python/paddle/io/dataloader/worker.py index a559a616bb2963..e4d5cdbb09545b 100644 --- a/python/paddle/io/dataloader/worker.py +++ b/python/paddle/io/dataloader/worker.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations import os import queue @@ -76,7 +77,7 @@ def is_alive(self): _worker_info = None -def get_worker_info(): +def get_worker_info() -> WorkerInfo | None: """ Get DataLoader worker process information function, this function is used to split data copy in worker process for IterableDataset From d3aec45bf44b8bf077aabeeeaf004ed112a1793e Mon Sep 17 00:00:00 2001 From: ooooo <3164076421@qq.com> Date: Wed, 3 Jul 2024 09:44:01 +0800 Subject: [PATCH 2/4] add # type: ignore --- python/paddle/io/dataloader/worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/io/dataloader/worker.py b/python/paddle/io/dataloader/worker.py index e4d5cdbb09545b..65ad7f66bdf334 100644 --- a/python/paddle/io/dataloader/worker.py +++ b/python/paddle/io/dataloader/worker.py @@ -116,10 +116,10 @@ def get_worker_info() -> WorkerInfo | None: ... iter_start = self.start ... iter_end = self.end ... else: - ... per_worker = int( + ... per_worker = int( # type: ignore ... math.ceil((self.end - self.start) / float( ... worker_info.num_workers))) - ... worker_id = worker_info.id + ... worker_id = worker_info.id # type: ignore ... iter_start = self.start + worker_id * per_worker ... iter_end = min(iter_start + per_worker, self.end) ... From 69ba4cf62278d2ca3bd5f779661265ebecd28012 Mon Sep 17 00:00:00 2001 From: ooooo <3164076421@qq.com> Date: Wed, 3 Jul 2024 10:17:48 +0800 Subject: [PATCH 3/4] fix --- python/paddle/io/dataloader/worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/io/dataloader/worker.py b/python/paddle/io/dataloader/worker.py index 65ad7f66bdf334..63ac4efd143873 100644 --- a/python/paddle/io/dataloader/worker.py +++ b/python/paddle/io/dataloader/worker.py @@ -116,9 +116,9 @@ def get_worker_info() -> WorkerInfo | None: ... iter_start = self.start ... iter_end = self.end ... else: - ... per_worker = int( # type: ignore + ... per_worker = int( ... math.ceil((self.end - self.start) / float( - ... worker_info.num_workers))) + ... worker_info.num_workers))) # type: ignore ... worker_id = worker_info.id # type: ignore ... iter_start = self.start + worker_id * per_worker ... iter_end = min(iter_start + per_worker, self.end) From 9aa8fd6acbc33390fc6305601221e9981422eeeb Mon Sep 17 00:00:00 2001 From: ooooo <3164076421@qq.com> Date: Wed, 3 Jul 2024 10:26:52 +0800 Subject: [PATCH 4/4] update --- python/paddle/io/dataloader/worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/io/dataloader/worker.py b/python/paddle/io/dataloader/worker.py index 63ac4efd143873..b1284a646f656c 100644 --- a/python/paddle/io/dataloader/worker.py +++ b/python/paddle/io/dataloader/worker.py @@ -118,8 +118,8 @@ def get_worker_info() -> WorkerInfo | None: ... else: ... per_worker = int( ... math.ceil((self.end - self.start) / float( - ... worker_info.num_workers))) # type: ignore - ... worker_id = worker_info.id # type: ignore + ... worker_info.num_workers))) # type: ignore[attr-defined] + ... worker_id = worker_info.id # type: ignore[attr-defined] ... iter_start = self.start + worker_id * per_worker ... iter_end = min(iter_start + per_worker, self.end) ...