From 6d993f3dbd3858629f3e6b8612e0977c4fab49ea Mon Sep 17 00:00:00 2001 From: Dirk Groeneveld Date: Mon, 26 Feb 2024 16:58:25 -0800 Subject: [PATCH] Try the `spawn` start method --- olmo/data/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/olmo/data/__init__.py b/olmo/data/__init__.py index 52421b57a..3ebc1102b 100644 --- a/olmo/data/__init__.py +++ b/olmo/data/__init__.py @@ -1,5 +1,6 @@ from pathlib import Path from typing import Any, Dict, List, Optional, cast +import multiprocessing as mp from torch.utils.data import DataLoader, DistributedSampler @@ -50,6 +51,7 @@ def build_eval_dataloader( batch_size: int, shuffle: bool = True, ) -> DataLoader: + mp.set_start_method('spawn') dataset = build_memmap_dataset(train_config, data_config, include_instance_metadata=True) collator = DataCollator(pad_direction=data_config.pad_direction, pad_token_id=train_config.model.pad_token_id) if data_config.drop_last: @@ -80,6 +82,7 @@ def build_eval_dataloader( def build_train_dataloader(train_config: TrainConfig) -> DataLoader: assert train_config.device_train_batch_size is not None + mp.set_start_method('spawn') collator = DataCollator( pad_direction=train_config.data.pad_direction, pad_token_id=train_config.model.pad_token_id )