Multi gpu training fails using strategy='ddp'

[dudeperf3ct]

Running video classification using flash

Linked issue: #1188

Code Sample

import os
from argparse import ArgumentParser
import torch
print(f"Is cuda available? {torch.cuda.is_available()}")
print(f"Number of gpus detected: {torch.cuda.device_count()}")
from torch.utils.data.sampler import RandomSampler

import flash
from flash.core.finetuning import NoFreeze
from flash.core.data.utils import download_data
from flash.video import VideoClassificationData, VideoClassifier

if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument('--seed', type=int, default=1234)
    parser.add_argument('--backbone', type=str, default="x3d_xs")
    parser.add_argument('--download', type=bool, default=True)
    parser.add_argument('--train_folder', type=str, default=os.path.join(os.getcwd(),
                        "./data/kinetics/train"))
    parser.add_argument('--val_folder', type=str, default=os.path.join(os.getcwd(),
                        "./data/kinetics/val"))
    parser.add_argument('--predict_folder', type=str, default=os.path.join(os.getcwd(),
                        "./data/kinetics/predict"))
    parser.add_argument('--max_epochs', type=int, default=1)
    parser.add_argument('--learning_rate', type=float, default=1e-3)
    parser.add_argument('--gpus', type=int, default=None)
    parser.add_argument('--fast_dev_run', type=int, default=False)
    args = parser.parse_args()


    if args.download:
        # Dataset Credit:Download a video clip dataset.
        # Find more datasets at https://pytorchvideo.readthedocs.io/en/latest/data.html
        download_data("https://pl-flash-data.s3.amazonaws.com/kinetics.zip",
                      os.path.join(os.getcwd(), "data/"))

    datamodule = VideoClassificationData.from_folders(
        train_folder=args.train_folder,
        val_folder=args.val_folder,
        predict_folder=args.predict_folder,
        batch_size=8,
        clip_sampler="uniform",
        clip_duration=2,
        video_sampler=RandomSampler,
        decode_audio=False,
        num_workers=8,
    )

    model = VideoClassifier(backbone=args.backbone, num_classes=datamodule.num_classes, pretrained=False)

    trainer = flash.Trainer(max_epochs=args.max_epochs, gpus=args.gpus, strategy='ddp', fast_dev_run=args.fast_dev_run)
    trainer.finetune(model, datamodule=datamodule, strategy=NoFreeze())

Is cuda available? True
Number of gpus detected: 8
Using 'x3d_xs' provided by Facebook Research/PyTorchVideo (https://github.com/facebookresearch/pytorchvideo).                                                                          
GPU available: True, used: True                                                                                                                                                        
TPU available: False, using: 0 TPU cores                                                                                                                                               
IPU available: False, using: 0 IPUs
/opt/conda/lib/python3.8/site-packages/torch/cuda/__init__.py:52: UserWarning: CUDA initialization: CUDA driver initialization failed, you might not have a CUDA gpu. (Triggered intern
ally at  ../c10/cuda/CUDAFunctions.cpp:115.)
  return torch._C._cuda_getDeviceCount() > 0
 Traceback (most recent call last):                                                                                                                                                     
  File "/workspace/test_train.py", line 61, in <module>                                                                                                                                
    trainer = flash.Trainer(max_epochs=args.max_epochs, gpus=args.gpus, strategy='ddp', fast_dev_run=args.fast_dev_run)                                                                
  File "/opt/conda/lib/python3.8/site-packages/flash/core/trainer.py", line 71, in insert_env_defaults                                                                                 
    return fn(self, **kwargs)                                                                                                                                                          
  File "/opt/conda/lib/python3.8/site-packages/flash/core/trainer.py", line 91, in __init__                                                                                            
    super().__init__(*args, **kwargs)                                                                                                                                                  
  File "/opt/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/env_vars_connector.py", line 38, in insert_env_defaults                                            
    return fn(self, **kwargs)                                                                                                                                                          
  File "/opt/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 426, in __init__                                                                            
    gpu_ids, tpu_cores = self._parse_devices(gpus, auto_select_gpus, tpu_cores)                                                                                                        
  File "/opt/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1543, in _parse_devices                                                                     
    gpu_ids = device_parser.parse_gpu_ids(gpus)                                                                                                                                        
  File "/opt/conda/lib/python3.8/site-packages/pytorch_lightning/utilities/device_parser.py", line 89, in parse_gpu_ids                                                                
    return _sanitize_gpu_ids(gpus)                                                                                                                                                     
  File "/opt/conda/lib/python3.8/site-packages/pytorch_lightning/utilities/device_parser.py", line 151, in _sanitize_gpu_ids                                                           
    raise MisconfigurationException(                                                                                                                                                   
pytorch_lightning.utilities.exceptions.MisconfigurationException: You requested GPUs: [0, 1, 2, 3, 4, 5, 6, 7]                                                                         
 But your machine only has: []

I ran python -c 'import torch; print(torch.cuda.is_available())' and pytorch is able to detect all gpus and cuda is available at start of the script. Only when i run using flash trainer using ddp strategy, I get the above error.

The script runs fine when using 1 gpu.

Is cuda available? True
Number of gpus detected: 8
Using 'x3d_xs' provided by Facebook Research/PyTorchVideo (https://github.com/facebookresearch/pytorchvideo).
GPU available: True, used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/1
----------------------------------------------------------------------------------------------------
distributed_backend=nccl
All distributed processes registered. Starting with 1 processes
----------------------------------------------------------------------------------------------------

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]

Configurations:

I am running pytorch lightning ngc with --gpus all and --shm-size=1g flags
pytorch/lightning/flash : 1.9.0a0/1.5.10/0.8.0dev

nvcc -V
nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2021 NVIDIA Corporation
Built on Mon_May__3_19:15:13_PDT_2021
Cuda compilation tools, release 11.3, V11.3.109
Build cuda_11.3.r11.3/compiler.29920130_0

I have 8x V100 with driver version 418.67 and cuda version 10.1.

[dudeperf3ct]

@rohitgr7 I was suggested to ask the question in lightning forum as the issue comes from PL library instead of Flash.

Hey @dudeperf3ct sorry it still isn't working for you. Could you open an issue for that one on the PL repo? All device handling is managed by PL so they should be able to provide better support 😃

Originally posted by @ethanwharris in #1188 (comment)

[ethanwharris]

Yeah, @rohitgr7 I have transferred this back to the PL repo. For context, this is working within our CI with 2 GPUs using DDP. So I believe that the issue must be either related to environment issues or coming from PL 😃

@dudeperf3ct Just to check, could you try running the MNIST hello world example with the same config (8 GPUs, DDP) to confirm that you still get the same error?

[dudeperf3ct]

@ethanwharris I confirm that I can run mnist example using strategy='ddp' successfully

import os

import torch
from pytorch_lightning import LightningModule, Trainer
from torch import nn
from torch.nn import functional as F
from torch.utils.data import DataLoader, random_split
from torchmetrics import Accuracy
from torchvision import transforms
from torchvision.datasets import MNIST

PATH_DATASETS = os.environ.get("PATH_DATASETS", ".")
AVAIL_GPUS = max(1, torch.cuda.device_count())
BATCH_SIZE = 256 if AVAIL_GPUS else 64

class MNISTModel(LightningModule):
    def __init__(self):
        super().__init__()
        self.l1 = torch.nn.Linear(28 * 28, 10)

    def forward(self, x):
        return torch.relu(self.l1(x.view(x.size(0), -1)))

    def training_step(self, batch, batch_nb):
        x, y = batch
        loss = F.cross_entropy(self(x), y)
        return loss

    def configure_optimizers(self):
        return torch.optim.Adam(self.parameters(), lr=0.02)

mnist_model = MNISTModel()

# Init DataLoader from MNIST Dataset
train_ds = MNIST(PATH_DATASETS, train=True, download=True, transform=transforms.ToTensor())
train_loader = DataLoader(train_ds, batch_size=BATCH_SIZE)

# Initialize a trainer
trainer = Trainer(
    gpus=AVAIL_GPUS,
    max_epochs=3,
    progress_bar_refresh_rate=20,
    strategy='ddp'
)

# Train the model ⚡
trainer.fit(mnist_model, train_loader)

[ethanwharris]

Transferring back to Flash in that case 😆 Could you try running your Flash script without the if __name__ == '__main__': and also add a print statement for args.gpus? The Flash Trainer is a very thin wrapper for the PL trainer, so it's quite strange that it trips up in the Flash example and not in the PL example... so trying to find what changed between them to cause the error

[dudeperf3ct]

Sure thing!

I ran a minimal version of the video classification example. Faced with same error.

from torch.utils.data.sampler import RandomSampler

import flash
from flash.core.finetuning import NoFreeze
from flash.video import VideoClassificationData, VideoClassifier

datamodule = VideoClassificationData.from_folders(
    train_folder="./data/kinetics/train",
    val_folder="./data/kinetics/val",
    predict_folder="./data/kinetics/predict",
    batch_size=8,
    clip_sampler="uniform",
    clip_duration=2,
    video_sampler=RandomSampler,
    decode_audio=False,
    num_workers=2,
)

model = VideoClassifier(backbone="x3d_xs", num_classes=datamodule.num_classes, pretrained=False)

trainer = flash.Trainer(max_epochs=2, gpus=8, strategy='ddp', fast_dev_run=False)
trainer.finetune(model, datamodule=datamodule, strategy=NoFreeze())

[ethanwharris]

Hey @dudeperf3ct Just ran the video classification example on a machine with 8 GPUs and it is working fine for me, so it must be an environement issue. Some things you could check:

• make sure that CUDA_VISIBLE_DEVICES environment variable isn't set or is set to [0,1,2,3,4,5,6,7]
• you could try reinstalling Torch / Flash / PL

Unfortunately as I can't reproduce it there isn't much else I can suggest, but I'll let you know if anything else comes to mind 😃

[rohitgr7]

what's the output for torch.cuda.device_count()??

[dudeperf3ct]

torch.cuda.device_count(): 8

[rohitgr7]

the error seems to be coming from _sanitize_gpu_ids which just uses torch.cuda.device_count() internally. Are you sure you are checking it in the same env where you are running the script. Can you print it before running trainer.fit?

[dudeperf3ct]

FYI, the mnist example with trainer.fit() and strategy='ddp' is working fine. The error comes when running a video classification using flash.

I ran by adding print statement before running flash.Trainer for this code. The torch is able to detect the gpus before Trainer initializes but fails after Trainer initialization.

Using 'x3d_xs' provided by Facebook Research/PyTorchVideo (https://github.com/facebookresearch/pytorchvideo).                                                                          
Is cuda available? True                                                                                                                                                                
Number of gpus detected: 8                                                                                                                                                             
GPU available: True, used: True                                                                                                                                                        
TPU available: False, using: 0 TPU cores                                                                                                                                               
IPU available: False, using: 0 IPUs                                                                                                                                                    
/opt/conda/lib/python3.8/site-packages/torch/cuda/__init__.py:52: UserWarning: CUDA initialization: CUDA driver initialization failed, you might not have a CUDA gpu. (Triggered intern
ally at  ../c10/cuda/CUDAFunctions.cpp:115.)                                                                                                                                           
  return torch._C._cuda_getDeviceCount() > 0                                                                                                                                           
Is cuda available? False                                                                                                                                                               
Number of gpus detected: 0                                                                                                                                                             
Traceback (most recent call last):                                                                                                                                                     
  File "/workspace/test_train.py", line 64, in <module>                                                                                                                                
    trainer = flash.Trainer(max_epochs=2, gpus=8, strategy='ddp', fast_dev_run=False)                                                                                                  
  File "/opt/conda/lib/python3.8/site-packages/flash/core/trainer.py", line 71, in insert_env_defaults                                                                                 
    return fn(self, **kwargs)                                                                                                                                                          
  File "/opt/conda/lib/python3.8/site-packages/flash/core/trainer.py", line 91, in __init__                                                                                            
    super().__init__(*args, **kwargs)                                                                                                                                                  
  File "/opt/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/env_vars_connector.py", line 38, in insert_env_defaults                                            
    return fn(self, **kwargs)                                                                                                                                                          
  File "/opt/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 426, in __init__                                                                            
    gpu_ids, tpu_cores = self._parse_devices(gpus, auto_select_gpus, tpu_cores)                                                                                                        
  File "/opt/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1543, in _parse_devices                                                                     
    gpu_ids = device_parser.parse_gpu_ids(gpus)                                                                                                                                        
  File "/opt/conda/lib/python3.8/site-packages/pytorch_lightning/utilities/device_parser.py", line 89, in parse_gpu_ids                                                                
    return _sanitize_gpu_ids(gpus)                                                                                                                                                     
  File "/opt/conda/lib/python3.8/site-packages/pytorch_lightning/utilities/device_parser.py", line 151, in _sanitize_gpu_ids                                                           
    raise MisconfigurationException(                                                                                                                                                   
pytorch_lightning.utilities.exceptions.MisconfigurationException: You requested GPUs: [0, 1, 2, 3, 4, 5, 6, 7]                                                                         
 But your machine only has: []