diff --git a/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile b/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile
index b5ffa409809a..0b3e9d67375f 100644
--- a/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile
+++ b/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile
@@ -22,8 +22,7 @@ RUN python3 -m pip install --no-cache-dir -U torch==$PYTORCH torchvision torchau
 
 RUN python3 -m pip install --no-cache-dir ./transformers[deepspeed-testing]
 
-# The base image ships with `torch_tensorrt` which currently causes the DeepSpeed CI fail from test collection
-RUN python3 -m pip uninstall -y torch_tensorrt
+RUN python3 -m pip install torch-tensorrt==1.3.0 --find-links https://github.com/pytorch/TensorRT/releases/expanded_assets/v1.3.0
 
 # Pre-build **latest** DeepSpeed, so it would be ready for testing (otherwise, the 1st deepspeed test will timeout)
 RUN python3 -m pip uninstall -y deepspeed