diff --git a/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile b/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile
index 4012b30a8509..b5ffa409809a 100644
--- a/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile
+++ b/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile
@@ -22,6 +22,9 @@ RUN python3 -m pip install --no-cache-dir -U torch==$PYTORCH torchvision torchau
 
 RUN python3 -m pip install --no-cache-dir ./transformers[deepspeed-testing]
 
+# The base image ships with `torch_tensorrt` which currently causes the DeepSpeed CI fail from test collection
+RUN python3 -m pip uninstall -y torch_tensorrt
+
 # Pre-build **latest** DeepSpeed, so it would be ready for testing (otherwise, the 1st deepspeed test will timeout)
 RUN python3 -m pip uninstall -y deepspeed
 # This has to be run (again) inside the GPU VMs running the tests.