From 78372608184176ac62b78e287ce61cae9316eb8d Mon Sep 17 00:00:00 2001
From: "Jona T. Feucht" <14951074+jonafeucht@users.noreply.github.com>
Date: Fri, 8 Nov 2024 18:34:16 +0100
Subject: [PATCH] feat: OpenCL AMD GPUs

---
 .github/workflows/dev.opencl.yml  |  4 +--
 Dockerfile.opencl.amd             | 50 +++++++++++++++++++++----------
 README.md                         | 13 +++++---
 docker-compose.dev.opencl.amd.yml |  4 +--
 4 files changed, 47 insertions(+), 24 deletions(-)

diff --git a/.github/workflows/dev.opencl.yml b/.github/workflows/dev.opencl.yml
index 168566f..777a3b8 100644
--- a/.github/workflows/dev.opencl.yml
+++ b/.github/workflows/dev.opencl.yml
@@ -27,8 +27,8 @@ jobs:
             include:
               - docker_file: Dockerfile.opencl.nvidia
                 label: "dev-opencl-nvidia"
-            #   - docker_file: Dockerfile.opencl.amd
-            #     label: "dev-opencl-amd"
+              - docker_file: Dockerfile.opencl.amd
+                label: "dev-opencl-amd"
 
         steps:
             - uses: actions/checkout@v4
diff --git a/Dockerfile.opencl.amd b/Dockerfile.opencl.amd
index 5366373..7cd0e6b 100644
--- a/Dockerfile.opencl.amd
+++ b/Dockerfile.opencl.amd
@@ -1,7 +1,9 @@
-FROM ubuntu:22.04
+FROM ubuntu:noble
 
 ARG DEBIAN_FRONTEND=noninteractive
 
+USER root
+
 ARG PYTORCH_OCL_VERSION=0.2.0
 ARG TORCH_VERSION=2.4
 ARG PYTHON_VERSION=cp312
@@ -13,34 +15,29 @@ ENV WHL_URL=https://github.com/artyom-beilis/pytorch_dlprim/releases/download/${
 WORKDIR /app
 COPY . /app
 
-RUN apt-get update && apt-get install -y \
+RUN apt-get update && \
+    apt install software-properties-common -y && \
+    add-apt-repository ppa:kisak/kisak-mesa -y && \
+    apt-get update -y && \
+    apt dist-upgrade -y && \
+    apt-get install -y \
     wget \
     software-properties-common \
     apt-transport-https \
     ca-certificates \
     curl \
-    gpg \
+    gpg && \
     rm -rf /var/lib/apt/lists/*
 
-# Add AMD GPU repository and install drivers
-RUN apt update && \
-    wget https://repo.radeon.com/amdgpu-install/6.2.3/ubuntu/jammy/amdgpu-install_6.2.60203-1_all.deb && \
-    apt install ./amdgpu-install_6.2.60203-1_all.deb -y && \
-    amdgpu-install --usecase=graphics,opencl --opencl=rocr,legacy --accept-eula --no-32 -y && \
-    rm ./amdgpu-install_6.2.60203-1_all.deb
-
 # Install OpenCL dependencies
 RUN apt-get update && apt-get install -y \
     ocl-icd-opencl-dev \
     opencl-headers \
     ocl-icd-libopencl1 \
     clinfo \
+    mesa-opencl-icd \
     && rm -rf /var/lib/apt/lists/*
 
-# Configure OpenCL vendors
-RUN mkdir -p /etc/OpenCL/vendors && \
-    echo "libamdocl64.so" > /etc/OpenCL/vendors/amdocl64.icd
-
 # Install Python 3.12
 RUN add-apt-repository ppa:deadsnakes/ppa -y && \
     apt-get update && \
@@ -56,11 +53,32 @@ RUN python3.12 -m venv /app/venv
 ENV PATH="/app/venv/bin:$PATH"
 RUN pip install --upgrade pip
 
-# Add render and video groups
+# Install AMD GPU drivers
+RUN apt update -y && \
+    wget https://repo.radeon.com/amdgpu-install/6.2.3/ubuntu/noble/amdgpu-install_6.2.60203-1_all.deb && \
+    apt install ./amdgpu-install_6.2.60203-1_all.deb -y && \
+    rm -rf ./amdgpu-install_6.2.60203-1_all.deb 
+
+# Configure OpenCL vendors
+RUN mkdir -p /etc/OpenCL/vendors && \
+    dpkg-divert --divert '/etc/OpenCL/vendors/mesa.icd.disabled' --rename '/etc/OpenCL/vendors/mesa.icd' && \
+    echo "libamdocl64.so" > /etc/OpenCL/vendors/amdocl64.icd && \
+    chmod 755 /etc/OpenCL/vendors
+
 RUN groupadd -f render && groupadd -f video
+RUN usermod -a -G render,video root
+
+RUN mkdir -p /dev/dri && \
+    touch /dev/dri/renderD128 && \
+    chmod 666 /dev/dri/renderD128
+
+RUN chmod 666 /dev/dri/render* || true
+RUN chmod 666 /dev/dri/card* || true
 
 # Set environment variables for OpenCL
-ENV OCL_ICD_VENDORS=/etc/OpenCL/vendors
+ENV OCL_ICD_VENDORS="/etc/OpenCL/vendors"
+ENV OPENCL_VENDOR_PATH="/etc/OpenCL/vendors"
+ENV RUSTICL_ENABLE=radeonsi
 
 COPY requirements.txt /app/requirements.txt
 RUN /app/venv/bin/pip install --no-cache-dir -r requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu
diff --git a/README.md b/README.md
index fbd274e..e87fe69 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,7 @@
 
 - For ease of use it's recommended to use the provided [docker-compose.yml](https://github.com/doppeltilde/image_video_classification/blob/main/docker-compose.yml).
 
-### **CPU Support**
+### **CPU Support (AMD64 & ARM64)**
 Use the `latest` tag.
 ```yml
 services:
@@ -30,7 +30,9 @@ services:
 ```
 
 ### **NVIDIA GPU Support**
-**CUDA:**
+> [!IMPORTANT]
+> Nvidia driver >=560 needs to be installed on the host machine.
+**CUDA (AMD64 & ARM64):**
 ```yml
 services:
   image_video_classification_cuda:
@@ -55,7 +57,7 @@ services:
               count: all
               capabilities: [ gpu ]
 ```
-**OpenCL:**
+**OpenCL (AMD64):**
 ```yml
 services:
   image_video_classification_opencl:
@@ -82,7 +84,9 @@ services:
 ```
 
 ### **AMD GPU Support**
-**OpenCL:**
+> [!WARNING]
+> Unless AMD starts to support older Cards again (e.g. gfx803), ROCm will not be available.
+**OpenCL (AMD64):**
 ```yml
 services:
   image_video_classification_cuda:
@@ -110,6 +114,7 @@ services:
       - "109"
 ```
 
+## Environment Variables
 - Create a `.env` file and set the preferred values.
 ```sh
 DEFAULT_MODEL_NAME=Falconsai/nsfw_image_detection
diff --git a/docker-compose.dev.opencl.amd.yml b/docker-compose.dev.opencl.amd.yml
index 158e4af..1eeb517 100644
--- a/docker-compose.dev.opencl.amd.yml
+++ b/docker-compose.dev.opencl.amd.yml
@@ -19,8 +19,8 @@ services:
     security_opt:
       - seccomp:unconfined
     group_add:
-      - "39"
-      - "109"
+      - "video"
+      - "render"
     volumes:
       - ./models:/root/.cache/huggingface/hub:rw
       - /tmp/.X11-unix:/tmp/.X11-unix