From 52530d6a6e0f25a5370a48317399530aee8aab83 Mon Sep 17 00:00:00 2001 From: makolon Date: Wed, 4 Dec 2024 10:19:01 +0900 Subject: [PATCH 1/4] Add docker environment --- docker/BUILD_DOCKER_IMAGE.sh | 17 ++++++++++++ docker/README.md | 37 +++++++++++++++++++++++++++ docker/RUN_DOCKER_CONTAINER.sh | 22 ++++++++++++++++ docker/docker-compose.yml | 29 +++++++++++++++++++++ docker/initialize-docker-container.sh | 13 ++++++++++ docker/lerobot-gpu-dev/Dockerfile | 10 ++++++++ docker/wandb_key.txt.keep | 0 7 files changed, 128 insertions(+) create mode 100755 docker/BUILD_DOCKER_IMAGE.sh create mode 100644 docker/README.md create mode 100755 docker/RUN_DOCKER_CONTAINER.sh create mode 100644 docker/docker-compose.yml create mode 100755 docker/initialize-docker-container.sh create mode 100644 docker/wandb_key.txt.keep diff --git a/docker/BUILD_DOCKER_IMAGE.sh b/docker/BUILD_DOCKER_IMAGE.sh new file mode 100755 index 0000000000..68031723e1 --- /dev/null +++ b/docker/BUILD_DOCKER_IMAGE.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +if [ ! -z $1 ]; then + export TAG_NAME=$1 +else + export TAG_NAME="lerobot-gpu-dev" +fi + +# Setting up dataset path +if [ -n "$DATASETS_PATH" ]; then + echo "DATASETS_PATH is set to '$DATASETS_PATH'. Mounting this path to the container as '/root/datasets'." +else + export DATASETS_PATH="$PWD/../datasets" + echo "DATASETS_PATH is not set. Using default path: $DATASETS_PATH" +fi + +docker-compose -p lerobot build diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000000..4924ff4c03 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,37 @@ +# Docker Environment +## How to Use +### (Optional) Set wandb key +Note: `wandb_key.txt` is git-ignored +```bash +cp wandb_key.txt.keep wandb_key.txt +echo '{YOUR_WANDB_KEY}' >> wandb_key.txt +``` + +### (Optional) Setup `DATASETS_PATH` to mount into the container +```bash +$ export DATASETS_PATH={YOUR_HOST_DATASETS_PATH_TO_MOUNT} +``` +The directory is mounted as `/root/datasets` in the container. + +### 1. Build docker image +Creating an image named `{YOUR_HOST_NAME}/lerobot:latest` +```bash +./BUILD_DOCKER_IMAGE.sh +``` + +### 2. Run docker container (takes some time to install additional packages) +Creating a container named `{YOUR_HOST_NAME}_gnfactor` +```bash +./RUN_DOCKER_CONTAINER.sh +``` + +### 3. Enter the container +```bash +docker exec -it {YOUR_HOST_NAME}_lerobot bash +``` + +### 4. Move to `/root/lerobot` directory in the container where this repository (lerobot) is mounted +```bash +cd /root/lerobot +``` + diff --git a/docker/RUN_DOCKER_CONTAINER.sh b/docker/RUN_DOCKER_CONTAINER.sh new file mode 100755 index 0000000000..a07df6a11c --- /dev/null +++ b/docker/RUN_DOCKER_CONTAINER.sh @@ -0,0 +1,22 @@ +#!/bin/sh +xhost + + +if [ ! -z $1 ]; then + export TAG_NAME=$1 +else + export TAG_NAME="lerobot-gpu-dev" +fi + +# Setting up dataset path +if [ -n "$DATASETS_PATH" ]; then + echo "DATASETS_PATH is set to '$DATASETS_PATH'. Mounting this path to the container as '/root/datasets'." +else + export DATASETS_PATH="$PWD/../datasets" + echo "DATASETS_PATH is not set. Using default path: $DATASETS_PATH" +fi + +# Setting up wandb key +export WANDB_API_KEY=`cat wandb_key.txt` + +docker-compose -p lerobot up & + diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000000..5d1a31b0dd --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,29 @@ +version: "3.9" + +################################################################################ + +services: + latest: + runtime: nvidia + build: + context: . + dockerfile: ${TAG_NAME}/Dockerfile + image: ${USER}/lerobot:${TAG_NAME} + container_name: ${USER}_lerobot + network_mode: host + privileged: true + shm_size: '50gb' + volumes: + - ${PWD}/../:/root/lerobot + - ${DATASETS_PATH}:/root/datasets + - /tmp/.x11-unix:/tmp/.x11-unix + environment: + - DISPLAY=${DISPLAY} + - QT_X11_NO_MITSHM=1 + - WANDB_API_KEY=${WANDB_API_KEY} + # Hack with HDF5 file locking + - HDF5_USE_FILE_LOCKING=FALSE + # For debugging with Hydra + - HYDRA_FULL_ERROR=1 + command: /root/lerobot/docker/initialize-docker-container.sh + diff --git a/docker/initialize-docker-container.sh b/docker/initialize-docker-container.sh new file mode 100755 index 0000000000..68da490295 --- /dev/null +++ b/docker/initialize-docker-container.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# install octo +source ~/.bashrc +cd ~/lerobot && poetry install --extras "aloha xarm pusht" + +# git safe directory +cd ~/lerobot && git config --global --add safe.directory /root/lerobot + +cd ~/lerobot + +# https://stackoverflow.com/questions/30209776/docker-container-will-automatically-stop-after-docker-run-d +tail -f /dev/null + diff --git a/docker/lerobot-gpu-dev/Dockerfile b/docker/lerobot-gpu-dev/Dockerfile index 561a7cff60..fa81c0a250 100644 --- a/docker/lerobot-gpu-dev/Dockerfile +++ b/docker/lerobot-gpu-dev/Dockerfile @@ -64,5 +64,15 @@ RUN echo 'if [ "$HOME" != "/root" ]; then ln -sf /root/.local/bin/poetry $HOME/. RUN poetry config virtualenvs.create false RUN poetry config virtualenvs.in-project true +# Setup X11 +RUN apt-get update && apt-get install -y --no-install-recommends \ + libx11-dev \ + libxcursor-dev \ + libxcb1-dev \ + libxi-dev \ + libxkbcommon-dev \ + libxkbcommon-x11-dev && \ + apt clean && rm -rf /var/lib/apt/lists/* + # Set EGL as the rendering backend for MuJoCo ENV MUJOCO_GL="egl" diff --git a/docker/wandb_key.txt.keep b/docker/wandb_key.txt.keep new file mode 100644 index 0000000000..e69de29bb2 From 9e753abeb15aa53818f1e39d95b8a378be58f288 Mon Sep 17 00:00:00 2001 From: k1000dai Date: Tue, 15 Apr 2025 18:33:59 +0200 Subject: [PATCH 2/4] add python3-dev --- docker/lerobot-gpu-dev/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/lerobot-gpu-dev/Dockerfile b/docker/lerobot-gpu-dev/Dockerfile index fa81c0a250..0e1242b4eb 100644 --- a/docker/lerobot-gpu-dev/Dockerfile +++ b/docker/lerobot-gpu-dev/Dockerfile @@ -14,7 +14,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ tcpdump sysstat screen tmux \ libglib2.0-0 libgl1-mesa-glx libegl1-mesa \ speech-dispatcher portaudio19-dev libgeos-dev \ - python${PYTHON_VERSION} python${PYTHON_VERSION}-venv \ + python${PYTHON_VERSION} python${PYTHON_VERSION}-venv python${PYTHON_VERSION}-dev \ && apt-get clean && rm -rf /var/lib/apt/lists/* # Install ffmpeg build dependencies. See: From b17a3d661b8540116a155e4da690c56c43cb4554 Mon Sep 17 00:00:00 2001 From: k1000dai Date: Tue, 15 Apr 2025 20:27:35 +0200 Subject: [PATCH 3/4] remove redundunt --- docker/BUILD_DOCKER_IMAGE.sh | 17 ------------ docker/README.md | 37 --------------------------- docker/RUN_DOCKER_CONTAINER.sh | 22 ---------------- docker/docker-compose.yml | 29 --------------------- docker/initialize-docker-container.sh | 13 ---------- docker/wandb_key.txt.keep | 0 6 files changed, 118 deletions(-) delete mode 100755 docker/BUILD_DOCKER_IMAGE.sh delete mode 100644 docker/README.md delete mode 100755 docker/RUN_DOCKER_CONTAINER.sh delete mode 100644 docker/docker-compose.yml delete mode 100755 docker/initialize-docker-container.sh delete mode 100644 docker/wandb_key.txt.keep diff --git a/docker/BUILD_DOCKER_IMAGE.sh b/docker/BUILD_DOCKER_IMAGE.sh deleted file mode 100755 index 68031723e1..0000000000 --- a/docker/BUILD_DOCKER_IMAGE.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh - -if [ ! -z $1 ]; then - export TAG_NAME=$1 -else - export TAG_NAME="lerobot-gpu-dev" -fi - -# Setting up dataset path -if [ -n "$DATASETS_PATH" ]; then - echo "DATASETS_PATH is set to '$DATASETS_PATH'. Mounting this path to the container as '/root/datasets'." -else - export DATASETS_PATH="$PWD/../datasets" - echo "DATASETS_PATH is not set. Using default path: $DATASETS_PATH" -fi - -docker-compose -p lerobot build diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index 4924ff4c03..0000000000 --- a/docker/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# Docker Environment -## How to Use -### (Optional) Set wandb key -Note: `wandb_key.txt` is git-ignored -```bash -cp wandb_key.txt.keep wandb_key.txt -echo '{YOUR_WANDB_KEY}' >> wandb_key.txt -``` - -### (Optional) Setup `DATASETS_PATH` to mount into the container -```bash -$ export DATASETS_PATH={YOUR_HOST_DATASETS_PATH_TO_MOUNT} -``` -The directory is mounted as `/root/datasets` in the container. - -### 1. Build docker image -Creating an image named `{YOUR_HOST_NAME}/lerobot:latest` -```bash -./BUILD_DOCKER_IMAGE.sh -``` - -### 2. Run docker container (takes some time to install additional packages) -Creating a container named `{YOUR_HOST_NAME}_gnfactor` -```bash -./RUN_DOCKER_CONTAINER.sh -``` - -### 3. Enter the container -```bash -docker exec -it {YOUR_HOST_NAME}_lerobot bash -``` - -### 4. Move to `/root/lerobot` directory in the container where this repository (lerobot) is mounted -```bash -cd /root/lerobot -``` - diff --git a/docker/RUN_DOCKER_CONTAINER.sh b/docker/RUN_DOCKER_CONTAINER.sh deleted file mode 100755 index a07df6a11c..0000000000 --- a/docker/RUN_DOCKER_CONTAINER.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh -xhost + - -if [ ! -z $1 ]; then - export TAG_NAME=$1 -else - export TAG_NAME="lerobot-gpu-dev" -fi - -# Setting up dataset path -if [ -n "$DATASETS_PATH" ]; then - echo "DATASETS_PATH is set to '$DATASETS_PATH'. Mounting this path to the container as '/root/datasets'." -else - export DATASETS_PATH="$PWD/../datasets" - echo "DATASETS_PATH is not set. Using default path: $DATASETS_PATH" -fi - -# Setting up wandb key -export WANDB_API_KEY=`cat wandb_key.txt` - -docker-compose -p lerobot up & - diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 5d1a31b0dd..0000000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,29 +0,0 @@ -version: "3.9" - -################################################################################ - -services: - latest: - runtime: nvidia - build: - context: . - dockerfile: ${TAG_NAME}/Dockerfile - image: ${USER}/lerobot:${TAG_NAME} - container_name: ${USER}_lerobot - network_mode: host - privileged: true - shm_size: '50gb' - volumes: - - ${PWD}/../:/root/lerobot - - ${DATASETS_PATH}:/root/datasets - - /tmp/.x11-unix:/tmp/.x11-unix - environment: - - DISPLAY=${DISPLAY} - - QT_X11_NO_MITSHM=1 - - WANDB_API_KEY=${WANDB_API_KEY} - # Hack with HDF5 file locking - - HDF5_USE_FILE_LOCKING=FALSE - # For debugging with Hydra - - HYDRA_FULL_ERROR=1 - command: /root/lerobot/docker/initialize-docker-container.sh - diff --git a/docker/initialize-docker-container.sh b/docker/initialize-docker-container.sh deleted file mode 100755 index 68da490295..0000000000 --- a/docker/initialize-docker-container.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# install octo -source ~/.bashrc -cd ~/lerobot && poetry install --extras "aloha xarm pusht" - -# git safe directory -cd ~/lerobot && git config --global --add safe.directory /root/lerobot - -cd ~/lerobot - -# https://stackoverflow.com/questions/30209776/docker-container-will-automatically-stop-after-docker-run-d -tail -f /dev/null - diff --git a/docker/wandb_key.txt.keep b/docker/wandb_key.txt.keep deleted file mode 100644 index e69de29bb2..0000000000 From 5e005881057fce9fe8110d60f48996dbf26acb55 Mon Sep 17 00:00:00 2001 From: k1000dai Date: Tue, 15 Apr 2025 20:32:06 +0200 Subject: [PATCH 4/4] use python3-dev --- README.md | 2 +- docker/lerobot-gpu-dev/Dockerfile | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/README.md b/README.md index af8d0aa89b..3ca20147b0 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ pip install -e . ``` > **NOTE:** If you encounter build errors, you may need to install additional dependencies (`cmake`, `build-essential`, and `ffmpeg libs`). On Linux, run: -`sudo apt-get install cmake build-essential python-dev pkg-config libavformat-dev libavcodec-dev libavdevice-dev libavutil-dev libswscale-dev libswresample-dev libavfilter-dev pkg-config`. For other systems, see: [Compiling PyAV](https://pyav.org/docs/develop/overview/installation.html#bring-your-own-ffmpeg) +`sudo apt-get install cmake build-essential python3-dev pkg-config libavformat-dev libavcodec-dev libavdevice-dev libavutil-dev libswscale-dev libswresample-dev libavfilter-dev pkg-config`. For other systems, see: [Compiling PyAV](https://pyav.org/docs/develop/overview/installation.html#bring-your-own-ffmpeg) For simulations, 🤗 LeRobot comes with gymnasium environments that can be installed as extras: - [aloha](https://github.com/huggingface/gym-aloha) diff --git a/docker/lerobot-gpu-dev/Dockerfile b/docker/lerobot-gpu-dev/Dockerfile index 0e1242b4eb..4d25b25506 100644 --- a/docker/lerobot-gpu-dev/Dockerfile +++ b/docker/lerobot-gpu-dev/Dockerfile @@ -64,15 +64,5 @@ RUN echo 'if [ "$HOME" != "/root" ]; then ln -sf /root/.local/bin/poetry $HOME/. RUN poetry config virtualenvs.create false RUN poetry config virtualenvs.in-project true -# Setup X11 -RUN apt-get update && apt-get install -y --no-install-recommends \ - libx11-dev \ - libxcursor-dev \ - libxcb1-dev \ - libxi-dev \ - libxkbcommon-dev \ - libxkbcommon-x11-dev && \ - apt clean && rm -rf /var/lib/apt/lists/* - # Set EGL as the rendering backend for MuJoCo ENV MUJOCO_GL="egl"