From 1acd10cfbba8f293fef851ebbcdada606e7171d9 Mon Sep 17 00:00:00 2001 From: Christian Schou Oxvig Date: Fri, 16 May 2025 11:05:00 +0000 Subject: [PATCH 01/10] First draft of devcontainer for cotainr --- .devcontainer/devcontainer.json | 44 +++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..be317ebb --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,44 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/universal +{ + "name": "Cotainr devcontainer", + + // Default image to use as a base for the dev container. + // If you don't have access to this registry, you can build the container locally from the ./github/dockerfiles/Dockerfile. + "image": "ghcr.io/deic-hpc/cotainr-dev_env-apptainer-1.3.4:docker_dev_env_PR_pipeline", + + // Set security options needed to run Apptainer/SingularityCE in the container. + "securityOpt": ["systempaths=unconfined"], + + // Sync the uv python venv on startup + // https://code.visualstudio.com/remote/advancedcontainers/start-processes + "postStartCommand": "uv sync --frozen", + + // Set environment variables to use in IDE proceses in the container. + "remoteEnv": { + "UV_LINK_MODE": "copy" // hardlinking does not seem to work properly in a volume mount + }, + + // Run container as non-root user. More info: https://aka.ms/dev-containers-non-root. + // https://code.visualstudio.com/remote/advancedcontainers/add-nonroot-user#_specifying-a-user-for-vs-code + "remoteUser": "1000:1000", + + // Configure tool-specific properties. + "customizations": { + // Define VS Code extensions to install in the dev container. + // https://code.visualstudio.com/docs/devcontainers/containers#_managing-extensions + "vscode": { + "extensions": [ + "ms-python.python", + "ms-azuretools.vscode-containers", + "ms-vscode.makefile-tools", + "github.vscode-github-actions", + "streetsidesoftware.code-spell-checker", + "tamasfe.even-better-toml", + "redhat.vscode-yaml", + "DavidAnson.vscode-markdownlint", + "trond-snekvik.simple-rst" + ] + } + } +} From c9b2fd37fbf3c77898c30c04a4459afe06df781a Mon Sep 17 00:00:00 2001 From: Christian Schou Oxvig Date: Fri, 23 May 2025 10:11:29 +0000 Subject: [PATCH 02/10] Dev container pre-commit and vscode integrations. --- .devcontainer/devcontainer.json | 35 ++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index be317ebb..547a858d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,32 +2,49 @@ // README at: https://github.com/devcontainers/templates/tree/main/src/universal { "name": "Cotainr devcontainer", - // Default image to use as a base for the dev container. // If you don't have access to this registry, you can build the container locally from the ./github/dockerfiles/Dockerfile. "image": "ghcr.io/deic-hpc/cotainr-dev_env-apptainer-1.3.4:docker_dev_env_PR_pipeline", - // Set security options needed to run Apptainer/SingularityCE in the container. - "securityOpt": ["systempaths=unconfined"], - + "securityOpt": [ + "systempaths=unconfined", + "no-new-privileges" + ], + // Mounts to persist select data between container restarts and rebuilds. + "mounts": [ + { + "source": "pre-commit-env", + "target": "/pre-commit-env", + "type": "volume" + } + ], // Sync the uv python venv on startup // https://code.visualstudio.com/remote/advancedcontainers/start-processes "postStartCommand": "uv sync --frozen", - // Set environment variables to use in IDE proceses in the container. "remoteEnv": { - "UV_LINK_MODE": "copy" // hardlinking does not seem to work properly in a volume mount + "UV_LINK_MODE": "copy", // hardlinking does not seem to work properly in a volume mount + "PRE_COMMIT_HOME": "/pre-commit-env" }, - // Run container as non-root user. More info: https://aka.ms/dev-containers-non-root. // https://code.visualstudio.com/remote/advancedcontainers/add-nonroot-user#_specifying-a-user-for-vs-code - "remoteUser": "1000:1000", - + // containerUser is needed over remoteUser to have volumes mounted with the correct permissions for use by the non-root user. + "containerUser": "1000:1000", // Configure tool-specific properties. "customizations": { // Define VS Code extensions to install in the dev container. // https://code.visualstudio.com/docs/devcontainers/containers#_managing-extensions "vscode": { + "settings": { + "terminal.integrated.env.linux": { + "GIT_EDITOR": "code --wait" // Use VS Code when editing commit messages + }, + "github.copilot.chat.codeGeneration.instructions": [ + { + "text": "This dev container is used for developing and testing Cotainr. Cotainr is a tool for building Singularity / Apptainer containers in a rootless setting. It is written in pure Python and uses pytest for testing as well as sphinx for building the documentation. The documentation is written as restructured text." + } + ] + }, "extensions": [ "ms-python.python", "ms-azuretools.vscode-containers", From cc5c862deb81c1923bdab8828d281cc06f3ee7de Mon Sep 17 00:00:00 2001 From: Christian Schou Oxvig Date: Fri, 23 May 2025 10:33:59 +0000 Subject: [PATCH 03/10] Cleaned up comments in devcontainer.json --- .devcontainer/devcontainer.json | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 547a858d..07474e41 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,39 +2,47 @@ // README at: https://github.com/devcontainers/templates/tree/main/src/universal { "name": "Cotainr devcontainer", + // Default image to use as a base for the dev container. // If you don't have access to this registry, you can build the container locally from the ./github/dockerfiles/Dockerfile. "image": "ghcr.io/deic-hpc/cotainr-dev_env-apptainer-1.3.4:docker_dev_env_PR_pipeline", + // Set security options needed to run Apptainer/SingularityCE in the container. + // These should be sufficient for running the container using rootless Podman. + // More settings may be needed for other container runtimes, e.g. Docker. "securityOpt": [ "systempaths=unconfined", "no-new-privileges" ], + // Mounts to persist select data between container restarts and rebuilds. "mounts": [ - { + {// Persist the pre-commit environment "source": "pre-commit-env", "target": "/pre-commit-env", "type": "volume" } ], + // Sync the uv python venv on startup // https://code.visualstudio.com/remote/advancedcontainers/start-processes "postStartCommand": "uv sync --frozen", + // Set environment variables to use in IDE proceses in the container. "remoteEnv": { "UV_LINK_MODE": "copy", // hardlinking does not seem to work properly in a volume mount "PRE_COMMIT_HOME": "/pre-commit-env" }, + // Run container as non-root user. More info: https://aka.ms/dev-containers-non-root. // https://code.visualstudio.com/remote/advancedcontainers/add-nonroot-user#_specifying-a-user-for-vs-code // containerUser is needed over remoteUser to have volumes mounted with the correct permissions for use by the non-root user. "containerUser": "1000:1000", + // Configure tool-specific properties. "customizations": { - // Define VS Code extensions to install in the dev container. - // https://code.visualstudio.com/docs/devcontainers/containers#_managing-extensions "vscode": { + // VS Code settings for the container. "settings": { "terminal.integrated.env.linux": { "GIT_EDITOR": "code --wait" // Use VS Code when editing commit messages @@ -45,6 +53,8 @@ } ] }, + // VS Code extensions to install in the dev container. + // https://code.visualstudio.com/docs/devcontainers/containers#_managing-extensions "extensions": [ "ms-python.python", "ms-azuretools.vscode-containers", From 35b18f902582dff76764ecfb6f864f300601a5e8 Mon Sep 17 00:00:00 2001 From: Christian Schou Oxvig Date: Fri, 23 May 2025 11:55:46 +0000 Subject: [PATCH 04/10] Persisted bash history and set LANG in devcontainer.json --- .devcontainer/devcontainer.json | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 07474e41..5c6e01f3 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -21,6 +21,12 @@ "source": "pre-commit-env", "target": "/pre-commit-env", "type": "volume" + }, + { + // Persist the command history + "source": "commandhistory", + "target": "/commandhistory", + "type": "volume" } ], @@ -28,10 +34,13 @@ // https://code.visualstudio.com/remote/advancedcontainers/start-processes "postStartCommand": "uv sync --frozen", - // Set environment variables to use in IDE proceses in the container. + // Set environment variables to use in IDE processes in the container. "remoteEnv": { - "UV_LINK_MODE": "copy", // hardlinking does not seem to work properly in a volume mount - "PRE_COMMIT_HOME": "/pre-commit-env" + "LANG": "C.UTF-8", // Set the locale to C.UTF-8 to avoid issues with non-ASCII characters + "UV_LINK_MODE": "copy", // uv hardlinking does not work in a volume mount + "PRE_COMMIT_HOME": "/pre-commit-env", // Set the pre-commit env path to a persisted mounted volume + "HISTFILE": "/commandhistory/.bash_history", // Set the history file to a persisted mounted volume + "PROMPT_COMMAND": "history -a" // Append to the history file after each command }, // Run container as non-root user. More info: https://aka.ms/dev-containers-non-root. From 898a7dfe45de6d6946b5890f0fff4cac0817a2f5 Mon Sep 17 00:00:00 2001 From: Christian Schou Oxvig Date: Mon, 26 May 2025 08:56:16 +0000 Subject: [PATCH 05/10] Updated dev container to use main branch container. --- .devcontainer/devcontainer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 5c6e01f3..c332cb98 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,11 +1,11 @@ // For format details, see https://aka.ms/devcontainer.json. For config options, see the // README at: https://github.com/devcontainers/templates/tree/main/src/universal { - "name": "Cotainr devcontainer", + "name": "Cotainr dev container", // Default image to use as a base for the dev container. // If you don't have access to this registry, you can build the container locally from the ./github/dockerfiles/Dockerfile. - "image": "ghcr.io/deic-hpc/cotainr-dev_env-apptainer-1.3.4:docker_dev_env_PR_pipeline", + "image": "ghcr.io/deic-hpc/cotainr-dev_env-apptainer-1.3.4:main", // Set security options needed to run Apptainer/SingularityCE in the container. // These should be sufficient for running the container using rootless Podman. From a3a3121533dc48b5e93987554ccf9d24ba8f7b1e Mon Sep 17 00:00:00 2001 From: Christian Schou Oxvig Date: Mon, 26 May 2025 08:58:34 +0000 Subject: [PATCH 06/10] Added missing relnotes .PHONY target in docs Makefile. --- doc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Makefile b/doc/Makefile index 564d4d63..c7218898 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -16,7 +16,7 @@ help: @python3 -c """import sphinx.util; print(' %s to autogenerate the API reference rst files' % sphinx.util.console.blue('apidoc'.ljust(10)))""" @python3 -c """import sphinx.util; print(' %s to autogenerate the release notes rst file' % sphinx.util.console.blue('relnotes'.ljust(10)))""" -.PHONY: help apidoc Makefile +.PHONY: help apidoc relnotes Makefile apidoc: @$(SPHINXAPIDOC) --force --separate --no-toc -o api_reference $(SOURCEDIR)/../$(PACKAGENAME) $(SOURCEDIR)/../$(PACKAGENAME)/tests/* From 9570460979833c1dceec15b1a9a353d10a7f7667 Mon Sep 17 00:00:00 2001 From: Christian Schou Oxvig Date: Mon, 26 May 2025 09:05:41 +0000 Subject: [PATCH 07/10] Added common runtime env vars to Dockerfile. --- .devcontainer/devcontainer.json | 4 ++-- .github/workflows/dockerfiles/Dockerfile | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c332cb98..7ed85003 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -36,8 +36,8 @@ // Set environment variables to use in IDE processes in the container. "remoteEnv": { - "LANG": "C.UTF-8", // Set the locale to C.UTF-8 to avoid issues with non-ASCII characters - "UV_LINK_MODE": "copy", // uv hardlinking does not work in a volume mount + //"LANG": "C.UTF-8", // Set the locale to C.UTF-8 to avoid issues with non-ASCII characters + //"UV_LINK_MODE": "copy", // uv hardlinking does not work in a volume mount "PRE_COMMIT_HOME": "/pre-commit-env", // Set the pre-commit env path to a persisted mounted volume "HISTFILE": "/commandhistory/.bash_history", // Set the history file to a persisted mounted volume "PROMPT_COMMAND": "history -a" // Append to the history file after each command diff --git a/.github/workflows/dockerfiles/Dockerfile b/.github/workflows/dockerfiles/Dockerfile index 8978d2ed..01685c36 100644 --- a/.github/workflows/dockerfiles/Dockerfile +++ b/.github/workflows/dockerfiles/Dockerfile @@ -1,5 +1,5 @@ FROM ubuntu:latest -LABEL authors="julius" +LABEL authors="DeiC HPC" # get build-arg inputs to set the singularity provider and versions ARG SINGULARITY_PROVIDER @@ -75,5 +75,10 @@ RUN set -eux;\ singularity --version # Download and install unmanaged uv +# Set UV_LINK_MODE to copy since hardlinking does not work with volume mounts RUN set -eux;\ curl -LsSf https://astral.sh/uv/install.sh | env UV_UNMANAGED_INSTALL="/usr/local/bin" sh +ENV UV_LINK_MODE="copy" + +# Set default locale to english (as needed for the docs prepare-release.py script) and utf-8 as required when using with e.g. a VS Code terminal. +ENV LANG="C.UTF-8" From 4009df81e55a39725d478254e00d476af8c70f3f Mon Sep 17 00:00:00 2001 From: Christian Schou Oxvig Date: Mon, 26 May 2025 09:08:25 +0000 Subject: [PATCH 08/10] Removed env vars from devcontainer.json that are now in the Dockerfile. --- .devcontainer/devcontainer.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 7ed85003..ec0016ab 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -36,8 +36,6 @@ // Set environment variables to use in IDE processes in the container. "remoteEnv": { - //"LANG": "C.UTF-8", // Set the locale to C.UTF-8 to avoid issues with non-ASCII characters - //"UV_LINK_MODE": "copy", // uv hardlinking does not work in a volume mount "PRE_COMMIT_HOME": "/pre-commit-env", // Set the pre-commit env path to a persisted mounted volume "HISTFILE": "/commandhistory/.bash_history", // Set the history file to a persisted mounted volume "PROMPT_COMMAND": "history -a" // Append to the history file after each command From e27cecca0e210798ea9455c42989a793db6f9fa9 Mon Sep 17 00:00:00 2001 From: Christian Schou Oxvig Date: Wed, 28 May 2025 07:40:07 +0000 Subject: [PATCH 09/10] Added comment to devcontainer.json about volume mounts with docker. --- .devcontainer/devcontainer.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ec0016ab..89f35756 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -44,6 +44,8 @@ // Run container as non-root user. More info: https://aka.ms/dev-containers-non-root. // https://code.visualstudio.com/remote/advancedcontainers/add-nonroot-user#_specifying-a-user-for-vs-code // containerUser is needed over remoteUser to have volumes mounted with the correct permissions for use by the non-root user. + // This has been tested with Podman. If using Docker, more configuration may be needed to avoid permission errors with the volume mounts. + // See https://github.com/microsoft/vscode-remote-release/issues/9931 for more details. "containerUser": "1000:1000", // Configure tool-specific properties. From 344f90734ad5de5cdd2bbabe930251124db79b89 Mon Sep 17 00:00:00 2001 From: Christian Schou Oxvig Date: Fri, 20 Jun 2025 09:14:17 +0000 Subject: [PATCH 10/10] Updated devcontainer.json to persist venv in volume mount. --- .devcontainer/devcontainer.json | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 89f35756..d7cb2ffe 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -17,13 +17,17 @@ // Mounts to persist select data between container restarts and rebuilds. "mounts": [ + {// Persist the virtual environment (with paths relative to the dev container setup) + "source": "uv-venv", + "target": "/uv-venv", + "type": "volume" + }, {// Persist the pre-commit environment "source": "pre-commit-env", "target": "/pre-commit-env", "type": "volume" }, - { - // Persist the command history + {// Persist the command history "source": "commandhistory", "target": "/commandhistory", "type": "volume" @@ -36,6 +40,7 @@ // Set environment variables to use in IDE processes in the container. "remoteEnv": { + "UV_PROJECT_ENVIRONMENT": "/uv-venv", // Set the uv virtual environment path to a persisted mounted volume "PRE_COMMIT_HOME": "/pre-commit-env", // Set the pre-commit env path to a persisted mounted volume "HISTFILE": "/commandhistory/.bash_history", // Set the history file to a persisted mounted volume "PROMPT_COMMAND": "history -a" // Append to the history file after each command @@ -56,6 +61,8 @@ "terminal.integrated.env.linux": { "GIT_EDITOR": "code --wait" // Use VS Code when editing commit messages }, + "python.defaultInterpreterPath": "/uv-venv/bin/python", // Set the default Python interpreter to the uv virtual environment + "python.terminal.activateEnvironment": true, // Activate the uv virtual environment in the terminal "github.copilot.chat.codeGeneration.instructions": [ { "text": "This dev container is used for developing and testing Cotainr. Cotainr is a tool for building Singularity / Apptainer containers in a rootless setting. It is written in pure Python and uses pytest for testing as well as sphinx for building the documentation. The documentation is written as restructured text."