Improve Linux Helix containers usability

src/almalinux/8/helix/amd64/Dockerfile

@@ -21,10 +21,7 @@ RUN dnf upgrade --refresh -y \
         file \
     && dnf clean all
 
-RUN python3 -m pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple \
-    && python3 -m pip install ./helix_scripts-*-py3-none-any.whl
-
-# Aspnetcore test runs expect python to be available without version suffix
+# Test runs expect python to be available without version suffix
 RUN alternatives --set python /usr/bin/python3 \
     && ln -sf /usr/bin/pip3 /usr/bin/pip
 
@@ -33,8 +30,14 @@ ENV LANG=en-US.UTF-8
 # create helixbot user and give rights to sudo without password
 RUN adduser --uid 1000 --shell /bin/bash --gid adm helixbot \
     && chmod 755 /root \
-    && echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers
+    && echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers.d/helixbot
 
 USER helixbot
+WORKDIR /home/helixbot
+ENV VIRTUAL_ENV=/home/helixbot/.vsts-env
 
-RUN python3 -m venv /home/helixbot/.vsts-env
+RUN python -m venv $VIRTUAL_ENV && \
+    ${VIRTUAL_ENV}/bin/pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple && \
+    ${VIRTUAL_ENV}/bin/pip install ./helix_scripts-*-py3-none-any.whl && \
+    rm ./helix_scripts-*-py3-none-any.whl
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"

src/alpine/3.18/helix/Dockerfile

@@ -10,7 +10,8 @@ RUN apk add --upgrade --no-cache \
 RUN python3 -m venv /venv && \
         source /venv/bin/activate && \
         pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple && \
-        pip install ./helix_scripts-*-py3-none-any.whl
+        pip install ./helix_scripts-*-py3-none-any.whl && \
+        rm ./helix_scripts-*-py3-none-any.whl
 
 FROM library/alpine:3.18
 
@@ -52,5 +53,5 @@ USER helixbot
 # Install Helix Dependencies
 ENV VIRTUAL_ENV=/home/helixbot/.vsts-env
 RUN python3 -m venv $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 COPY --from=venv --chown=helixbot /venv $VIRTUAL_ENV

src/alpine/3.20/helix/Dockerfile

@@ -12,7 +12,8 @@ RUN apk add --upgrade --no-cache \
 RUN python3 -m venv /venv && \
         source /venv/bin/activate && \
         pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple && \
-        pip install ./helix_scripts-*-py3-none-any.whl
+        pip install ./helix_scripts-*-py3-none-any.whl && \
+        rm ./helix_scripts-*-py3-none-any.whl
 
 FROM library/alpine:3.20
 
@@ -55,5 +56,5 @@ USER helixbot
 # Install Helix Dependencies
 ENV VIRTUAL_ENV=/home/helixbot/.vsts-env
 RUN python3 -m venv $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 COPY --from=venv --chown=helixbot /venv $VIRTUAL_ENV

src/alpine/3.21/helix/Dockerfile

@@ -12,7 +12,8 @@ RUN apk add --upgrade --no-cache \
 RUN python3 -m venv /venv && \
         source /venv/bin/activate && \
         pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple && \
-        pip install ./helix_scripts-*-py3-none-any.whl
+        pip install ./helix_scripts-*-py3-none-any.whl && \
+        rm ./helix_scripts-*-py3-none-any.whl
 
 FROM library/alpine:3.21
 
@@ -53,5 +54,5 @@ USER helixbot
 # Install Helix Dependencies
 ENV VIRTUAL_ENV=/home/helixbot/.vsts-env
 RUN python3 -m venv $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 COPY --from=venv --chown=helixbot /venv $VIRTUAL_ENV

src/azurelinux/3.0/helix/Dockerfile

@@ -12,8 +12,8 @@ RUN tdnf install --refresh -y \
 RUN python3 -m venv /venv && \
         source /venv/bin/activate && \
         pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple && \
-        pip install ./helix_scripts-*-py3-none-any.whl
-
+        pip install ./helix_scripts-*-py3-none-any.whl && \
+        rm ./helix_scripts-*-py3-none-any.whl
 
 FROM mcr.microsoft.com/azurelinux/base/core:3.0
 
@@ -29,6 +29,7 @@ RUN tdnf install --setopt tsflags=nodocs --refresh -y \
         python3 \
         python3-pip \
         shadow-utils \
+        sudo \
         tar \
         tzdata \
         which \
@@ -37,13 +38,13 @@ RUN tdnf install --setopt tsflags=nodocs --refresh -y \
 # create helixbot user and give rights to sudo without password
 RUN /usr/sbin/useradd -c '' --uid 1000 --shell /bin/bash --groups adm helixbot && \
     chmod 755 /root && \
-    echo "helixbot ALL=(ALL)       NOPASSWD: ALL" >> /etc/sudoers && \
+    echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers.d/helixbot && \
     mkdir /home/helixbot/ && chown -R helixbot /home/helixbot/
 
 USER helixbot
 
 # Install Helix Dependencies
 ENV VIRTUAL_ENV=/home/helixbot/.vsts-env
 RUN python3 -m venv $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 COPY --from=venv --chown=helixbot /venv $VIRTUAL_ENV

src/cbl-mariner/2.0/helix/amd64/Dockerfile

@@ -13,23 +13,27 @@ RUN tdnf install --setopt tsflags=nodocs --refresh -y \
          llvm \
          python3-pip \
          shadow-utils \
+         sudo \
          tar \
          tzdata \
          which \
     && tdnf clean all
 
-RUN ln -sf /usr/bin/python3 /usr/bin/python && \
-    python -m pip install --upgrade setuptools && \
-    pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple && \
-    pip install ./helix_scripts-*-py3-none-any.whl && \
-    rm ./helix_scripts-*-py3-none-any.whl
+RUN ln -sf /usr/bin/python3 /usr/bin/python
 
 # create helixbot user and give rights to sudo without password
 RUN /usr/sbin/useradd -c '' --uid 1000 --shell /bin/bash --groups adm helixbot && \
     chmod 755 /root && \
-    echo "helixbot ALL=(ALL)       NOPASSWD: ALL" >> /etc/sudoers && \
+    echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers.d/helixbot && \
     mkdir /home/helixbot/ && chown -R helixbot /home/helixbot/
 
 USER helixbot
+WORKDIR /home/helixbot
+ENV VIRTUAL_ENV=/home/helixbot/.vsts-env
 
-RUN python -m venv /home/helixbot/.vsts-env
+RUN python -m venv $VIRTUAL_ENV && \
+    ${VIRTUAL_ENV}/bin/pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple && \
+    ${VIRTUAL_ENV}/bin/pip install --upgrade pip && \
+    ${VIRTUAL_ENV}/bin/pip install ./helix_scripts-*-py3-none-any.whl && \
+    rm ./helix_scripts-*-py3-none-any.whl
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"

src/centos-stream/10/helix/Dockerfile

@@ -11,8 +11,9 @@ RUN dnf upgrade --refresh -y \
 RUN python3 -m venv /venv \
     && . /venv/bin/activate \
     && pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple \
-    && pip install ./helix_scripts-*-py3-none-any.whl
-
+    && pip install ./helix_scripts-*-py3-none-any.whl \
+    && rm ./helix_scripts-*-py3-none-any.whl
+
 FROM quay.io/centos/centos:stream10
 
 # Install dependencies
@@ -50,12 +51,12 @@ ENV LANG=en_US.utf8
 # create helixbot user and give rights to sudo without password
 RUN /usr/sbin/adduser --uid 1000 --shell /bin/bash --gid adm helixbot \
     && chmod 755 /root \
-    && echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers
+    && echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers.d/helixbot
 
 USER helixbot
+ENV VIRTUAL_ENV=/home/helixbot/.vsts-env
 
 # Install Helix Dependencies
-ENV VIRTUAL_ENV=/home/helixbot/.vsts-env
 RUN python3 -m venv $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 COPY --from=venv --chown=helixbot /venv $VIRTUAL_ENV

src/centos-stream/9/helix/amd64/Dockerfile

@@ -32,17 +32,21 @@ RUN dnf upgrade --refresh -y \
         which \
     && dnf clean all
 
-RUN ln -sf /usr/bin/python3 /usr/bin/python \
-    && pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple \
-    && pip install ./helix_scripts-*-py3-none-any.whl
+RUN ln -sf /usr/bin/python3 /usr/bin/python
 
 ENV LANG=en_US.utf8
 
 # create helixbot user and give rights to sudo without password
 RUN /usr/sbin/adduser --uid 1000 --shell /bin/bash --gid adm helixbot \
     && chmod 755 /root \
-    && echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers
+    && echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers.d/helixbot
 
 USER helixbot
+WORKDIR /home/helixbot
+ENV VIRTUAL_ENV=/home/helixbot/.vsts-env
 
-RUN python -m venv /home/helixbot/.vsts-env
+RUN python -m venv $VIRTUAL_ENV && \
+    ${VIRTUAL_ENV}/bin/pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple && \
+    ${VIRTUAL_ENV}/bin/pip install ./helix_scripts-*-py3-none-any.whl && \
+    rm ./helix_scripts-*-py3-none-any.whl
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"

src/centos-stream/9/mlnet/helix/amd64/Dockerfile

@@ -46,6 +46,7 @@ RUN dnf upgrade --refresh -y \
         procps-ng \
         python3 \
         python3-devel \
+        python3-pip \
         readline-devel \
         sudo \
         swig \
@@ -82,17 +83,21 @@ RUN wget http://releases.llvm.org/3.9.1/openmp-3.9.1.src.tar.xz \
 # Sets the library path to pickup openmp
 ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64
 
-RUN ln -sf /usr/bin/python3 /usr/bin/python \
-    && pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple \
-    && pip install ./helix_scripts-*-py3-none-any.whl
+RUN ln -sf /usr/bin/python3 /usr/bin/python
 
 ENV LANG=en_US.utf8
 
 # create helixbot user and give rights to sudo without password
 RUN /usr/sbin/adduser --uid 1000 --shell /bin/bash --gid adm helixbot \
     && chmod 755 /root \
-    && echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers
+    && echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers.d/helixbot
 
 USER helixbot
+WORKDIR /home/helixbot
+ENV VIRTUAL_ENV=/home/helixbot/.vsts-env
 
-RUN python -m venv /home/helixbot/.vsts-env
+RUN python -m venv $VIRTUAL_ENV && \
+    ${VIRTUAL_ENV}/bin/pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple && \
+    ${VIRTUAL_ENV}/bin/pip install ./helix_scripts-*-py3-none-any.whl && \
+    rm ./helix_scripts-*-py3-none-any.whl
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"

src/debian/11/helix/amd64/Dockerfile

@@ -42,16 +42,20 @@ RUN apt-get update \
 
 ENV LANG=en_US.utf8
 
-RUN ln -sf /usr/bin/python3 /usr/bin/python \
-    && pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple \
-    && pip install ./helix_scripts-*-py3-none-any.whl
+RUN ln -sf /usr/bin/python3 /usr/bin/python
 
 # Create helixbot user and give rights to sudo without password
-# additionally, preinstall the virtualenv packages used for VSTS reporting to save time
 RUN /usr/sbin/adduser --disabled-password --gecos '' --uid 1000 --shell /bin/bash --ingroup adm helixbot \
     && chmod 755 /root \
-    && echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers
+    && echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers.d/helixbot
 
 USER helixbot
+WORKDIR /home/helixbot
+ENV VIRTUAL_ENV=/home/helixbot/.vsts-env
 
-RUN python -m venv /home/helixbot/.vsts-env
+RUN python -m venv $VIRTUAL_ENV && \
+    ${VIRTUAL_ENV}/bin/pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple && \
+    ${VIRTUAL_ENV}/bin/pip install --upgrade pip && \
+    ${VIRTUAL_ENV}/bin/pip install ./helix_scripts-*-py3-none-any.whl && \
+    rm ./helix_scripts-*-py3-none-any.whl
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"

src/debian/11/helix/arm64v8/Dockerfile

@@ -43,20 +43,20 @@ RUN apt-get update \
 
 ENV LANG=en_US.utf8
 
-RUN ln -sf /usr/bin/python3 /usr/bin/python \
-    && pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple \
-    && export CRYPTOGRAPHY_DONT_BUILD_RUST=1 \
-    && pip install ./helix_scripts-*-py3-none-any.whl
+RUN ln -sf /usr/bin/python3 /usr/bin/python
 
-# Create helixbot users and give rights to sudo without password
-# (we use two users here to ensure volume mounting works with two possible UIDs of the host UID)
-# additionally, preinstall the virtualenv packages used for VSTS reporting to save time
-RUN /usr/sbin/adduser --disabled-password --gecos '' --uid 1001 --shell /bin/bash --ingroup adm helixbot \
-    && /usr/sbin/adduser --disabled-password --gecos '' --uid 1000 --shell /bin/bash --ingroup adm helixbot2 \
+# Create helixbot user and give rights to sudo without password
+RUN /usr/sbin/adduser --disabled-password --gecos '' --uid 1000 --shell /bin/bash --ingroup adm helixbot \
     && chmod 755 /root \
-    && echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers \
-    && echo "helixbot2 ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers 
+    && echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers.d/helixbot
 
 USER helixbot
+WORKDIR /home/helixbot
+ENV VIRTUAL_ENV=/home/helixbot/.vsts-env
 
-RUN python -m venv /home/helixbot/.vsts-env
+RUN python -m venv $VIRTUAL_ENV && \
+    ${VIRTUAL_ENV}/bin/pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple && \
+    ${VIRTUAL_ENV}/bin/pip install --upgrade pip && \
+    ${VIRTUAL_ENV}/bin/pip install ./helix_scripts-*-py3-none-any.whl && \
+    rm ./helix_scripts-*-py3-none-any.whl
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"

src/debian/12/helix/amd64/Dockerfile

@@ -12,7 +12,8 @@ RUN apt-get update \
 RUN python3 -m venv /venv \
     && . /venv/bin/activate \
     && pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple \
-    && pip install ./helix_scripts-*-py3-none-any.whl
+    && pip install ./helix_scripts-*-py3-none-any.whl \
+    && rm ./helix_scripts-*-py3-none-any.whl
 
 FROM library/debian:bookworm
 
@@ -59,15 +60,14 @@ RUN apt-get update \
 ENV LANG=en_US.utf8
 
 # Create helixbot user and give rights to sudo without password
-# additionally, preinstall the virtualenv packages used for VSTS reporting to save time
 RUN /usr/sbin/adduser --disabled-password --gecos '' --uid 1000 --shell /bin/bash --ingroup adm helixbot \
     && chmod 755 /root \
-    && echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers
+    && echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers.d/helixbot
 
 USER helixbot
 
 # Install Helix Dependencies
 ENV VIRTUAL_ENV=/home/helixbot/.vsts-env
 RUN python3 -m venv $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 COPY --from=venv --chown=helixbot /venv $VIRTUAL_ENV

src/debian/12/helix/arm32v7/Dockerfile

@@ -21,7 +21,8 @@ ENV PATH="/root/.cargo/bin:${PATH}"
 RUN python3 -m venv /venv \
     && . /venv/bin/activate \
     && pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple \
-    && pip install ./helix_scripts-*-py3-none-any.whl
+    && pip install ./helix_scripts-*-py3-none-any.whl \
+    && rm ./helix_scripts-*-py3-none-any.whl
 
 FROM library/debian:bookworm
 
@@ -68,15 +69,14 @@ RUN apt-get update \
 ENV LANG=en_US.utf8
 
 # Create helixbot user and give rights to sudo without password
-# additionally, preinstall the virtualenv packages used for VSTS reporting to save time
 RUN /usr/sbin/adduser --disabled-password --gecos '' --uid 1000 --shell /bin/bash --ingroup adm helixbot \
     && chmod 755 /root \
-    && echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers
+    && echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers.d/helixbot
 
 USER helixbot
 
 # Install Helix Dependencies
 ENV VIRTUAL_ENV=/home/helixbot/.vsts-env
 RUN python3 -m venv $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 COPY --from=venv --chown=helixbot /venv $VIRTUAL_ENV

src/debian/12/helix/arm64v8/Dockerfile

@@ -12,7 +12,8 @@ RUN apt-get update \
 RUN python3 -m venv /venv \
     && . /venv/bin/activate \
     && pip download --no-deps helix-scripts --index-url https://dnceng.pkgs.visualstudio.com/public/_packaging/helix-client-prod/pypi/simple \
-    && pip install ./helix_scripts-*-py3-none-any.whl
+    && pip install ./helix_scripts-*-py3-none-any.whl \
+    && rm ./helix_scripts-*-py3-none-any.whl
 
 FROM library/debian:bookworm
 
@@ -58,15 +59,14 @@ RUN apt-get update \
 ENV LANG=en_US.utf8
 
 # Create helixbot user and give rights to sudo without password
-# additionally, preinstall the virtualenv packages used for VSTS reporting to save time
 RUN /usr/sbin/adduser --disabled-password --gecos '' --uid 1000 --shell /bin/bash --ingroup adm helixbot \
     && chmod 755 /root \
-    && echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers
+    && echo "helixbot ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers.d/helixbot
 
 USER helixbot
 
 # Install Helix Dependencies
 ENV VIRTUAL_ENV=/home/helixbot/.vsts-env
 RUN python3 -m venv $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 COPY --from=venv --chown=helixbot /venv $VIRTUAL_ENV