dependabot · jurre · Sep 1, 2022 · Aug 31, 2022
@@ -13,6 +13,8 @@
 **/.bundle
 **/coverage
 **/Gemfile.lock
+!updater/Gemfile.lock
+!updater/spec/fixtures/**/Gemfile.lock
 **/node_modules
 !**/spec/fixtures/*
 git.store

@@ -90,6 +90,46 @@ jobs:
             --rm dependabot-core-ci bash -c \
             "cd /home/dependabot/dependabot-core/${{ matrix.suite.path }} && ./script/ci-test"
 
+  updater:
+    name: Updater
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Build dependabot-core image
+        env:
+          DOCKER_BUILDKIT: 1
+        run: |
+          docker build \
+            -t "dependabot/dependabot-core:latest" \
+            --build-arg BUILDKIT_INLINE_CACHE=1 \
+            --cache-from ghcr.io/dependabot/dependabot-core \
+            .
+      - name: Free up disk space
+        run: |
+          sudo apt autoremove --purge && sudo apt -y clean
+          docker system prune -af --volumes
+          sudo swapoff -a
+          sudo rm -f /swapfile
+          df -h
+      - name: GPR login
+        run: docker login docker.pkg.github.com -u x -p ${{secrets.GITHUB_TOKEN}}
+      - name: GHCR login
+        run: docker login ghcr.io -u x -p ${{ secrets.GITHUB_TOKEN }}
+      - name: Build
+        run: script/build
+      - name: Lint
+        run: script/lint
+        env:
+          SKIP_BUILD: true
+      - name: Run updater tests
+        run: ./script/ci-test-updater
+        env:
+          SKIP_BUILD: true
+          DEPENDABOT_TEST_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
   lint:
     name: Lint
     runs-on: ubuntu-latest

@@ -0,0 +1,62 @@
+name: Push docker branch images
+env:
+  BASE_IMAGE: "ubuntu:20.04"
+  UPDATER_IMAGE: "dependabot/updater"
+  UPDATER_IMAGE_MIRROR: "ghcr.io/dependabot/dependabot-updater"
+on:
+  pull_request:
+    paths-ignore:
+      - "CHANGELOG.md"
+      - "common/lib/dependabot/version.rb"
+
+jobs:
+  push-updater-image:
+    name: Export dependabot-updater image to build artifacts
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    env:
+      TAG: ${{ github.sha }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Build dependabot-core image
+        env:
+          DOCKER_BUILDKIT: 1
+        run: |
+          docker build \
+            -t "dependabot/dependabot-core:$TAG" \
+            --build-arg BUILDKIT_INLINE_CACHE=1 \
+            --cache-from ghcr.io/dependabot/dependabot-core \
+            .
+      - name: Build dependabot-updater image
+        env:
+          DOCKER_BUILDKIT: 1
+        run: |
+          docker build \
+            -t "$UPDATER_IMAGE:$TAG" \
+            --build-arg BUILDKIT_INLINE_CACHE=1 \
+            --cache-from "$BASE_IMAGE" \
+            --cache-from "$UPDATER_IMAGE:latest" \
+            --build-arg OMNIBUS_VERSION=$TAG \
+            -f Dockerfile.updater \
+            .
+      - name: Log in to GHCR
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+      - name: Push branch image
+        if: ${{ github.repository == 'dependabot/dependabot-core' }}
+        run: |
+          docker tag "$UPDATER_IMAGE:$TAG" "$UPDATER_IMAGE_MIRROR:$TAG"
+          docker push "$UPDATER_IMAGE_MIRROR:$TAG"
+      - name: Save tagged image
+        run: |
+          docker save "$UPDATER_IMAGE:$TAG" > dependabot-updater.tar
+      - name: Archive image
+        uses: actions/upload-artifact@v3
+        with:
+          name: updater-${{ github.sha }}.tar
+          path: dependabot-updater.tar
+      - name: Set summary
+        run: echo "updater uploaded with tag $TAG" >> $GITHUB_STEP_SUMMARY
@@ -3,6 +3,8 @@ env:
   BASE_IMAGE: "ubuntu:20.04"
   CORE_IMAGE: "dependabot/dependabot-core"
   CORE_IMAGE_MIRROR: "ghcr.io/dependabot/dependabot-core"
+  UPDATER_IMAGE: "dependabot/dependabot-updater"
+  UPDATER_IMAGE_MIRROR: "ghcr.io/dependabot/dependabot-updater"
 on:
   push:
     branches:
@@ -49,6 +51,48 @@ jobs:
           docker push "$CORE_IMAGE:$VERSION"
           docker tag "$CORE_IMAGE:latest" "$CORE_IMAGE_MIRROR:$VERSION"
           docker push "$CORE_IMAGE_MIRROR:$VERSION"
+  push-updater-image:
+    name: Push dependabot-updater image to docker hub
+    runs-on: ubuntu-latest
+    if: ${{ github.repository == 'dependabot/dependabot-core' }}
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Build dependabot-updater image
+        env:
+          DOCKER_BUILDKIT: 1
+        run: |
+          VERSION="$(grep -Eo "[0-9]+\.[0-9]+\.[0-9]+" common/lib/dependabot/version.rb)"
+          docker build \
+            -t "$UPDATER_IMAGE:latest" \
+            --build-arg BUILDKIT_INLINE_CACHE=1 \
+            --cache-from "$BASE_IMAGE" \
+            --cache-from "$UPDATER_IMAGE:latest" \
+            --build-arg OMNIBUS_VERSION=$VERSION \
+            -f Dockerfile.updater \
+            .
+      - name: Log in to the Docker registry
+        run: |
+          echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+      - name: Log in to GHCR
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+      - name: Push latest image
+        run: |
+          docker push "$UPDATER_IMAGE:latest"
+          docker tag "$UPDATER_IMAGE:latest" "$UPDATER_IMAGE_MIRROR:latest"
+          docker push "$UPDATER_IMAGE_MIRROR:latest"
+      - name: Push tagged image
+        if: "contains(github.ref, 'refs/tags')"
+        run: |
+          VERSION="$(grep -Eo "[0-9]+\.[0-9]+\.[0-9]+" common/lib/dependabot/version.rb)"
+          docker tag "$UPDATER_IMAGE:latest" "$UPDATER_IMAGE:$VERSION"
+          docker push "$UPDATER_IMAGE:$VERSION"
+          docker tag "$UPDATER_IMAGE:latest" "$UPDATER_IMAGE_MIRROR:$VERSION"
+          docker push "$UPDATER_IMAGE_MIRROR:$VERSION"
   push-development-image:
     name: Push dependabot-core-development image to GHCR
     runs-on: ubuntu-latest

@@ -0,0 +1,103 @@
+# Runs all ecosystems cached and concurrently.
+name: Smoke
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+    paths-ignore:
+      - docs/**
+      - README.md
+env:
+  GH_TOKEN: ${{ secrets.E2E_PAT }}
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        suite:
+          - actions
+          - bundler
+          - cargo
+          - composer
+          - docker
+          - elm
+          - go
+          - gradle
+          - hex
+          - maven
+          - npm
+          - nuget
+          - pip
+          - pip-compile
+          - pipenv
+          - poetry
+          - pub
+          - submodules
+          - terraform
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Download CLI and test
+      run: |
+        gh release download --repo dependabot/cli -p "*linux-amd64.tar.gz"
+        tar xzvf *.tar.gz >/dev/null 2>&1
+        ./dependabot --version
+        URL=https://api.github.com/repos/dependabot/cli/contents/testdata/smoke-${{ matrix.suite }}.yaml
+        curl $(gh api $URL --jq .download_url) -o smoke.yaml
+
+    # Download the Proxy cache. The job is ideally 100% cached so no real calls are made.
+    - name: Download cache
+      run: |
+        mkdir cache
+        cd cache
+        gh run download --repo dependabot/cli --name cache-${{ matrix.suite }}
+
+    - name: GPR login
+      run: docker login docker.pkg.github.com -u x -p ${{secrets.GITHUB_TOKEN}}
+    - name: GHCR login
+      run: docker login ghcr.io -u x -p ${{ secrets.GITHUB_TOKEN }}
+    - name: Build dependabot-core image
+      env:
+        DOCKER_BUILDKIT: 1
+      run: |
+        docker build \
+          -t "dependabot/dependabot-core:latest" \
+          --build-arg BUILDKIT_INLINE_CACHE=1 \
+          --cache-from ghcr.io/dependabot/dependabot-core \
+          .
+    - name: Build dependabot-updater image
+      env:
+        DOCKER_BUILDKIT: 1
+        OMNIBUS_VERSION: latest
+      run: |
+        docker build \
+          -t "dependabot/updater:latest" \
+          --build-arg BUILDKIT_INLINE_CACHE=1 \
+          --cache-from ghcr.io/dependabot/dependabot-core \
+          --cache-from ghcr.io/dependabot/dependabot-updater/dependabot-updater \
+          --build-arg OMNIBUS_VERSION=$OMNIBUS_VERSION \
+          -f Dockerfile.updater \
+          .
+
+    - name: ${{ matrix.suite }}
+      env:
+        LOCAL_GITHUB_ACCESS_TOKEN: ${{ secrets.E2E_PAT }}
+      run: |
+        set -o pipefail
+        ./dependabot test -f=smoke.yaml -o=result.yaml --cache=cache --timeout=20m --updater-image=dependabot/updater:latest 2>&1 | tee -a log.txt
+
+    - name: Diff
+      if: always()
+      continue-on-error: true
+      run: diff --ignore-space-change smoke.yaml result.yaml && echo "Contents are identical"
+
+    - name: Create summary
+      run: tail -n100 log.txt | grep -P '\d+/\d+ calls cached \(\d+%\)' >> $GITHUB_STEP_SUMMARY
+
+    # No upload at the end:
+    # - If a test is uncachable in some regard, the cache would grow unbound.
+    # - We might want to consider erroring if the cache is changed.
@@ -7,6 +7,8 @@
 /dependabot-*.gem
 !bundler/spec/fixtures/projects/**/Gemfile.lock
 Gemfile.lock
+!updater/spec/fixtures/**/Gemfile.lock
+!updater/Gemfile.lock
 vendor
 !bundler/spec/fixtures/vendored_gems/vendor
 !common/spec/fixtures/projects/**/*/vendor

@@ -0,0 +1,58 @@
+ARG OMNIBUS_VERSION=required:fail_if_not_provided
+FROM dependabot/dependabot-core:$OMNIBUS_VERSION
+
+ENV DEPENDABOT_HOME /home/dependabot
+
+RUN mkdir $DEPENDABOT_HOME/dependabot-updater
+
+COPY --chown=dependabot:dependabot updater/Gemfile updater/Gemfile.lock $DEPENDABOT_HOME/dependabot-updater/
+
+COPY --chown=dependabot:dependabot .ruby-version ${DEPENDABOT_HOME}/.ruby-version
+COPY --chown=dependabot:dependabot .rubocop.yml ${DEPENDABOT_HOME}/.rubocop.yml
+
+WORKDIR ${DEPENDABOT_HOME}
+COPY --chown=dependabot:dependabot omnibus ${DEPENDABOT_HOME}/omnibus
+COPY --chown=dependabot:dependabot git_submodules ${DEPENDABOT_HOME}/git_submodules
+COPY --chown=dependabot:dependabot terraform ${DEPENDABOT_HOME}/terraform
+COPY --chown=dependabot:dependabot github_actions ${DEPENDABOT_HOME}/github_actions
+COPY --chown=dependabot:dependabot hex ${DEPENDABOT_HOME}/hex
+COPY --chown=dependabot:dependabot elm ${DEPENDABOT_HOME}/elm
+COPY --chown=dependabot:dependabot docker ${DEPENDABOT_HOME}/docker
+COPY --chown=dependabot:dependabot nuget ${DEPENDABOT_HOME}/nuget
+COPY --chown=dependabot:dependabot maven ${DEPENDABOT_HOME}/maven
+COPY --chown=dependabot:dependabot gradle ${DEPENDABOT_HOME}/gradle
+COPY --chown=dependabot:dependabot cargo ${DEPENDABOT_HOME}/cargo
+COPY --chown=dependabot:dependabot composer ${DEPENDABOT_HOME}/composer
+COPY --chown=dependabot:dependabot go_modules ${DEPENDABOT_HOME}/go_modules
+COPY --chown=dependabot:dependabot python ${DEPENDABOT_HOME}/python
+COPY --chown=dependabot:dependabot pub ${DEPENDABOT_HOME}/pub
+COPY --chown=dependabot:dependabot npm_and_yarn ${DEPENDABOT_HOME}/npm_and_yarn
+COPY --chown=dependabot:dependabot bundler ${DEPENDABOT_HOME}/bundler
+COPY --chown=dependabot:dependabot common ${DEPENDABOT_HOME}/common
+
+WORKDIR $DEPENDABOT_HOME/dependabot-updater
+
+RUN bundle config set --local path 'vendor' && \
+bundle config set --local without 'development' && \
+bundle install
+
+
+# START: HACKY WORKAROUND FOR NPM GIT INSTALLS SPAWNING CHILD PROCESS
+
+# TODO: Remove these hacks once we've deprecated npm 6 support as it no longer
+# spwans a child process to npm install git dependencies.
+
+# Create the config file manually intead of using yarn/npm config set as this
+# executes the package manager outputs to every job log
+COPY --chown=dependabot:dependabot updater/config/.yarnrc updater/config/.npmrc $DEPENDABOT_HOME/
+
+# END: HACKY WORKAROUND FOR NPM GIT INSTALLS SPAWNING CHILD PROCESS
+
+# Add project
+COPY --chown=dependabot:dependabot updater /home/dependabot/dependabot-updater
+
+# Fix for git vulnerability since we run as root
+# see https://github.blog/2022-04-12-git-security-vulnerability-announced/
+RUN git config --global --add safe.directory /home/dependabot/dependabot-updater/repo
+
+CMD ["bundle", "exec", "ruby", "bin/dependabot_update.rb"]