Skip to content

[Dev] Build 695 of branch main by @NGPixel #695

[Dev] Build 695 of branch main by @NGPixel

[Dev] Build 695 of branch main by @NGPixel #695

Workflow file for this run

name: Build and Release
run-name: ${{ github.ref_name == 'release' && '[Prod]' || '[Dev]' }} Build ${{ github.run_number }} of branch ${{ github.ref_name }} by @${{ github.actor }}
on:
push:
branches: [release]
workflow_dispatch:
inputs:
deploy:
description: 'Deploy to K8S'
default: 'Skip'
required: true
type: choice
options:
- Skip
- Staging Only
- Staging + Prod
sandbox:
description: 'Deploy to Sandbox'
default: true
required: true
type: boolean
sandboxNoDbRefresh:
description: 'Sandbox Disable Daily DB Refresh'
default: false
required: true
type: boolean
skiptests:
description: 'Skip Tests'
default: false
required: true
type: boolean
skiparm:
description: 'Skip ARM64 Build'
default: false
required: true
type: boolean
ignoreLowerCoverage:
description: 'Ignore Lower Coverage'
default: false
required: true
type: boolean
updateCoverage:
description: 'Update Baseline Coverage'
default: false
required: true
type: boolean
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
# -----------------------------------------------------------------
# PREPARE
# -----------------------------------------------------------------
prepare:
name: Prepare Release
runs-on: ubuntu-latest
outputs:
should_deploy: ${{ steps.buildvars.outputs.should_deploy }}
pkg_version: ${{ steps.buildvars.outputs.pkg_version }}
from_tag: ${{ steps.semver.outputs.nextStrict }}
to_tag: ${{ steps.semver.outputs.current }}
base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 1
fetch-tags: false
- name: Get Next Version (Prod)
if: ${{ github.ref_name == 'release' }}
id: semver
uses: ietf-tools/semver-action@v1
with:
token: ${{ github.token }}
branch: release
skipInvalidTags: true
- name: Get Dev Version
if: ${{ github.ref_name != 'release' }}
id: semverdev
uses: ietf-tools/semver-action@v1
with:
token: ${{ github.token }}
branch: release
skipInvalidTags: true
noVersionBumpBehavior: 'current'
noNewCommitBehavior: 'current'
- name: Set Release Flag
if: ${{ github.ref_name == 'release' }}
run: |
echo "IS_RELEASE=true" >> $GITHUB_ENV
- name: Create Draft Release
uses: ncipollo/[email protected]
if: ${{ github.ref_name == 'release' }}
with:
prerelease: true
draft: false
commit: ${{ github.sha }}
tag: ${{ steps.semver.outputs.nextStrict }}
name: ${{ steps.semver.outputs.nextStrict }}
body: '*pending*'
token: ${{ secrets.GITHUB_TOKEN }}
- name: Set Build Variables
id: buildvars
run: |
if [[ $IS_RELEASE ]]; then
echo "Using AUTO SEMVER mode: ${{ steps.semver.outputs.nextStrict }}"
echo "should_deploy=true" >> $GITHUB_OUTPUT
echo "pkg_version=${{ steps.semver.outputs.nextStrict }}" >> $GITHUB_OUTPUT
echo "::notice::Release ${{ steps.semver.outputs.nextStrict }} created using branch $GITHUB_REF_NAME"
else
echo "Using TEST mode: ${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER"
echo "should_deploy=false" >> $GITHUB_OUTPUT
echo "pkg_version=${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_OUTPUT
echo "::notice::Non-production build ${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER created using branch $GITHUB_REF_NAME"
fi
- name: Get Base Image Target Version
id: baseimgversion
run: |
echo "base_image_version=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT
# -----------------------------------------------------------------
# TESTS
# -----------------------------------------------------------------
tests:
name: Run Tests
uses: ./.github/workflows/tests.yml
if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
needs: [prepare]
with:
ignoreLowerCoverage: ${{ github.event.inputs.ignoreLowerCoverage == 'true' }}
skipSelenium: true
targetBaseVersion: ${{ needs.prepare.outputs.base_image_version }}
# -----------------------------------------------------------------
# RELEASE
# -----------------------------------------------------------------
release:
name: Make Release
if: ${{ !failure() && !cancelled() }}
needs: [tests, prepare]
runs-on: ubuntu-latest
permissions:
contents: write
packages: write
env:
SHOULD_DEPLOY: ${{needs.prepare.outputs.should_deploy}}
PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}
FROM_TAG: ${{needs.prepare.outputs.from_tag}}
TO_TAG: ${{needs.prepare.outputs.to_tag}}
TARGET_BASE: ${{needs.prepare.outputs.base_image_version}}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 1
fetch-tags: false
- name: Launch build VM
id: azlaunch
timeout-minutes: 10
run: |
echo "Authenticating to Azure..."
az login --service-principal -u ${{ secrets.AZ_BUILD_APP_ID }} -p ${{ secrets.AZ_BUILD_PWD }} --tenant ${{ secrets.AZ_BUILD_TENANT_ID }}
echo "Creating VM..."
vminfo=$(az vm create \
--resource-group ghaDatatracker \
--name tmpGhaBuildVM-${{ github.run_number }} \
--image Ubuntu2204 \
--admin-username azureuser \
--generate-ssh-keys \
--priority Spot \
--size Standard_D8ads_v5 \
--max-price -1 \
--ephemeral-os-disk \
--os-disk-size-gb 100 \
--eviction-policy Delete \
--nic-delete-option Delete \
--os-disk-delete-option Delete \
--output tsv \
--query "publicIpAddress")
echo "ipaddr=$vminfo" >> "$GITHUB_OUTPUT"
echo "VM Public IP: $vminfo"
cat ~/.ssh/id_rsa > ${{ github.workspace }}/prvkey.key
echo "Fetching SSH host public keys..."
until ssh-keyscan -t rsa $vminfo 2> /dev/null
do
echo "Will try again in 5 seconds..."
sleep 5
done
ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts
- name: Remote SSH into Build VM
uses: appleboy/ssh-action@25ce8cbbcb08177468c7ff7ec5cbfa236f9341e1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_ACTOR: ${{ github.actor }}
GITHUB_SHA: ${{ github.sha }}
GITHUB_REF_NAME: ${{ github.ref_name }}
GITHUB_RUN_ID: ${{ github.run_id }}
AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }}
AWS_DEFAULT_REGION: auto
AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }}
PKG_VERSION: ${{ env.PKG_VERSION }}
SHOULD_DEPLOY: ${{ env.SHOULD_DEPLOY }}
SKIP_TESTS: ${{ github.event.inputs.skiptests }}
DEBIAN_FRONTEND: noninteractive
BROWSERSLIST_IGNORE_OLD_DATA: 1
TARGET_BASE: ${{ env.TARGET_BASE }}
with:
host: ${{ steps.azlaunch.outputs.ipaddr }}
port: 22
username: azureuser
command_timeout: 60m
key_path: ${{ github.workspace }}/prvkey.key
envs: GITHUB_TOKEN,GITHUB_ACTOR,GITHUB_SHA,GITHUB_REF_NAME,GITHUB_RUN_ID,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_DEFAULT_REGION,AWS_ENDPOINT_URL,PKG_VERSION,SHOULD_DEPLOY,SKIP_TESTS,DEBIAN_FRONTEND,BROWSERSLIST_IGNORE_OLD_DATA
script_stop: true
script: |
export DEBIAN_FRONTEND=noninteractive
lsb_release -a
sudo apt-get update
sudo apt-get upgrade -y
sudo apt-get install wget unzip curl -y
echo "=========================================================================="
echo "Installing Docker..."
echo "=========================================================================="
curl -fsSL https://get.docker.com -o get-docker.sh
sudo sh get-docker.sh
sudo docker buildx create \
--name container-builder \
--driver docker-container \
--bootstrap --use
echo "=========================================================================="
echo "Login to ghcr.io..."
echo "=========================================================================="
echo $GITHUB_TOKEN | sudo docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
echo "=========================================================================="
echo "Installing GH CLI..."
echo "=========================================================================="
sudo mkdir -p -m 755 /etc/apt/keyrings \
&& wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
&& sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
&& sudo apt update \
&& sudo apt install gh -y
echo "=========================================================================="
echo "Installing AWS CLI..."
echo "=========================================================================="
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip
sudo ./aws/install
echo "=========================================================================="
echo "Install Node.js..."
echo "=========================================================================="
curl -fsSL https://deb.nodesource.com/setup_18.x -o nodesource_setup.sh
sudo bash nodesource_setup.sh
sudo apt-get install -y nodejs
sudo corepack enable
echo "=========================================================================="
echo "Install Python 3.x..."
echo "=========================================================================="
sudo apt-get install python3 python3-dev -y
python3 --version
echo "=========================================================================="
echo "Clone project..."
echo "=========================================================================="
sudo mkdir -p /workspace
sudo chown azureuser /workspace
cd /workspace
gh repo clone ietf-tools/datatracker -- --depth=1 --no-tags
cd datatracker
if [ "$SKIP_TESTS" = "false" ] || [ "$GITHUB_REF_NAME" = "release" ] ; then
echo "=========================================================================="
echo "Downloading coverage..."
echo "=========================================================================="
gh run download $GITHUB_RUN_ID -n coverage
fi
echo "=========================================================================="
echo "Building project..."
echo "=========================================================================="
echo "PKG_VERSION: $PKG_VERSION"
echo "GITHUB_SHA: $GITHUB_SHA"
echo "GITHUB_REF_NAME: $GITHUB_REF_NAME"
echo "Running frontend build script..."
echo "Compiling native node packages..."
yarn rebuild
echo "Packaging static assets..."
yarn build --base=https://static.ietf.org/dt/$PKG_VERSION/
yarn legacy:build
echo "Setting version $PKG_VERSION..."
sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py
sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py
sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py
if [ "$SHOULD_DEPLOY" = "true" ] ; then
echo "=========================================================================="
echo "Setting production flags in settings.py..."
echo "=========================================================================="
sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py
fi
echo "=========================================================================="
echo "Build release tarball..."
echo "=========================================================================="
mkdir -p /workspace/release
tar -czf /workspace/release.tar.gz -X dev/build/exclude-patterns.txt .
echo "=========================================================================="
echo "Collecting statics..."
echo "=========================================================================="
echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }}"
sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }} sh dev/build/collectstatics.sh
echo "Pushing statics..."
cd static
aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors
cd ..
echo "=========================================================================="
echo "Augment dockerignore for docker image build..."
echo "=========================================================================="
cat >> .dockerignore <<EOL
.devcontainer
.github
.vscode
k8s
playwright
svn-history
docker-compose.yml
EOL
echo "=========================================================================="
echo "Building Images..."
echo "=========================================================================="
sudo docker buildx build --file dev/build/Dockerfile --platform linux/amd64,linux/arm64 --tag ghcr.io/ietf-tools/datatracker:$PKG_VERSION --push .
- name: Fetch release tarball
run: |
mkdir -p /home/runner/work/release
chmod 0600 ${{ github.workspace }}/prvkey.key
scp -i ${{ github.workspace }}/prvkey.key azureuser@${{ steps.azlaunch.outputs.ipaddr }}:/workspace/release.tar.gz /home/runner/work/release/release.tar.gz
- name: Destroy Build VM + resources
if: always()
shell: pwsh
run: |
echo "Destroying VM..."
az vm delete -g ghaDatatracker -n tmpGhaBuildVM-${{ github.run_number }} --yes --force-deletion true
# $resourceOrderRemovalOrder = [ordered]@{
# "Microsoft.Compute/virtualMachines" = 0
# "Microsoft.Compute/disks" = 1
# "Microsoft.Network/networkInterfaces" = 2
# "Microsoft.Network/publicIpAddresses" = 3
# "Microsoft.Network/networkSecurityGroups" = 4
# "Microsoft.Network/virtualNetworks" = 5
# }
# echo "Fetching remaining resources..."
# $resources = az resource list --resource-group ghaDatatracker | ConvertFrom-Json
# $orderedResources = $resources
# | Sort-Object @{
# Expression = {$resourceOrderRemovalOrder[$_.type]}
# Descending = $False
# }
# echo "Deleting remaining resources..."
# $orderedResources | ForEach-Object {
# az resource delete --resource-group ghaDatatracker --ids $_.id --verbose
# }
echo "Logout from Azure..."
az logout
- name: Update CHANGELOG
id: changelog
uses: Requarks/changelog-action@v1
if: ${{ env.SHOULD_DEPLOY == 'true' }}
with:
token: ${{ github.token }}
fromTag: ${{ env.FROM_TAG }}
toTag: ${{ env.TO_TAG }}
writeToFile: false
- name: Download Coverage Results
if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
uses: actions/[email protected]
with:
name: coverage
- name: Prepare Coverage Action
if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
working-directory: ./dev/coverage-action
run: npm install
- name: Process Coverage Stats + Chart
id: covprocess
uses: ./dev/coverage-action/
if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
with:
token: ${{ github.token }}
tokenCommon: ${{ secrets.GH_COMMON_TOKEN }}
repoCommon: common
version: ${{needs.prepare.outputs.pkg_version}}
changelog: ${{ steps.changelog.outputs.changes }}
summary: ''
coverageResultsPath: coverage.json
histCoveragePath: historical-coverage.json
- name: Create Release
uses: ncipollo/[email protected]
if: ${{ env.SHOULD_DEPLOY == 'true' }}
with:
allowUpdates: true
makeLatest: true
draft: false
tag: ${{ env.PKG_VERSION }}
name: ${{ env.PKG_VERSION }}
body: ${{ steps.covprocess.outputs.changelog }}
artifacts: "/home/runner/work/release/release.tar.gz,coverage.json,historical-coverage.json"
token: ${{ secrets.GITHUB_TOKEN }}
- name: Update Baseline Coverage
uses: ncipollo/[email protected]
if: ${{ github.event.inputs.updateCoverage == 'true' || github.ref_name == 'release' }}
with:
allowUpdates: true
tag: baseline
omitBodyDuringUpdate: true
omitNameDuringUpdate: true
omitPrereleaseDuringUpdate: true
replacesArtifacts: true
artifacts: "coverage.json"
token: ${{ secrets.GITHUB_TOKEN }}
- name: Upload Build Artifacts
uses: actions/upload-artifact@v4
with:
name: release-${{ env.PKG_VERSION }}
path: /home/runner/work/release/release.tar.gz
# -----------------------------------------------------------------
# NOTIFY
# -----------------------------------------------------------------
notify:
name: Notify
if: ${{ always() }}
needs: [prepare, tests, release]
runs-on: ubuntu-latest
env:
PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}
steps:
- name: Notify on Slack (Success)
if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
uses: slackapi/[email protected]
with:
channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}
payload: |
{
"text": "Datatracker Build <https://github.com/ietf-tools/datatracker/actions/runs/${{ github.run_id }}|${{ env.PKG_VERSION }}> by ${{ github.triggering_actor }} - <@${{ secrets.SLACK_UID_RJSPARKS }}>",
"attachments": [
{
"color": "28a745",
"fields": [
{
"title": "Status",
"short": true,
"value": "Completed"
}
]
}
]
}
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }}
- name: Notify on Slack (Failure)
if: ${{ contains(join(needs.*.result, ','), 'failure') }}
uses: slackapi/[email protected]
with:
channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}
payload: |
{
"text": "Datatracker Build <https://github.com/ietf-tools/datatracker/actions/runs/${{ github.run_id }}|${{ env.PKG_VERSION }}> by ${{ github.triggering_actor }} - <@${{ secrets.SLACK_UID_RJSPARKS }}>",
"attachments": [
{
"color": "a82929",
"fields": [
{
"title": "Status",
"short": true,
"value": "Failed"
}
]
}
]
}
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }}
# -----------------------------------------------------------------
# SANDBOX
# -----------------------------------------------------------------
sandbox:
name: Deploy to Sandbox
if: ${{ !failure() && !cancelled() && github.event.inputs.sandbox == 'true' }}
needs: [prepare, release]
runs-on: [self-hosted, dev-server]
environment:
name: sandbox
env:
PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}
steps:
- uses: actions/checkout@v4
- name: Download a Release Artifact
uses: actions/[email protected]
with:
name: release-${{ env.PKG_VERSION }}
- name: Deploy to containers
env:
DEBIAN_FRONTEND: noninteractive
run: |
echo "Reset production flags in settings.py..."
sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = True/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'development'/" ietf/settings.py
echo "Install Deploy to Container CLI dependencies..."
cd dev/deploy-to-container
npm ci
cd ../..
echo "Start Deploy..."
node ./dev/deploy-to-container/cli.js --branch ${{ github.ref_name }} --domain dev.ietf.org --appversion ${{ env.PKG_VERSION }} --commit ${{ github.sha }} --ghrunid ${{ github.run_id }} --nodbrefresh ${{ github.event.inputs.sandboxNoDbRefresh }}
- name: Cleanup old docker resources
env:
DEBIAN_FRONTEND: noninteractive
run: |
docker image prune -a -f
# -----------------------------------------------------------------
# STAGING
# -----------------------------------------------------------------
staging:
name: Deploy to Staging
if: ${{ !failure() && !cancelled() && (github.event.inputs.deploy == 'Staging Only' || github.event.inputs.deploy == 'Staging + Prod' || github.ref_name == 'release') }}
needs: [prepare, release]
runs-on: ubuntu-latest
environment:
name: staging
env:
PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}
steps:
- name: Deploy to staging
uses: the-actions-org/workflow-dispatch@v4
with:
workflow: deploy.yml
repo: ietf-tools/infra-k8s
ref: main
token: ${{ secrets.GH_INFRA_K8S_TOKEN }}
inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }'
wait-for-completion: true
wait-for-completion-timeout: 10m
wait-for-completion-interval: 30s
display-workflow-run-url: false
# -----------------------------------------------------------------
# PROD
# -----------------------------------------------------------------
prod:
name: Deploy to Production
if: ${{ !failure() && !cancelled() && (github.event.inputs.deploy == 'Staging + Prod' || github.ref_name == 'release') }}
needs: [prepare, staging]
runs-on: ubuntu-latest
environment:
name: production
env:
PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}
steps:
- name: Deploy to production
uses: the-actions-org/workflow-dispatch@v4
with:
workflow: deploy.yml
repo: ietf-tools/infra-k8s
ref: main
token: ${{ secrets.GH_INFRA_K8S_TOKEN }}
inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }'
wait-for-completion: true
wait-for-completion-timeout: 10m
wait-for-completion-interval: 30s
display-workflow-run-url: false