Skip to content

Nightlies

Nightlies #1662

Workflow file for this run

name: Nightlies
on:
push:
branches:
- master
- actions
pull_request:
branches:
- '*'
schedule:
- cron: '0 0 * * *'
jobs:
setup:
name: 'build settings'
runs-on: ubuntu-latest
outputs:
settings: ${{ steps.settings.outputs.settings }}
deploy: ${{ steps.settings.outputs.deploy }}
env:
# Bump this value when a rebuild of nightlies (of the
# same compiler commits) has to be done.
nightlies_revision: 4
steps:
- name: Checkout nightlies
uses: actions/checkout@v4
with:
path: nightlies
- name: Generate version matrix
shell: bash
run: |
# Tracked branches
branches=('devel' 'version-2-2' 'version-2-0')
getHash() {
git ls-remote "https://github.com/$1" "$2" | cut -f 1
}
{
for branch in "${branches[@]}"; do
jq --null-input \
--arg branch "$branch" \
--arg commit "$(getHash nim-lang/Nim "$branch")" \
'{ branch: $branch, commit: $commit }'
done
} | jq -s '.' | tee versions.json
- name: Restore build settings
uses: actions/cache@v4
with:
path: settings
key: build-settings-${{ hashFiles('versions.json') }}-${{ env.nightlies_revision }}
restore-keys: build-settings-
- name: Generate build settings
id: settings
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
deploy=false
case '${{ github.event_name }}' in
schedule)
deploy=true
;;
push)
message=$(
cd nightlies
git show -s --format=%s '${{ github.ref }}'
)
if [[ $message == *'[deploy]'* ]]; then
deploy=true
else
deploy=false
fi
esac
declare -A refs
while IFS='=' read -r branch commit; do
refs[$branch]=$commit
done <<< "$(jq -r '.[] | .branch + "=" + .commit' versions.json)"
mkdir -p settings
# Delete older branch settings (if they were restored)
find settings -mindepth 1 -maxdepth 1 $(printf "! -name %q.json " "${!refs[@]}")
declare -A environment=(
[SOURCE_CODE_EPOCH]=$(date -u +%s)
)
for branch in "${!refs[@]}"; do
savedCommit=
savedNightliesRev=
savedBuildRevision=
commit=${refs[$branch]}
buildRevision=0
if [[ -e "settings/$branch.json" ]]; then
savedSettings=$(< "settings/$branch.json")
savedCommit=$(jq -r '.commit' <<< "$savedSettings")
savedNightliesRev=$(jq '.nightlies_revision // 0' <<< "$savedSettings")
savedBuildRevision=$(jq '.build_revision // -1' <<< "$savedSettings")
if [[ $savedNightliesRev -ne $nightlies_revision ]]; then
buildRevision=$(($savedBuildRevision + 1))
fi
fi
if [[ $savedCommit != "$commit" || $savedNightliesRev -ne $nightlies_revision ]]; then
echo "::group::Generating build settings for branch $branch"
{
{
for var in "${!environment[@]}"; do
jq --null-input \
--arg variable "$var" \
--arg value "${environment[$var]}" \
'{ ($variable): $value }'
done
} | jq -s '{ environment: (reduce .[] as $item (null; . + $item)) }'
tag=$(date -u --date="@${environment[SOURCE_CODE_EPOCH]}" +%F)-$branch-$commit
# Don't add revision to tags for now.
# if [[ $buildRevision -gt 0 ]]; then
# tag=$tag-$buildRevision
# fi
jq --null-input --arg tag "$tag" '{ release: $tag }'
jq --null-input --argjson nightlies_revision "$nightlies_revision" '{ nightlies_revision: $nightlies_revision }'
jq --null-input --argjson build_revision "$buildRevision" '{ build_revision: $build_revision }'
jq --null-input --arg csources "nim-lang/csources_v2" '{ csources: $csources }'
} | jq -s --arg commit "$commit" 'reduce .[] as $item ({ commit: $commit }; . + $item)' > "settings/$branch.json"
else
echo "::group::Stored settings for branch $branch @ $commit found."
fi
jq --arg branch "$branch" '.[] | select(.branch == $branch)' versions.json | jq -s 'add' - "settings/$branch.json" | tee -a settings.json
echo "::endgroup::"
done
echo "settings=$(jq -sc '.' settings.json)" >> $GITHUB_OUTPUT
echo "deploy=$deploy" >> $GITHUB_OUTPUT
sourceArchive:
needs: setup
strategy:
fail-fast: false
matrix:
setting: ${{ fromJson(needs.setup.outputs.settings) }}
name: 'source (${{ matrix.setting.branch }}, ${{ matrix.setting.commit }})'
runs-on: ubuntu-latest
env: ${{ matrix.setting.environment }}
steps:
- name: Checkout nightlies
uses: actions/checkout@v4
with:
path: nightlies
- name: 'Install dependencies'
run: |
sudo apt-get update
sudo apt-get install -y hub
- name: Restore Nim from cache
id: nim-cache
uses: actions/cache@v4
with:
path: nim/output/nim-*.tar.xz
key: 'source-${{ matrix.setting.commit }}-${{ matrix.setting.build_revision }}'
- name: Checkout Nim
if: steps.nim-cache.outputs.cache-hit != 'true'
uses: actions/checkout@v4
with:
repository: nim-lang/Nim
ref: ${{ matrix.setting.commit }}
path: nim
- name: Get csources version
if: steps.nim-cache.outputs.cache-hit != 'true'
id: csources-version
shell: bash
run: |
csources_repo=${{ matrix.setting.csources }}
# if one is not set, use csources_v2
: "${csources_repo:=nim-lang/csources_v2}"
csources_commit=$(git ls-remote "https://github.com/$csources_repo" master | cut -f 1)
if [[ -f nim/config/build_config.txt ]]; then
. nim/config/build_config.txt
csources_repo=${nim_csourcesUrl#https://github.com/}
csources_repo=${csources_repo%.git}
csources_commit=$nim_csourcesHash
fi
echo "repo=$csources_repo" >> $GITHUB_OUTPUT
echo "commit=$csources_commit" >> $GITHUB_OUTPUT
- name: Restore csources from cache
if: steps.nim-cache.outputs.cache-hit != 'true'
id: csources-cache
uses: actions/cache@v4
with:
path: csources/bin
key: >
csources-${{ runner.os }}-${{ steps.csources-version.outputs.repo }}-${{ steps.csources-version.outputs.commit }}
- name: Checkout csources
if: >
steps.nim-cache.outputs.cache-hit != 'true' &&
steps.csources-cache.outputs.cache-hit != 'true'
uses: actions/checkout@v4
with:
repository: ${{ steps.csources-version.outputs.repo }}
ref: ${{ steps.csources-version.outputs.commit }}
path: csources
- name: Setup environment
shell: bash
run: echo '${{ github.workspace }}/nim/bin' >> "$GITHUB_PATH"
- name: Build 1-stage csources compiler
if: steps.nim-cache.outputs.cache-hit != 'true'
shell: bash
run: |
if [[ ! -e csources/bin/nim ]]; then
make -C csources -j $(nproc) CC=gcc
else
echo 'Using prebuilt csources'
fi
cp csources/bin/nim nim/bin
- name: Build compiler
if: steps.nim-cache.outputs.cache-hit != 'true'
shell: bash
run: |
cd nim
nim c koch
./koch boot -d:release
- name: Generate csources
if: steps.nim-cache.outputs.cache-hit != 'true'
shell: bash
run: |
cd nim
./koch csources -d:danger '-d:gitHash:${{ matrix.setting.commit }}'
- name: Bundle external sources
if: steps.nim-cache.outputs.cache-hit != 'true'
shell: bash
run: |
cd nim
version=$(nim secret --hints:off <<< 'echo NimVersion; quit 0')
major=${version%%.*}
minor=${version#*.}
minor=${minor%.*}
patch=${version##*.}
./koch nimble
# Only Nim >= 1.4.0 bundles fusion.
if [[ $major -ge 1 && $minor -ge 4 ]]; then
./koch fusion
fi
- name: Build source archive
if: steps.nim-cache.outputs.cache-hit != 'true'
shell: bash
run: |
cd nim
./koch xz
mkdir -p output
source=$(basename build/nim-*.tar.xz)
version=${source%.tar.xz}
version=${version#nim-}
cp build/$source output/nim-$version.tar.xz
- name: Publish release
if: needs.setup.outputs.deploy == 'true'
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
if ! hub -C nightlies release show '${{ matrix.setting.release }}' >/dev/null 2>&1; then
cat << EOF | hub -C nightlies release create -a nim/output/* -F - '${{ matrix.setting.release }}' >/dev/null
Nightly build on $(date -u --date="@$SOURCE_CODE_EPOCH" "+%F") for branch ${{ matrix.setting.branch }}
Commit: https://github.com/nim-lang/Nim/commit/${{ matrix.setting.commit }}
Generated release binaries will be uploaded as they're made available.
EOF
else
echo "Release '${{ matrix.setting.release }}' has already been created, skipping."
fi
- name: Upload source archive to artifacts
uses: actions/upload-artifact@v3
with:
name: 'nim-${{ matrix.setting.commit }}'
path: nim/output/*
build:
needs: [ setup, sourceArchive ]
strategy:
fail-fast: false
matrix:
setting: ${{ fromJson(needs.setup.outputs.settings) }}
target:
- os: linux
triple: x86_64-linux-musl
- os: linux
triple: i686-linux-musl
- os: linux
triple: aarch64-linux-musl
- os: linux
triple: armv7l-linux-musleabihf
- os: macosx
triple: x86_64-apple-darwin14
- os: windows
triple: x86_64-w64-mingw32
- os: windows
triple: i686-w64-mingw32
include:
- target:
os: linux
builder: ubuntu-20.04
- target:
os: macosx
builder: macos-13
- target:
os: windows
builder: windows-2019
env: ${{ matrix.setting.environment }}
name: '${{ matrix.target.triple }} (${{ matrix.setting.branch }}, ${{ matrix.setting.commit }})'
runs-on: ${{ matrix.builder }}
steps:
- name: Checkout build scripts
uses: actions/checkout@v4
with:
path: nightlies
- name: Cache build outputs
id: built
uses: actions/cache@v4
with:
path: output
key: >
output-${{ hashFiles('nightlies/lib.sh') }}-${{ hashFiles('nightlies/build-release.sh') }}-${{ matrix.target.triple }}-${{ matrix.setting.commit }}-${{ matrix.setting.build_revision }}
- name: Cache dependencies
if: steps.built.outputs.cache-hit != 'true'
uses: actions/cache@v4
with:
path: external
key: >
deps-${{ hashFiles('nightlies/lib.sh') }}-${{ hashFiles('nightlies/deps.sh') }}-${{ hashFiles('nightlies/buildreq.txt') }}-${{ runner.os }}-${{ matrix.target.triple }}-${{ matrix.setting.build_revision }}
- name: Install dependencies
if: steps.built.outputs.cache-hit != 'true'
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: nightlies/deps.sh -t '${{ matrix.target.triple }}'
- name: Download generated source package
if: steps.built.outputs.cache-hit != 'true'
uses: actions/download-artifact@v3
with:
name: 'nim-${{ matrix.setting.commit }}'
path: source
- name: Extract source package
if: steps.built.outputs.cache-hit != 'true'
id: source
shell: bash
run: |
source=( source/nim-*.tar.xz )
version=${source[0]##*nim-}
version=${version%%.tar.xz}
case '${{ runner.os }}' in
'Windows')
7z x -so "${source[0]}" | 7z x -si -ttar -aoa
;;
*)
tar xJf "${source[0]}"
;;
esac
echo "version=$version" >> $GITHUB_OUTPUT
- name: Setup environment
if: steps.built.outputs.cache-hit != 'true'
shell: bash
run: |
echo "-d:gitHash:\"${{ matrix.setting.commit }}\"" >> external/nim.cfg
- name: Build release binaries
if: steps.built.outputs.cache-hit != 'true'
shell: bash
run: |
nightlies/build-release.sh 'nim-${{ steps.source.outputs.version }}'
- name: Prepare binaries for uploads
id: release
shell: bash
run: |
source nightlies/lib.sh
artifact=$(< output/nim.txt)
echo "artifact=$artifact" >> $GITHUB_OUTPUT
# Github Actions work based on native Windows path, so we're doing
# some quick conversions here.
echo "artifact_nativepath=$(nativepath "$artifact")" >> $GITHUB_OUTPUT
- name: Upload release binaries
if: needs.setup.outputs.deploy == 'true'
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
cd nightlies
if ! gh release view '${{ matrix.setting.release }}' | grep "$(basename '${{ steps.release.outputs.artifact }}')" >/dev/null 2>&1; then
gh release upload '${{ matrix.setting.release }}' '${{ steps.release.outputs.artifact }}'
else
echo "Binary already released for tag '${{ matrix.setting.release }}', not overwritting."
fi
- name: Upload binaries to artifacts
uses: actions/upload-artifact@v3
with:
name: 'binaries-${{ matrix.setting.branch }}-${{ matrix.setting.commit }}'
path: '${{ steps.release.outputs.artifact_nativepath }}'
latestTag:
needs: [ setup, build ]
strategy:
fail-fast: false
matrix:
setting: ${{ fromJson(needs.setup.outputs.settings) }}
if: needs.setup.outputs.deploy == 'true'
name: 'Update latest tags for ${{ matrix.setting.release }}'
runs-on: ubuntu-latest
steps:
- name: Store build settings
shell: bash
run: |
cat <<< '${{ toJson(matrix.setting) }}' > setting.json
- name: Check whether deployment was done for this setting set
id: deploy-cache
uses: actions/cache@v4
with:
path: setting.json
key: deploy-${{ hashFiles('setting.json') }}-${{ env.nightlies_revision }}
- name: Checkout nightlies
if: steps.deploy-cache.outputs.cache-hit != 'true'
uses: actions/checkout@v4
with:
path: nightlies
- name: Download generated source package
if: steps.deploy-cache.outputs.cache-hit != 'true'
uses: actions/download-artifact@v3
with:
name: 'nim-${{ matrix.setting.commit }}'
path: source
- name: Download built binaries from artifacts
if: steps.deploy-cache.outputs.cache-hit != 'true'
uses: actions/download-artifact@v3
with:
name: 'binaries-${{ matrix.setting.branch }}-${{ matrix.setting.commit }}'
path: binaries
- name: 'Push latest-${{ matrix.setting.branch }} tag'
if: steps.deploy-cache.outputs.cache-hit != 'true'
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
tag='latest-${{ matrix.setting.branch }}'
mkdir -p assets
source=("$GITHUB_WORKSPACE"/source/*)
# Rename the source archive as source.tar.xz
mv -v "${source[0]}" "assets/source.tar.xz"
for artifact in binaries/*; do
# Trim version from artifact name
shortName=${artifact#*nim-*-}
mv -v "$artifact" "assets/$shortName"
done
cat << EOF > release-notes.md
This release is an alias for the latest successful nightly build \
for branch \`${{ matrix.setting.branch }}\`: https://github.com/${{ github.repository }}/releases/tag/${{ matrix.setting.release }}
Each \`latest-\` alias is guaranteed to contain binaries for all \
supported architectures.
EOF
cd nightlies
# By deleting and recreating the tag, Github recognize it as the true
# "latest" tags.
echo "Deleting release $tag (errors are ignored)"
gh release delete -y "$tag" || :
git push --delete origin "$tag" || :
echo "Publishing $tag"
gh release create \
-t 'Latest successful build for branch ${{ matrix.setting.branch }}' \
-F "$GITHUB_WORKSPACE/release-notes.md" \
"$tag" \
"$GITHUB_WORKSPACE/assets"/*