Skip to content

Commit 647f969

Browse files
pkgwpkgw
committed
ci: stub CI automation, finally
1 parent 56618cb commit 647f969

7 files changed

+428
-1
lines changed

‎.config/cranko/bootstrap.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@ qnames = [
33
'pwkit',
44
'pypa',
55
]
6-
version = '1.1.0'
6+
version = '1.0.0'

‎ci/azure-build-and-test.yml

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
# Main build jobs for CI/CD on Azure Pipelines.
2+
3+
parameters:
4+
- name: builds
5+
type: object
6+
default:
7+
8+
- name: linux_310
9+
vmImage: ubuntu-20.04
10+
vars:
11+
PYTHON_SERIES: "3.10"
12+
13+
- name: macos_310
14+
vmImage: macos-11
15+
vars:
16+
PYTHON_SERIES: "3.10"
17+
18+
jobs:
19+
- ${{ each build in parameters.builds }}:
20+
- job: ${{ format('build_{0}', build.name) }}
21+
pool:
22+
vmImage: ${{ build.vmImage }}
23+
variables:
24+
${{ insert }}: ${{ build.vars }}
25+
steps:
26+
27+
- template: azure-job-setup.yml
28+
parameters:
29+
setupBuild: true
30+
31+
- bash: |
32+
set -euo pipefail
33+
source activate-conda.sh
34+
set -x
35+
\conda create -y -n build setuptools pip python="$PYTHON_SERIES"
36+
conda activate build
37+
pip install $BASH_WORKSPACE/sdist/*.tar.gz
38+
displayName: Install from sdist
39+
40+
- job: docs
41+
pool:
42+
vmImage: ubuntu-latest
43+
steps:
44+
- template: azure-job-setup.yml
45+
parameters:
46+
setupBuild: true
47+
48+
- bash: |
49+
set -euo pipefail
50+
source activate-conda.sh
51+
set -x
52+
\conda create -y -n build setuptools pip python=3.8
53+
conda activate build
54+
pip install $BASH_WORKSPACE/sdist/*.tar.gz
55+
displayName: Install from sdist
56+
57+
- bash: |
58+
set -euo pipefail
59+
source activate-conda.sh
60+
conda activate build
61+
set -x
62+
\conda install -y astropy graphviz numpydoc sphinx sphinx-automodapi sphinx_rtd_theme
63+
cd docs
64+
make html
65+
make linkcheck
66+
displayName: Build docs

‎ci/azure-deployment.yml

+105
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
# Deployment workflows, run when all CI tests pass on the `master` or `rc
2+
# branches. NB: right now we do nothing on `master`! (But we need a dummy job to
3+
# keep Pipelines happy.) That could change.
4+
5+
parameters:
6+
- name: isMainDev
7+
type: boolean
8+
default: false
9+
- name: isRelease
10+
type: boolean
11+
default: false
12+
13+
jobs:
14+
15+
- ${{ if parameters.isMainDev }}:
16+
- job: dummy_setup_only
17+
pool:
18+
vmImage: ubuntu-latest
19+
steps:
20+
- template: azure-job-setup.yml
21+
22+
- ${{ if parameters.isRelease }}:
23+
- job: branch_and_tag
24+
pool:
25+
vmImage: ubuntu-latest
26+
variables:
27+
- group: Deployment Credentials
28+
steps:
29+
- template: azure-job-setup.yml
30+
parameters:
31+
setupCranko: true
32+
setupGit: true
33+
- bash: |
34+
set -xeou pipefail
35+
cranko release-workflow tag
36+
git push --tags origin release:release
37+
displayName: Tag and push
38+
env:
39+
GITHUB_TOKEN: $(GITHUB_TOKEN)
40+
41+
- job: github_releases
42+
dependsOn: branch_and_tag # otherwise, GitHub creates the tag itself
43+
pool:
44+
vmImage: ubuntu-latest
45+
variables:
46+
- group: Deployment Credentials
47+
steps:
48+
- template: azure-job-setup.yml
49+
parameters:
50+
setupCranko: true
51+
setupGit: true
52+
- bash: |
53+
set -xeou pipefail
54+
cranko github create-releases
55+
displayName: Create GitHub releases
56+
env:
57+
GITHUB_TOKEN: $(GITHUB_TOKEN)
58+
59+
- job: python_publish
60+
pool:
61+
vmImage: ubuntu-latest
62+
variables:
63+
- group: Deployment Credentials
64+
steps:
65+
- template: azure-job-setup.yml
66+
parameters:
67+
setupBuild: true
68+
setupCranko: true
69+
- bash: |
70+
set -eo pipefail # no -u to work around Conda activation bug (2021 August)
71+
source activate-conda.sh
72+
set -x
73+
conda install -y twine
74+
75+
cranko python install-token
76+
77+
# If we ever become a monorepo, we'll need to figure out how to make sure
78+
# that we only upload the proper packages.
79+
if cranko show if-released --exit-code pwkit ; then
80+
twine upload $BASH_WORKSPACE/sdist/*.tar.gz
81+
fi
82+
displayName: Publish PyPI releases
83+
env:
84+
PYPI_TOKEN: $(PYPI_TOKEN)
85+
86+
- job: zenodo_publish
87+
pool:
88+
vmImage: ubuntu-latest
89+
variables:
90+
- group: Deployment Credentials
91+
92+
steps:
93+
- template: azure-job-setup.yml
94+
parameters:
95+
setupCranko: true
96+
97+
- bash: cranko zenodo upload-artifacts --metadata=ci/zenodo.json5 $BASH_WORKSPACE/sdist/*.tar.gz
98+
displayName: Upload source tarball
99+
env:
100+
ZENODO_TOKEN: $(ZENODO_TOKEN)
101+
102+
- bash: cranko zenodo publish --metadata=ci/zenodo.json5
103+
displayName: Publish to Zenodo
104+
env:
105+
ZENODO_TOKEN: $(ZENODO_TOKEN)

‎ci/azure-job-setup.yml

+99
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
# Setup steps for the build and deployment processes.
2+
3+
parameters:
4+
- name: setupBuild
5+
type: boolean
6+
default: false
7+
8+
- name: setupCranko
9+
type: boolean
10+
default: false
11+
12+
- name: setupGit
13+
type: boolean
14+
default: false
15+
16+
steps:
17+
- download: current
18+
19+
- checkout: self
20+
submodules: recursive
21+
22+
- ${{ if parameters.setupCranko }}:
23+
- bash: |
24+
set -euo pipefail
25+
d="$(mktemp -d /tmp/cranko.XXXXXX)"
26+
cd "$d"
27+
curl --proto '=https' --tlsv1.2 -sSf https://pkgw.github.io/cranko/fetch-latest.sh | sh
28+
echo "##vso[task.prependpath]$d"
29+
displayName: Install latest Cranko (not Windows)
30+
condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'))
31+
32+
- pwsh: |
33+
$d = Join-Path $Env:Temp cranko-$(New-Guid)
34+
[void][System.IO.Directory]::CreateDirectory($d)
35+
cd $d
36+
[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072
37+
iex ((New-Object System.Net.WebClient).DownloadString('https://pkgw.github.io/cranko/fetch-latest.ps1'))
38+
echo "##vso[task.prependpath]$d"
39+
displayName: Install latest Cranko (Windows)
40+
condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
41+
42+
# work around https://github.com/microsoft/azure-pipelines-tasks/issues/10653
43+
# also note that `set -x` messes up `task.setvariable`
44+
- bash: |
45+
set -xeuo pipefail
46+
47+
BASH_WORKSPACE="$(Pipeline.Workspace)"
48+
ARTIFACT_STAGING="$(Build.ArtifactStagingDirectory)"
49+
50+
if [[ $AGENT_OS == Windows_NT ]] ; then
51+
BASH_WORKSPACE=$(echo "$BASH_WORKSPACE" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|')
52+
ARTIFACT_STAGING=$(echo "$ARTIFACT_STAGING" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|')
53+
fi
54+
55+
set +x
56+
echo "##vso[task.setvariable variable=BASH_WORKSPACE;]$BASH_WORKSPACE"
57+
echo "##vso[task.setvariable variable=ARTIFACT_STAGING;]$ARTIFACT_STAGING"
58+
displayName: Ensure bash-friendly variable names
59+
60+
- bash: |
61+
set -xeuo pipefail
62+
git switch -c release
63+
git pull --ff-only $BASH_WORKSPACE/git-release/release.bundle
64+
displayName: Restore release commit
65+
66+
- ${{ if parameters.setupBuild }}:
67+
- bash: |
68+
set -euo pipefail
69+
70+
if [[ $AGENT_OS == Windows_NT ]] ; then
71+
CONDA=$(echo "$CONDA" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|')
72+
condabin="$CONDA/Scripts"
73+
else
74+
condabin="$CONDA/bin"
75+
fi
76+
77+
if [[ $AGENT_OS == Darwin ]] ; then
78+
sudo chown -R $USER $CONDA
79+
fi
80+
81+
cat >activate-conda.sh <<EOF
82+
eval "\$($condabin/conda shell.bash hook)"
83+
conda activate
84+
EOF
85+
displayName: Set up Conda activation
86+
87+
- bash: |
88+
set -euo pipefail
89+
source activate-conda.sh
90+
set -x
91+
conda config --add channels conda-forge
92+
displayName: Activate conda-forge
93+
94+
- ${{ if parameters.setupGit }}:
95+
- bash: |
96+
cranko github install-credential-helper
97+
displayName: Set up Git pushes
98+
env:
99+
GITHUB_TOKEN: $(GITHUB_TOKEN)

‎ci/azure-pipelines.yml

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Copyright 2023 Peter Williams and collaborators
2+
# Licensed under the MIT License
3+
4+
trigger:
5+
branches:
6+
include:
7+
- master
8+
- rc
9+
10+
stages:
11+
- stage: SourceDist
12+
jobs:
13+
- template: azure-sdist.yml
14+
15+
- stage: MainBuild
16+
jobs:
17+
- template: azure-build-and-test.yml
18+
19+
- stage: Deploy
20+
condition: and(succeeded('MainBuild'), ne(variables['build.reason'], 'PullRequest'))
21+
jobs:
22+
- template: azure-deployment.yml
23+
parameters:
24+
isMainDev: ${{ ne(variables['Build.SourceBranchName'], 'rc') }}
25+
isRelease: ${{ eq(variables['Build.SourceBranchName'], 'rc') }}

‎ci/azure-sdist.yml

+86
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# First stage of CI: create the `sdist` and Cranko release commit.
2+
3+
jobs:
4+
5+
- job: sdist
6+
pool:
7+
vmImage: ubuntu-latest
8+
9+
# Need Zenodo credentials to generate DOIs during formal releases. Try not to
10+
# provide credentials otherwise, in case of malicious PRs. But note that such
11+
# a PR could just remove this filter anyway -- that's why modern CI systems
12+
# don't provide secrets during PR builds in the first place.
13+
${{ if and(eq(variables['Build.SourceBranchName'], 'rc'), ne(variables['build.reason'], 'PullRequest')) }}:
14+
variables:
15+
- group: Deployment Credentials
16+
17+
steps:
18+
19+
- checkout: self
20+
21+
- bash: |
22+
set -euo pipefail
23+
d="$(mktemp -d /tmp/cranko.XXXXXX)"
24+
cd "$d"
25+
curl --proto '=https' --tlsv1.2 -sSf https://pkgw.github.io/cranko/fetch-latest.sh | sh
26+
echo "##vso[task.prependpath]$d"
27+
displayName: Install latest Cranko
28+
29+
- bash: cranko release-workflow apply-versions
30+
displayName: Apply Cranko versions
31+
32+
- bash: cranko zenodo preregister --metadata=ci/zenodo.json5 pwkit CHANGELOG.md
33+
displayName: "Preregister Zenodo DOI"
34+
${{ if and(eq(variables['Build.SourceBranchName'], 'rc'), ne(variables['build.reason'], 'PullRequest')) }}:
35+
env:
36+
ZENODO_TOKEN: $(ZENODO_TOKEN)
37+
38+
- bash: |
39+
set -xeuo pipefail
40+
git add .
41+
cranko release-workflow commit
42+
git show
43+
displayName: Make release commit
44+
45+
# Note: in order for Cranko to work at all, and this step in particular,
46+
# "shallow" Git checkouts must be disabled in the project's Azure pipelines
47+
# configuration.
48+
- bash: |
49+
artifact_dir="$(Build.ArtifactStagingDirectory)/git-release"
50+
mkdir -p "$artifact_dir"
51+
git bundle create "$artifact_dir/release.bundle" origin/master..HEAD
52+
displayName: Bundle release commit
53+
54+
- task: PublishPipelineArtifact@1
55+
displayName: Publish release git bundle artifact
56+
inputs:
57+
targetPath: '$(Build.ArtifactStagingDirectory)/git-release'
58+
artifactName: git-release
59+
60+
# TODO: it seems that at the moment, there is no better way to create an
61+
# sdist than to invoke setup.py directly. People seem to be working on
62+
# alternatives but nothing is official yet.
63+
64+
- bash: |
65+
set -euo pipefail
66+
eval "$($CONDA/bin/conda shell.bash hook)"
67+
set -x
68+
conda activate
69+
conda config --add channels conda-forge
70+
conda install -y numpy
71+
displayName: Set up dependencies
72+
73+
- bash: |
74+
set -euo pipefail
75+
eval "$($CONDA/bin/conda shell.bash hook)"
76+
conda activate
77+
set -x
78+
python setup.py sdist
79+
tar tzf dist/*.tar.gz |sort
80+
displayName: Create sdist
81+
82+
- task: PublishPipelineArtifact@1
83+
displayName: Publish sdist artifact
84+
inputs:
85+
targetPath: dist
86+
artifactName: sdist

0 commit comments

Comments
 (0)