Skip to content

Commit e468998

Browse files
dimpasedimpase
authored
Merge branch 'develop' into yet_more_more_spkgs-confs
2 parents 1b8e6b8 + 1cf0c13 commit e468998

File tree

179 files changed

+1772
-1585
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

179 files changed

+1772
-1585
lines changed

‎.ci/merge-fixes.sh‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/sh
2+
# Merge open PRs from sagemath/sage labeled "blocker".
3+
REPO="sagemath/sage"
4+
GH="gh -R $REPO"
5+
PRs="$($GH pr list --label "p: blocker / 1" --json number --jq '.[].number')"
6+
if [ -z "$PRs" ]; then
7+
echo 'Nothing to do: Found no open PRs with "blocker" status.'
8+
else
9+
echo "Found PRs: " $PRs
10+
export GIT_AUTHOR_NAME="ci-sage workflow"
11+
export GIT_AUTHOR_EMAIL="[email protected]"
12+
export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
13+
export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
14+
git tag -f test_base
15+
git commit -q -m "Uncommitted changes" --no-allow-empty -a
16+
for a in $PRs; do
17+
echo "::group::Merging PR https://github.com/$REPO/pull/$a"
18+
git tag -f test_head
19+
$GH pr checkout -b pr-$a $a
20+
git fetch --unshallow --all
21+
git checkout -q test_head
22+
if git merge --no-edit --squash -q pr-$a; then
23+
echo "::endgroup::"
24+
if git commit -q -m "Merge https://github.com/$REPO/pull/$a" -a --no-allow-empty; then
25+
echo "Merged #$a"
26+
else
27+
echo "Empty, skipped"
28+
fi
29+
else
30+
echo "::endgroup::"
31+
echo "Failure merging #$a, resetting"
32+
git reset --hard
33+
fi
34+
done
35+
git log test_base..HEAD
36+
fi

‎.github/workflows/build.yml‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,30 @@ concurrency:
2424
cancel-in-progress: true
2525

2626
jobs:
27+
get_ci_fixes:
28+
runs-on: ubuntu-latest
29+
steps:
30+
- name: Checkout
31+
id: checkout
32+
uses: actions/checkout@v4
33+
- name: Merge CI fixes from sagemath/sage
34+
run: |
35+
.ci/merge-fixes.sh
36+
env:
37+
GH_TOKEN: ${{ github.token }}
38+
- name: Store CI fixes in upstream artifact
39+
run: |
40+
mkdir -p upstream
41+
git format-patch --stdout test_base > upstream/ci_fixes.patch
42+
- uses: actions/upload-artifact@v3
43+
with:
44+
path: upstream
45+
name: upstream
46+
2747
build:
2848
runs-on: ubuntu-latest
2949
container: ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}}
50+
needs: [get_ci_fixes]
3051
steps:
3152
- name: Checkout
3253
id: checkout
@@ -68,6 +89,19 @@ jobs:
6889
if [ ! -f worktree-image/.gitignore ]; then cp .gitignore worktree-image/; fi
6990
(cd worktree-image && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
7091
92+
- name: Download upstream artifact
93+
uses: actions/download-artifact@v3
94+
with:
95+
path: upstream
96+
name: upstream
97+
98+
- name: Apply CI fixes from sagemath/sage
99+
# After applying the fixes, make sure all changes are marked as uncommitted changes.
100+
run: |
101+
if [ -r upstream/ci_fixes.patch ]; then
102+
(cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch
103+
fi
104+
71105
- name: Incremental build, test changed files (sage -t --new)
72106
id: incremental
73107
run: |
@@ -76,15 +110,15 @@ jobs:
76110
./bootstrap && make build && ./sage -t --new -p2
77111
working-directory: ./worktree-image
78112
env:
79-
MAKE: make -j2
113+
MAKE: make -j2 --output-sync=recurse
80114
SAGE_NUM_THREADS: 2
81115

82116
- name: Build and test modularized distributions
83117
if: always() && steps.worktree.outcome == 'success'
84118
run: make V=0 tox && make pypi-wheels
85119
working-directory: ./worktree-image
86120
env:
87-
MAKE: make -j2
121+
MAKE: make -j2 --output-sync=recurse
88122
SAGE_NUM_THREADS: 2
89123

90124
- name: Set up node to install pyright
@@ -123,7 +157,7 @@ jobs:
123157
make build
124158
working-directory: ./worktree-image
125159
env:
126-
MAKE: make -j2
160+
MAKE: make -j2 --output-sync=recurse
127161
SAGE_NUM_THREADS: 2
128162

129163
- name: Pytest

‎.github/workflows/ci-conda.yml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,12 @@ jobs:
5151
steps:
5252
- uses: actions/checkout@v4
5353

54+
- name: Merge CI fixes from sagemath/sage
55+
run: |
56+
.ci/merge-fixes.sh
57+
env:
58+
GH_TOKEN: ${{ github.token }}
59+
5460
- name: Check for Miniconda
5561
id: check_conda
5662
run: echo ::set-output name=installed::$CONDA
@@ -99,9 +105,6 @@ jobs:
99105
echo "::remove-matcher owner=configure-system-package-warning::"
100106
echo "::remove-matcher owner=configure-system-package-error::"
101107
102-
# Manually install ptyprocess for now, until https://github.com/sagemath/sage/issues/32147 is fixed
103-
pip install --no-build-isolation -v -v ptyprocess==0.5.1
104-
105108
- name: Build
106109
shell: bash -l {0}
107110
run: |

‎.github/workflows/ci-macos.yml‎

Lines changed: 33 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -30,89 +30,40 @@ env:
3030
TARGETS_OPTIONAL: ptest
3131

3232
jobs:
33-
local-macos:
33+
stage-1:
34+
uses: ./.github/workflows/macos.yml
35+
with:
36+
stage: "1"
3437

35-
runs-on: ${{ matrix.os }}
36-
strategy:
37-
fail-fast: false
38-
matrix:
39-
stage: ["1", "2", "2-optional-0-o", "2-optional-p-z", "2-experimental-0-o", "2-experimental-p-z"]
40-
# python3_xcode is only accepted if enough packages are available from the system
41-
# --> to test "minimal", we would need https://github.com/sagemath/sage/issues/30949
42-
tox_env: [homebrew-macos-usrlocal-minimal, homebrew-macos-usrlocal-standard, homebrew-macos-usrlocal-maximal, homebrew-macos-usrlocal-python3_xcode-standard, conda-forge-macos-minimal, conda-forge-macos-standard, conda-forge-macos-maximal]
43-
xcode_version_factor: [default]
44-
os: [ macos-11, macos-12 ]
45-
env:
46-
TOX_ENV: local-${{ matrix.tox_env }}
47-
LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-tox-local-${{ matrix.tox_env }}-${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
48-
LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_env }}--${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
49-
steps:
50-
- uses: actions/checkout@v4
51-
- name: Select Xcode version
52-
run: |
53-
if [ ${{ matrix.xcode_version_factor }} != default ]; then sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode_version_factor }}.app; fi
54-
- name: Install test prerequisites
55-
run: |
56-
brew install tox
57-
- uses: actions/download-artifact@v3
58-
with:
59-
path: sage-local-artifact
60-
name: ${{ env.LOCAL_ARTIFACT_NAME }}
61-
if: contains(matrix.stage, '2')
62-
- name: Extract sage-local artifact
63-
# This is macOS tar -- cannot use --listed-incremental
64-
run: |
65-
export SAGE_LOCAL=$(pwd)/.tox/$TOX_ENV/local
66-
.github/workflows/extract-sage-local.sh sage-local-artifact/sage-local-*.tar
67-
if: contains(matrix.stage, '2')
68-
- name: Build and test with tox
69-
# We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
70-
# For doctesting, we use a lower parallelization to avoid timeouts.
71-
run: |
72-
case "${{ matrix.stage }}" in
73-
1) export TARGETS_PRE="all-sage-local" TARGETS="all-sage-local" TARGETS_OPTIONAL="build/make/Makefile"
74-
;;
75-
2) export TARGETS_PRE="all-sage-local" TARGETS="build doc-html" TARGETS_OPTIONAL="ptest"
76-
;;
77-
2-optional*) export TARGETS_PRE="build/make/Makefile" TARGETS="build/make/Makefile"
78-
targets_pattern="${{ matrix.stage }}"
79-
targets_pattern="${targets_pattern#2-optional-}"
80-
export TARGETS_OPTIONAL=$( echo $(export PATH=build/bin:$PATH && sage-package list :optional: --has-file 'spkg-install.in|spkg-install|requirements.txt' --no-file huge|has_nonfree_dependencies | grep -v sagemath_doc | grep "^[$targets_pattern]" ) )
81-
;;
82-
2-experimental*) export TARGETS_PRE="build/make/Makefile" TARGETS="build/make/Makefile"
83-
targets_pattern="${{ matrix.stage }}"
84-
targets_pattern="${targets_pattern#2-experimental-}"
85-
export TARGETS_OPTIONAL=$( echo $(export PATH=build/bin:$PATH && sage-package list :experimental: --has-file 'spkg-install.in|spkg-install|requirements.txt' --no-file huge|has_nonfree_dependencies | grep -v sagemath_doc | grep "^[$targets_pattern]" ) )
86-
;;
87-
esac
88-
MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
89-
- name: Prepare logs artifact
90-
run: |
91-
mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
92-
if: always()
93-
- uses: actions/upload-artifact@v3
94-
with:
95-
path: artifacts
96-
name: ${{ env.LOGS_ARTIFACT_NAME }}
97-
if: always()
98-
- name: Print out logs for immediate inspection
99-
# and markup the output with GitHub Actions logging commands
100-
run: |
101-
.github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
102-
if: always()
103-
- name: Prepare sage-local artifact
104-
# This also includes the copies of homebrew or conda installed in the tox environment.
105-
# We use absolute pathnames in the tar file.
106-
# This is macOS tar -- cannot use --remove-files.
107-
# We remove the $SAGE_LOCAL/lib64 link, which will be recreated by the next stage.
108-
run: |
109-
mkdir -p sage-local-artifact && (cd .tox/$TOX_ENV && rm -f "local/lib64" && tar -cf - $(pwd)) > sage-local-artifact/sage-${{ env.TOX_ENV }}-${{ matrix.stage }}.tar
110-
if: contains(matrix.stage, '1')
111-
- uses: actions/upload-artifact@v3
112-
with:
113-
path: sage-local-artifact/sage-${{ env.TOX_ENV }}-${{ matrix.stage }}.tar
114-
name: ${{ env.LOCAL_ARTIFACT_NAME }}
115-
if: always()
38+
stage-2:
39+
uses: ./.github/workflows/macos.yml
40+
with:
41+
stage: "2"
42+
needs: [stage-1]
43+
44+
stage-2-optional-0-o:
45+
uses: ./.github/workflows/macos.yml
46+
with:
47+
stage: "2-optional-0-o"
48+
needs: [stage-2]
49+
50+
stage-2-optional-p-z:
51+
uses: ./.github/workflows/macos.yml
52+
with:
53+
stage: "2-optional-p-z"
54+
needs: [stage-2-optional-0-o]
55+
56+
stage-2-experimental-0-o:
57+
uses: ./.github/workflows/macos.yml
58+
with:
59+
stage: "2-optional-0-o"
60+
needs: [stage-2-optional-p-z]
61+
62+
stage-2-experimental-p-z:
63+
uses: ./.github/workflows/macos.yml
64+
with:
65+
stage: "2-experimental-p-z"
66+
needs: [stage-2-experimental-0-o]
11667

11768
dist:
11869

‎.github/workflows/doc-build-pdf.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,12 @@ jobs:
2121
- name: Checkout
2222
uses: actions/checkout@v4
2323

24+
- name: Merge CI fixes from sagemath/sage
25+
run: |
26+
.ci/merge-fixes.sh
27+
env:
28+
GH_TOKEN: ${{ github.token }}
29+
2430
- name: Prepare
2531
run: |
2632
apt-get update && apt-get install -y zip

‎.github/workflows/doc-build.yml‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,30 @@ concurrency:
1212
cancel-in-progress: true
1313

1414
jobs:
15+
get_ci_fixes:
16+
runs-on: ubuntu-latest
17+
steps:
18+
- name: Checkout
19+
id: checkout
20+
uses: actions/checkout@v4
21+
- name: Merge CI fixes from sagemath/sage
22+
run: |
23+
.ci/merge-fixes.sh
24+
env:
25+
GH_TOKEN: ${{ github.token }}
26+
- name: Store CI fixes in upstream artifact
27+
run: |
28+
mkdir -p upstream
29+
git format-patch --stdout test_base > upstream/ci_fixes.patch
30+
- uses: actions/upload-artifact@v3
31+
with:
32+
path: upstream
33+
name: upstream
34+
1535
build-docs:
1636
runs-on: ubuntu-latest
1737
container: ghcr.io/sagemath/sage/sage-ubuntu-focal-standard-with-targets:dev
38+
needs: [get_ci_fixes]
1839
steps:
1940
- name: Checkout
2041
uses: actions/checkout@v4
@@ -54,14 +75,27 @@ jobs:
5475
# Keep track of changes to built HTML
5576
new_version=$(cat src/VERSION.txt); (cd /sage/local/share/doc/sage/html/en && find . -name "*.html" | xargs sed -i '/class="sidebar-brand-text"/s/Sage [0-9a-z.]* /Sage '$new_version' /'; git init && (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; git add -A && git commit --quiet -m "old")
5677
78+
- name: Download upstream artifact
79+
uses: actions/download-artifact@v3
80+
with:
81+
path: upstream
82+
name: upstream
83+
84+
- name: Apply CI fixes from sagemath/sage
85+
# After applying the fixes, make sure all changes are marked as uncommitted changes.
86+
run: |
87+
if [ -r upstream/ci_fixes.patch ]; then
88+
(cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch
89+
fi
90+
5791
- name: Incremental build
5892
id: incremental
5993
run: |
6094
# Now re-bootstrap and build. The build is incremental because we were careful with the timestamps.
6195
./bootstrap && make build
6296
working-directory: ./worktree-image
6397
env:
64-
MAKE: make -j2
98+
MAKE: make -j2 --output-sync=recurse
6599
SAGE_NUM_THREADS: 2
66100

67101
- name: Build (fallback to non-incremental)
@@ -72,7 +106,7 @@ jobs:
72106
make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status && make build
73107
working-directory: ./worktree-image
74108
env:
75-
MAKE: make -j2
109+
MAKE: make -j2 --output-sync=recurse
76110
SAGE_NUM_THREADS: 2
77111

78112
- name: Build docs
@@ -89,7 +123,7 @@ jobs:
89123
./config.status && make doc-html
90124
working-directory: ./worktree-image
91125
env:
92-
MAKE: make -j2
126+
MAKE: make -j2 --output-sync=recurse
93127
SAGE_NUM_THREADS: 2
94128

95129
- name: Copy docs

‎.github/workflows/docker.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,12 @@ jobs:
179179
(export PATH=$(pwd)/build/bin:$PATH; (cd upstream && bash -x update-pkgs.sh) && sed -i.bak '/upstream/d' .dockerignore && echo "/:toolchain:/i ADD upstream upstream" | sed -i.bak -f - build/bin/write-dockerfile.sh && git diff)
180180
if: inputs.upstream_artifact
181181

182+
- name: Merge CI fixes from sagemath/sage
183+
run: |
184+
.ci/merge-fixes.sh
185+
env:
186+
GH_TOKEN: ${{ github.token }}
187+
182188
- name: Try to login to ghcr.io
183189
if: inputs.docker_push_repository != ''
184190
# https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable

0 commit comments

Comments
 (0)