Skip to content

Commit 144dd4a

Browse files
tobiasdieztobiasdiez
authored
Merge remote-tracking branch 'upstream/develop' into nep29
2 parents 4c3d853 + e7477f8 commit 144dd4a

File tree

5,849 files changed

+199793
-701075
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

5,849 files changed

+199793
-701075
lines changed

‎.ci/create-changes-html.sh‎

Lines changed: 56 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,97 +1,125 @@
11
#!/bin/sh
22
if [ $# != 2 ]; then
3-
echo >&2 "usage: $0 BASE_DOC_COMMIT DOC_REPO"
4-
echo >&2 "creates CHANGES.html in the current directory"
5-
echo >&2 "for the diffs of DOC_REPO against BASE_DOC_COMMIT"
3+
echo >&2 "Usage: $0 DIFF_TEXT DOC_REPO"
4+
echo >&2 "This script generates a CHANGES.html file in the current directory"
5+
echo >&2 "and adds anchor targets in the documents within DOC_REPO"
6+
echo >&2 "based on the diff hunks in the DIFF_TEXT file."
67
exit 1
78
fi
8-
BASE_DOC_COMMIT="$1"
9+
DIFF_TEXT="$1"
910
DOC_REPOSITORY="$2"
1011

11-
# Wipe out chronic diffs between old doc and new doc
12-
(cd $DOC_REPOSITORY && find . -name "*.html" | xargs sed -i -e '\;<script type="application/vnd\.jupyter\.widget-state+json">;,\;</script>; d')
1312
# Create CHANGES.html
1413
echo '<html>' > CHANGES.html
1514
echo '<head>' >> CHANGES.html
1615
echo '<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/default.min.css">' >> CHANGES.html
1716
echo '<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>' >> CHANGES.html
1817
echo '<script>hljs.highlightAll();</script>' >> CHANGES.html
1918
cat >> CHANGES.html << EOF
19+
<style>
20+
p.diff a:first-child {
21+
font-weight: bold;
22+
font-size: x-large;
23+
}
24+
</style>
2025
<script>
2126
document.addEventListener('DOMContentLoaded', () => {
27+
// This URL is hardcoded in the file .github/workflows/doc-publish.yml.
28+
// See NETLIFY_ALIAS of the "Deploy to Netlify" step.
29+
const baseDocURL = 'https://doc-develop--sagemath.netlify.app'
2230
const diffSite = 'https://pianomister.github.io/diffsite'
23-
const baseDocURL = 'https://sagemath-tobias.netlify.app'
2431
const diffParagraphs = document.querySelectorAll('p.diff');
2532
diffParagraphs.forEach(paragraph => {
2633
const rootURL = window.location.origin;
27-
const docAnchor = paragraph.querySelector('a'); // first "a" element
34+
const docAnchor = paragraph.querySelector('a');
2835
const url = new URL(docAnchor.href);
2936
const path = url.pathname;
3037
const anchor = document.createElement('a');
3138
anchor.href = diffSite + '/?url1=' + rootURL + path + '&url2=' + baseDocURL + path;
3239
anchor.textContent = 'compare with the base';
3340
anchor.setAttribute('target', '_blank');
41+
paragraph.innerHTML += '&nbsp;&nbsp;';
3442
paragraph.appendChild(anchor);
35-
paragraph.innerHTML += '&nbsp;';
36-
const hunkAnchors = paragraph.querySelectorAll('a.hunk');
37-
hunkAnchors.forEach(hunkAnchor => {
43+
const hunks = paragraph.parentNode.querySelectorAll('p.hunk');
44+
hunks.forEach(hunk => {
45+
const hunkAnchor = hunk.querySelector('a');
3846
const url = new URL(hunkAnchor.href);
3947
const path = url.pathname;
4048
const pathHash = path + url.hash.replace('#', '%23');
4149
const anchor = document.createElement('a');
4250
anchor.href = diffSite + '/?url1=' + rootURL + pathHash + '&url2=' + baseDocURL + path;
43-
anchor.textContent = hunkAnchor.textContent;
51+
anchor.textContent = 'compare with the base';
4452
anchor.setAttribute('target', '_blank');
45-
paragraph.appendChild(anchor);
46-
paragraph.innerHTML += '&nbsp;';
53+
hunk.innerHTML += '&nbsp;&nbsp;';
54+
hunk.appendChild(anchor);
4755
});
4856
});
4957
});
5058
</script>
5159
EOF
5260
echo '</head>' >> CHANGES.html
5361
echo '<body>' >> CHANGES.html
54-
(cd $DOC_REPOSITORY && git diff $BASE_DOC_COMMIT -- *.html) > diff.txt
5562
python3 - << EOF
5663
import os, re, html
57-
with open('diff.txt', 'r') as f:
64+
from itertools import chain
65+
with open('$DIFF_TEXT', 'r') as f:
5866
diff_text = f.read()
5967
diff_blocks = re.split(r'^(?=diff --git)', diff_text, flags=re.MULTILINE)
6068
out_blocks = []
6169
for block in diff_blocks:
6270
match = re.search(r'^diff --git a/(.*) b/\1', block, flags=re.MULTILINE)
6371
if match:
64-
doc = match.group(1)
65-
file_path = os.path.join('$DOC_REPOSITORY', doc)
72+
path = match.group(1)
73+
file_path = os.path.join('$DOC_REPOSITORY', path)
6674
try:
6775
with open(file_path, 'r') as file:
6876
content = file.readlines()
6977
except FileNotFoundError:
7078
content = []
7179
count = 0
80+
hunks = []
81+
hunk_lines = []
82+
in_hunk = False
7283
for line in block.splitlines():
7384
if line.startswith('@@ -'):
85+
if hunk_lines:
86+
hunks.append('<pre><code class="language-diff">'
87+
+ html.escape('\n'.join(hunk_lines)).strip()
88+
+ '</code></pre>')
89+
hunk_lines = []
7490
search_result = re.search(r'@@ -(\d+),(\d+) \+(\d+),(\d+)', line)
7591
if search_result:
76-
line_number = int(search_result.group(3))
77-
for i in range(line_number - 1, -1, -1):
78-
if content[i].startswith('<'):
92+
line_number = int(search_result.group(3)) - 1
93+
span = int(search_result.group(4))
94+
for i in chain(range(line_number, line_number + span), range(line_number - 1, -1, -1)):
95+
try:
96+
ln = content[i]
97+
except IndexError:
98+
continue
99+
for idx, char in enumerate(ln):
100+
if not char.isspace():
101+
break
102+
else:
103+
idx = len(ln)
104+
if ln.startswith('<', idx) and not ln.startswith('</', idx):
79105
count += 1
80-
content[i] = f'<span id="hunk{count}" style="visibility: hidden;"></span>' + content[i]
106+
content[i] = ln[:idx] + f'<span id="hunk{count}" style="visibility: hidden;"></span>' + ln[idx:]
107+
hunks.append(f'<p class="hunk"><a href="{path}#hunk{count}" class="hunk" target="_blank">hunk #{count}</a></p>')
81108
break
109+
hunk_lines.append(line)
110+
if hunk_lines:
111+
hunks.append('<pre><code class="language-diff">'
112+
+ html.escape('\n'.join(hunk_lines)).strip()
113+
+ '</code></pre>')
82114
if content:
83115
with open(file_path, 'w') as file:
84116
file.writelines(content)
85-
path = 'html/' + doc
86-
hunks = '&nbsp;'.join(f'<a href="{path}#hunk{i+1}" class="hunk" target="_blank">#{i + 1}</a>' for i in range(count))
87-
out_blocks.append(f'<p class="diff"><a href="{path}">{doc}</a>&nbsp;' + hunks + '&emsp;</p>'
88-
+ '\n<pre><code class="language-diff">'
89-
+ html.escape(block).strip() + '</code></pre>')
117+
out_blocks.append(f'<div class="diff"><p class="diff"><a href="{path}">{path}</a></p>\n' + '\n'.join(hunks) + '\n</div>')
90118
output_text = '\n'.join(out_blocks)
91119
with open('diff.html', 'w') as f:
92120
f.write(output_text)
93121
EOF
94122
cat diff.html >> CHANGES.html
95123
echo '</body>' >> CHANGES.html
96124
echo '</html>' >> CHANGES.html
97-
rm diff.txt diff.html
125+
rm diff.html

‎.ci/docker-exec-script.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/sh -x
2+
if [ $# -lt 3 ]; then
3+
echo >&2 "usage: docker-exec-script.sh CONTAINER WORKDIR [VAR=VALUE...] SCRIPT"
4+
exit 1
5+
fi
6+
CONTAINER=$1
7+
WORKDIR=$2
8+
shift 2
9+
(echo "cd \"$WORKDIR\"";
10+
while [ $# -gt 1 ]; do
11+
echo "export \"$1\""
12+
shift
13+
done;
14+
cat "$1") | docker exec -i $CONTAINER bash -ex

‎.ci/merge-fixes.sh‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
#!/bin/sh
2-
# Apply open PRs labeled "blocker" from sagemath/sage as patches.
2+
# Apply open PRs labeled "p: CI Fix" from sagemath/sage as patches.
3+
# (policy set by vote in 2024-03,
4+
# https://groups.google.com/g/sage-devel/c/OKwwUGyKveo/m/vpyCXYBqAAAJ)
5+
#
36
# This script is invoked by various workflows in .github/workflows
47
#
58
# The repository variable SAGE_CI_FIXES_FROM_REPOS can be set
@@ -20,15 +23,15 @@ for REPO in ${SAGE_CI_FIXES_FROM_REPOSITORIES:-sagemath/sage}; do
2023
echo "Nothing to do for 'none' in SAGE_CI_FIXES_FROM_REPOSITORIES"
2124
;;
2225
*/*)
23-
echo "Getting open PRs with 'blocker' status from https://github.com/$REPO/pulls?q=is%3Aopen+label%3A%22p%3A+blocker+%2F+1%22"
26+
echo "Getting open PRs with 'p: CI Fix' label from https://github.com/$REPO/pulls?q=is%3Aopen+label%3A%22p%3A+CI+Fix%22"
2427
GH="gh -R $REPO"
2528
REPO_FILE="upstream/ci-fixes-${REPO%%/*}-${REPO##*/}"
26-
PRs="$($GH pr list --label "p: blocker / 1" --json number --jq '.[].number' | tee $REPO_FILE)"
29+
PRs="$($GH pr list --label "p: CI Fix" --json number --jq '.[].number' | tee $REPO_FILE)"
2730
date -u +"%Y-%m-%dT%H:%M:%SZ" > $REPO_FILE.date # Record the date, for future reference
2831
if [ -z "$PRs" ]; then
29-
echo "Nothing to do: Found no open PRs with 'blocker' status in $REPO."
32+
echo "Nothing to do: Found no open PRs with 'p: CI Fix' label in $REPO."
3033
else
31-
echo "Found open PRs with 'blocker' status in $REPO: $(echo $PRs)"
34+
echo "Found open PRs with 'p: CI Fix' label in $REPO: $(echo $PRs)"
3235
git tag -f test_base
3336
git commit -q -m "Uncommitted changes" --no-allow-empty -a
3437
for a in $PRs; do
@@ -42,6 +45,7 @@ for REPO in ${SAGE_CI_FIXES_FROM_REPOSITORIES:-sagemath/sage}; do
4245
# Considered alternative: Use https://github.com/$REPO/pull/$a.diff,
4346
# which squashes everything into one diff without commit metadata.
4447
PULL_URL="https://github.com/$REPO/pull/$a"
48+
PULL_SHORT="$REPO#$a"
4549
PULL_FILE="$REPO_FILE-$a"
4650
PATH=build/bin:$PATH build/bin/sage-download-file --quiet "$PULL_URL.patch" $PULL_FILE.patch
4751
date -u +"%Y-%m-%dT%H:%M:%SZ" > $PULL_FILE.date # Record the date, for future reference
@@ -67,7 +71,7 @@ for REPO in ${SAGE_CI_FIXES_FROM_REPOSITORIES:-sagemath/sage}; do
6771
git am --signoff --show-current-patch=diff
6872
echo "--------------------------------------------------------------------8<-----------------------------"
6973
echo "::endgroup::"
70-
echo "Failure applying $PULL_URL as a patch, resetting"
74+
echo "Failure applying $PULL_SHORT as a patch, resetting"
7175
git am --signoff --abort
7276
fi
7377
done

‎.ci/retrofit-worktree.sh‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,32 +12,35 @@ export GIT_AUTHOR_EMAIL="[email protected]"
1212
export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
1313
export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
1414

15+
set -e
16+
1517
# Set globally for other parts of the workflow
1618
git config --global user.name "$GIT_AUTHOR_NAME"
1719
git config --global user.email "$GIT_AUTHOR_EMAIL"
1820

19-
set -ex
21+
set -x
2022

2123
# If actions/checkout downloaded our source tree using the GitHub REST API
2224
# instead of with git (because do not have git installed in our image),
2325
# we first make the source tree a repo.
24-
if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi
26+
if [ ! -d .git ]; then git init; fi
2527

26-
# Tag this state of the source tree "new". This is what we want to build and test.
28+
# Commit and tag this state of the source tree "new". This is what we want to build and test.
29+
git add -A && git commit --quiet --allow-empty -m "new"
2730
git tag -f new
2831

2932
# Our container image contains a source tree in $WORKTREE_DIRECTORY with a full build of Sage.
3033
# But $WORKTREE_DIRECTORY is not a git repository.
3134
# We make $WORKTREE_DIRECTORY a worktree whose index is at tag "new".
3235
# We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
33-
# Then we update worktree and index with "git reset --hard new".
36+
# Then we update worktree and index with "git checkout new".
3437
# (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
35-
# Finally we reset the index to "old". (This keeps all mtimes unchanged.)
36-
# The changed files now show up as uncommitted changes.
37-
# The final "git add -N" makes sure that files that were added in "new" do not show
38-
# as untracked files, which would be removed by "git clean -fx".
38+
if [ -L $WORKTREE_NAME ]; then
39+
rm -f $WORKTREE_NAME
40+
fi
41+
git worktree prune --verbose
3942
git worktree add --detach $WORKTREE_NAME
4043
rm -rf $WORKTREE_DIRECTORY/.git && mv $WORKTREE_NAME/.git $WORKTREE_DIRECTORY/
4144
rm -rf $WORKTREE_NAME && ln -s $WORKTREE_DIRECTORY $WORKTREE_NAME
4245
if [ ! -f $WORKTREE_NAME/.gitignore ]; then cp .gitignore $WORKTREE_NAME/; fi
43-
(cd $WORKTREE_NAME && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
46+
(cd $WORKTREE_NAME && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git checkout -f new && git clean -fd && git status)

0 commit comments

Comments
 (0)