Merge branch 'develop' into u/lftabera/ticket/8558

Author: lftabera

.ci/build-docker.sh

@@ -26,7 +26,7 @@ docker_build() {
     # Docker's --cache-from does not really work with multi-stage builds: https://github.com/moby/moby/issues/34715
     # So we just have to rely on the local cache.
     time docker build -f docker/Dockerfile \
---build-arg "--build-arg "MAKEFLAGS=${MAKEFLAGS}" --build-arg "SAGE_NUM_THREADS=${SAGE_NUM_THREADS}" --build-arg "MAKEFLAGS_DOCBUILD=${MAKEFLAGS}" --build-arg "SAGE_NUM_THREADS_DOCBUILD=${SAGE_NUM_THREADS_DOCBUILD}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@
+--build-arg "MAKEFLAGS=${MAKEFLAGS}" --build-arg "SAGE_NUM_THREADS=${SAGE_NUM_THREADS}" --build-arg "MAKEFLAGS_DOCBUILD=${MAKEFLAGS}" --build-arg "SAGE_NUM_THREADS_DOCBUILD=${SAGE_NUM_THREADS_DOCBUILD}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@
 }
 
 # We use a multi-stage build /docker/Dockerfile. For the caching to be
@@ -53,7 +53,7 @@ docker tag "$DOCKER_IMAGE_CLI" "$DOCKER_IMAGE_BINDER"
 # Display the layers of this image
 docker history "$DOCKER_IMAGE_CLI"
 # Build the developer image with the build artifacts intact.
-# Note: It's important to build the dev image last because it might be tagged as ARTIFACT_BASE.
+# Note: It is important to build the dev image last because it might be tagged as ARTIFACT_BASE.
 docker_build --target sagemath-dev --tag "$DOCKER_IMAGE_DEV" .
 # Display the layers of this image
 docker history "$DOCKER_IMAGE_DEV"

.ci/update-env.sh

@@ -27,8 +27,6 @@ if [ -n "$CI_MONKEY_PATCH" ]; then
     $SCRIPT
 fi
 
-WITH_PYTHON=${WITH_PYTHON:-2}
-
 # From the docker documentation: "A tag name must be valid ASCII and may
 # contain lowercase and uppercase letters, digits, underscores, periods and
 # dashes. A tag name may not start with a period or a dash and may contain a
@@ -38,10 +36,6 @@ export DOCKER_TAG=`echo $DOCKER_TAG | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '
 [[ -z "$DOCKER_TAG" ]] && export DOCKER_TAG=none
 [[ "$DOCKER_TAG" = "master" ]] && export DOCKER_TAG=latest
 
-if [ $WITH_PYTHON = 3 ]; then
-    export DOCKER_TAG=${DOCKER_TAG}-py3
-fi
-
 export DOCKER_IMAGE_CLI=${DOCKER_NAMESPACE:-sagemath}/sagemath:$DOCKER_TAG
 export DOCKER_IMAGE_DEV=${DOCKER_NAMESPACE:-sagemath}/sagemath-dev:$DOCKER_TAG
 

.github/workflows/ci-cygwin-minimal.yml

@@ -1,11 +1,20 @@
 name: Build & Test WSL
 
-on: [push, pull_request]
+on:
+  pull_request:
+    types: [opened, synchronize]
+  push:
+    tags:
+      - '*'
+    branches:
+      - 'public/build/**wsl**'
+  workflow_dispatch:
+    # Allow to run manually
 
 jobs:
   windows:
     runs-on: windows-latest
-    name: Windows (using WSL)
+    name: Ubuntu 20.04
     # Following https://trac.sagemath.org/ticket/25206#comment:63
     steps:
     - name: Configure git
@@ -18,22 +27,10 @@ jobs:
         Ubuntu\ubuntu2004.exe install --root
     - name: Install dependencies
       run: |
-        Function ExtractPackages
-        {
-           param($path)
-           @(Get-Content $path | Where-Object { !$_.StartsWith("#") })
-        }
-        $packages = Get-ChildItem build/pkgs -recurse -Include debian.txt, debian-bootstrap.txt | foreach-object { ExtractPackages $_.FullName } | sort -Unique
-        # Add tox which is use to build
-        $packages += 'tox'
-        echo "Install the following packages:" @packages
-        # Add deadsnakes so that libpython3.7-dev is found
-        & wsl sudo add-apt-repository ppa:deadsnakes/ppa
-        # Install
-        & wsl sudo apt-get update -y 
-        & wsl sudo apt-get install -y @packages
+        wsl sudo apt-get update -y 
+        wsl sudo apt-get install -y tox
     - name: Build
-      run: wsl tox -e local -- SAGE_NUM_THREADS=4 build
+      run: wsl tox -e local-sudo-ubuntu-standard -- SAGE_NUM_THREADS=4 build
       env:
         # WSL runs everything as root, so we have to enable build as root user
         EXTRA_CONFIGURE_ARGS: "--enable-build-as-root"
@@ -42,7 +39,7 @@ jobs:
         MAKE: "make -j2"
         WSLENV: EXTRA_CONFIGURE_ARGS:MAKE
     - name: Test
-      run: wsl tox -e local -- SAGE_NUM_THREADS=4 ptest
+      run: wsl tox -e local-sudo-ubuntu-standard -- SAGE_NUM_THREADS=4 ptest
     - name: Prepare logs artifact
       run: mkdir -p "artifacts/logs"; cp -r .tox/*/log "artifacts/logs"
       shell: bash

.github/workflows/ci-cygwin-standard.yml

@@ -1,26 +1,27 @@
-#!/usr/bin/env bash
+#!/bin/dash
 # to be run from $SAGE_ROOT, with arguments sage-local-${{ env.PREVIOUS_STAGES }}.tar
 
+if [ -z "$SAGE_LOCAL" ]; then
+    SAGE_LOCAL=$(pwd)/local
+fi
+
 # Show all tar files
 ls -l $*
 
 # We specifically use the cygwin tar so that symlinks are saved/restored correctly on Windows.
 for a in $*; do
     echo Extracting $a
-    tar xf $a
+    (cd / && tar xf -) < $a
     rm -f $a
 done
 
-# Also get rid of the stages that were not extracted
-rm -f sage-local-*.tar
-
 # We set the installation records to the same mtime so that no rebuilds due to dependencies
 # among these packages are triggered.
-(cd local/var/lib/sage/installed/ && touch .dummy && touch --reference=.dummy *)
+(cd "$SAGE_LOCAL"/var/lib/sage/installed/ && touch .dummy && touch --reference=.dummy *)
 
 # Show what has been built already.
-ls -l local local/var/lib/sage/installed/
+ls -l "$SAGE_LOCAL" "$SAGE_LOCAL"/var/lib/sage/installed/
 df -h
 
 # Rebase!
-src/bin/sage-rebase.sh local
+exec src/bin/sage-rebase.sh --all "$SAGE_LOCAL"

.github/workflows/ci-wsl.yml

@@ -0,0 +1,47 @@
+name: Lint
+
+on: [push, pull_request]
+
+jobs:
+  lint-pycodestyle:
+    name: Code style check with pycodestyle
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install pycodestyle
+      run: pip install tox pycodestyle
+    - name: Lint using pycodestyle
+      run: tox -e pycodestyle-minimal
+  lint-relint:
+    name: Code style check with relint
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install relint
+      run: pip install tox relint
+    - name: Lint using relint
+      run: tox -e relint src/sage/
+  lint-pyright:
+    name: Static type check with pyright
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Set up node to install pyright
+      uses: actions/setup-node@v1
+      with:
+        node-version: '12'
+    - name: Install pyright
+      run: npm install -g pyright
+    - name: Lint using pyright
+      run: pyright

.github/workflows/extract-sage-local.sh

@@ -23,6 +23,8 @@ on:
   push:
     tags:
       - '*'
+  workflow_dispatch:
+    # Allow to run manually
 
 env:
   TARGETS_PRE: build/make/Makefile
@@ -36,7 +38,9 @@ jobs:
       fail-fast: false
       max-parallel: 6
       matrix:
-        tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-eoan, ubuntu-focal, debian-jessie, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, centos-7, centos-8, gentoo, archlinux-latest, slackware-14.2, conda-forge, ubuntu-bionic-i386, ubuntu-eoan-i386, debian-buster-i386, centos-7-i386]
+        # This list is different from the one in tox.yml:
+        # Trac #31526 switches gcc 4.x-based distributions to using the gcc_spkg configuration factor
+        tox_system_factor: [ubuntu-trusty-gcc_spkg, ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-groovy, ubuntu-hirsute, debian-jessie-gcc_spkg, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-17-gcc_spkg, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, centos-7-gcc_spkg, centos-8, gentoo, archlinux-latest, opensuse-15, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-gcc_spkg]
         tox_packages_factor: [maximal]
         targets_pattern: [0-g, h-o, p, q-z]
     env:
@@ -52,15 +56,21 @@ jobs:
         run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
       - name: free disk space
         run: |
+          df -h
           sudo swapoff -a
           sudo rm -f /swapfile
-          sudo apt clean
+          sudo apt-get clean
           docker rmi $(docker image ls -aq)
+          echo "Largest packages:"
+          dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 50
+          sudo apt-get --fix-broken --yes remove $(dpkg-query -f '${Package}\n' -W | grep -E '^(ghc-|google-cloud-sdk|google-chrome|firefox|mysql-server|dotnet-sdk|hhvm|mono)') || echo "(error ignored)"
           df -h
       - name: Install test prerequisites
         run: |
           sudo DEBIAN_FRONTEND=noninteractive apt-get update
-          sudo DEBIAN_FRONTEND=noninteractive apt-get install python-tox
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install tox
+          sudo apt-get clean
+          df -h
       - name: Try to login to docker.pkg.github.com
         # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
         run: |
@@ -69,7 +79,7 @@ jobs:
               TOKEN="${{ secrets.GITHUB_TOKEN }}"
             fi
             if echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin; then
-              echo "::set-env name=DOCKER_PUSH_REPOSITORY::docker.pkg.github.com/${{ github.repository }}/"
+              echo "DOCKER_PUSH_REPOSITORY=docker.pkg.github.com/${{ github.repository }}/" >> $GITHUB_ENV
             fi
       - run: |
           set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
@@ -99,20 +109,49 @@ jobs:
 
   local-macos:
 
-    runs-on: macos-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       max-parallel: 3
       matrix:
-        tox_system_factor: [homebrew-macos, homebrew-macos-python3_xcode, homebrew-macos-python3_xcode-nokegonly, homebrew-macos-python3_pythonorg, conda-forge-macos]
+        tox_system_factor: [homebrew-macos, conda-forge-macos, homebrew-macos-python3_xcode]
         tox_packages_factor: [maximal]
+        # As of 2021-03, default xcode is 12.4
+        # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md#xcode
+        xcode_version_factor: [default]
         targets_pattern: [0-g, h-o, p, q-z]
+        os: [ macos-10.15, macos-11.0 ]
+        include:
+          # Test xcode 11.7 only on macos-10.15
+          - tox_system_factor: homebrew-macos
+            tox_packages_factor: maximal
+            xcode_version_factor: 11.7
+            targets_pattern: 0-g
+            os: macos-10.15
+          - tox_system_factor: homebrew-macos
+            tox_packages_factor: maximal
+            xcode_version_factor: 11.7
+            targets_pattern: h-o
+            os: macos-10.15
+          - tox_system_factor: homebrew-macos
+            tox_packages_factor: maximal
+            xcode_version_factor: 11.7
+            targets_pattern: p
+            os: macos-10.15
+          - tox_system_factor: homebrew-macos
+            tox_packages_factor: maximal
+            xcode_version_factor: 11.7
+            targets_pattern: q-z
+            os: macos-10.15
     env:
       TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
-      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
+      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}-${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
       TARGETS_OPTIONAL: "$( echo $(PATH=build/bin:$PATH build/bin/sage-package list :experimental: | grep -v  database_stein_watkins\\$ | grep -v polytopes_db_4d | grep '^[${{ matrix.targets_pattern }}]' ) )"
     steps:
       - uses: actions/checkout@v2
+      - name: Select Xcode version
+        run: |
+          if [ ${{ matrix.xcode_version_factor }} != default ]; then sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode_version_factor }}.app; fi
       - name: Install test prerequisites
         run: |
           brew install tox

.github/workflows/lint.yml

@@ -23,6 +23,8 @@ on:
   push:
     tags:
       - '*'
+  workflow_dispatch:
+    # Allow to run manually
 
 env:
   TARGETS_PRE: sagelib-build-deps
@@ -50,15 +52,21 @@ jobs:
         run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
       - name: free disk space
         run: |
+          df -h
           sudo swapoff -a
           sudo rm -f /swapfile
-          sudo apt clean
+          sudo apt-get clean
           docker rmi $(docker image ls -aq)
+          echo "Largest packages:"
+          dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 50
+          sudo apt-get --fix-broken --yes remove $(dpkg-query -f '${Package}\n' -W | grep -E '^(ghc-|google-cloud-sdk|google-chrome|firefox|mysql-server|dotnet-sdk|hhvm|mono)') || echo "(error ignored)"
           df -h
       - name: Install test prerequisites
         run: |
           sudo DEBIAN_FRONTEND=noninteractive apt-get update
-          sudo DEBIAN_FRONTEND=noninteractive apt-get install python-tox
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install tox
+          sudo apt-get clean
+          df -h
       - name: Try to login to docker.pkg.github.com
         # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
         run: |
@@ -67,7 +75,7 @@ jobs:
               TOKEN="${{ secrets.GITHUB_TOKEN }}"
             fi
             if echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin; then
-              echo "::set-env name=DOCKER_PUSH_REPOSITORY::docker.pkg.github.com/${{ github.repository }}/"
+              echo "DOCKER_PUSH_REPOSITORY=docker.pkg.github.com/${{ github.repository }}/" >> $GITHUB_ENV
             fi
       - run: |
           set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
@@ -112,7 +120,7 @@ jobs:
       - name: Install test prerequisites
         run: |
           sudo DEBIAN_FRONTEND=noninteractive apt-get update
-          sudo DEBIAN_FRONTEND=noninteractive apt-get install python-tox
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install tox
       - name: Build and test with tox
         # We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
         # For doctesting, we use a lower parallelization to avoid timeouts.