Skip to content

Commit

Permalink
Rollup merge of #99992 - jyn514:shell-scripts, r=Mark-Simulacrum
Browse files Browse the repository at this point in the history
Add `x.sh` and `x.ps1` shell scripts

This is a more ambitious version of #98716.
It still changes the x.py shebang back to python3, for compatibility with non-Unix systems,
but also adds alternative entrypoints for systems without `python3` installed.

These scripts will be necessary for the rust entrypoint (#94829), so I see
little downside in adding them early.

I'll update the dev-guide to suggest using these instead of x.py once this is merged.

Fixes #98650

r? `@Mark-Simulacrum` cc `@dtolnay` `@CAD97` `@yoshuawuyts`
  • Loading branch information
matthiaskrgr authored Aug 11, 2022
2 parents 37efd55 + 775c3c0 commit 20f124a
Show file tree
Hide file tree
Showing 6 changed files with 94 additions and 34 deletions.
11 changes: 8 additions & 3 deletions src/bootstrap/mk/Makefile.in
Original file line number Diff line number Diff line change
Expand Up @@ -66,16 +66,21 @@ TESTS_IN_2 := \
src/test/ui \
src/tools/linkchecker

## MSVC native builders

# these intentionally don't use `$(BOOTSTRAP)` so we can test the shebang on Windows
ci-subset-1:
$(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_2:%=--exclude %)
$(Q)$(CFG_SRC_DIR)/x.py test --stage 2 $(TESTS_IN_2:%=--exclude %)
ci-subset-2:
$(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_2)
$(Q)$(CFG_SRC_DIR)/x.ps1 test --stage 2 $(TESTS_IN_2)

## MingW native builders

TESTS_IN_MINGW_2 := \
src/test/ui

ci-mingw-subset-1:
$(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_MINGW_2:%=--exclude %)
$(Q)$(CFG_SRC_DIR)/x.sh test --stage 2 $(TESTS_IN_MINGW_2:%=--exclude %)
ci-mingw-subset-2:
$(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_MINGW_2)

Expand Down
18 changes: 15 additions & 3 deletions src/ci/docker/host-x86_64/x86_64-gnu-llvm-12/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
FROM ubuntu:20.04

ARG DEBIAN_FRONTEND=noninteractive

# NOTE: intentionally installs both python2 and python3 so we can test support for both.
RUN apt-get update && apt-get install -y --no-install-recommends \
g++ \
gcc-multilib \
Expand All @@ -10,6 +12,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
curl \
ca-certificates \
python2.7 \
python3.9 \
git \
cmake \
sudo \
Expand All @@ -23,6 +26,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
xz-utils \
nodejs

# Install powershell so we can test x.ps1 on Linux
RUN apt-get update && \
apt-get install -y apt-transport-https software-properties-common && \
curl -s "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb" > packages-microsoft-prod.deb && \
dpkg -i packages-microsoft-prod.deb && \
apt-get update && \
apt-get install -y powershell

COPY scripts/sccache.sh /scripts/
RUN sh /scripts/sccache.sh

Expand All @@ -33,21 +44,22 @@ ENV RUST_CONFIGURE_ARGS \
--enable-llvm-link-shared \
--set rust.thin-lto-import-instr-limit=10

ENV SCRIPT python2.7 ../x.py --stage 2 test --exclude src/tools/tidy && \
# NOTE: intentionally uses all of `x.py`, `x.sh`, and `x.ps1` to make sure they all work on Linux.
ENV SCRIPT ../x.py --stage 2 test --exclude src/tools/tidy && \
# Run the `mir-opt` tests again but this time for a 32-bit target.
# This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have
# both 32-bit and 64-bit outputs updated by the PR author, before
# the PR is approved and tested for merging.
# It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`,
# despite having different output on 32-bit vs 64-bit targets.
python2.7 ../x.py --stage 2 test src/test/mir-opt \
../x.sh --stage 2 test src/test/mir-opt \
--host='' --target=i686-unknown-linux-gnu && \
# Run the UI test suite again, but in `--pass=check` mode
#
# This is intended to make sure that both `--pass=check` continues to
# work.
#
python2.7 ../x.py --stage 2 test src/test/ui --pass=check \
../x.ps1 --stage 2 test src/test/ui --pass=check \
--host='' --target=i686-unknown-linux-gnu && \
# Run tidy at the very end, after all the other tests.
python2.7 ../x.py --stage 2 test src/tools/tidy
8 changes: 5 additions & 3 deletions src/tools/tidy/src/bins.rs
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,8 @@ mod os_impl {

#[cfg(unix)]
pub fn check(path: &Path, bad: &mut bool) {
use std::ffi::OsStr;

const ALLOWED: &[&str] = &["configure"];

crate::walk_no_read(
Expand All @@ -117,9 +119,9 @@ mod os_impl {
},
&mut |entry| {
let file = entry.path();
let filename = file.file_name().unwrap().to_string_lossy();
let extensions = [".py", ".sh"];
if extensions.iter().any(|e| filename.ends_with(e)) {
let extension = file.extension();
let scripts = ["py", "sh", "ps1"];
if scripts.into_iter().any(|e| extension == Some(OsStr::new(e))) {
return;
}

Expand Down
28 changes: 28 additions & 0 deletions x.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#!/usr/bin/env pwsh

# See x.sh for why these scripts exist.

$xpy = Join-Path $PSScriptRoot x.py
# Start-Process for some reason splits arguments on spaces. (Isn't powershell supposed to be simpler than bash?)
# Double-quote all the arguments so it doesn't do that.
$xpy_args = @("""$xpy""")
foreach ($arg in $args) {
$xpy_args += """$arg"""
}

foreach ($python in "py", "python3", "python", "python2") {
# NOTE: this only tests that the command exists in PATH, not that it's actually
# executable. The latter is not possible in a portable way, see
# https://github.com/PowerShell/PowerShell/issues/12625.
if (Get-Command $python -ErrorAction SilentlyContinue) {
if ($python -eq "py") {
# Use python3, not python2
$xpy_args = @("-3") + $xpy_args
}
$process = Start-Process -NoNewWindow -Wait -PassThru $python $xpy_args
Exit $process.ExitCode
}
}

Write-Error "${PSCommandPath}: error: did not find python installed"
Exit 1
30 changes: 5 additions & 25 deletions x.py
Original file line number Diff line number Diff line change
@@ -1,36 +1,16 @@
#!/usr/bin/env bash
#!/usr/bin/env python3
# Some systems don't have `python3` in their PATH. This isn't supported by x.py directly;
# they should use `x.sh` or `x.ps1` instead.

# Modern Linux and macOS systems commonly only have a thing called `python3` and
# not `python`, while Windows commonly does not have `python3`, so we cannot
# directly use python in the shebang and have it consistently work. Instead we
# embed some bash to look for a python to run the rest of the script.
#
# On Windows, `py -3` sometimes works. We need to try it first because `python3`
# sometimes tries to launch the app store on Windows.
'''':
for PYTHON in "py -3" python3 python python2; do
if command -v $PYTHON >/dev/null; then
exec $PYTHON "$0" "$@"
break
fi
done
echo "$0: error: did not find python installed" >&2
exit 1
'''
# The rest of this file is Python.
#
# This file is only a "symlink" to bootstrap.py, all logic should go there.

import os
import sys

# If this is python2, check if python3 is available and re-execute with that
# interpreter.
# interpreter. Only python3 allows downloading CI LLVM.
#
# `./x.py` would not normally benefit from this because the bash above tries
# python3 before 2, but this matters if someone ran `python x.py` and their
# system's `python` is python2.
# This matters if someone's system `python` is python2.
if sys.version_info.major < 3:
try:
os.execvp("py", ["py", "-3"] + sys.argv)
Expand Down
33 changes: 33 additions & 0 deletions x.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/bin/sh

# Modern Linux and macOS systems commonly only have a thing called `python3` and
# not `python`, while Windows commonly does not have `python3`, so we cannot
# directly use python in the x.py shebang and have it consistently work. Instead we
# have a shell script to look for a python to run x.py.

set -eu

realpath() {
if [ -d "$1" ]; then
CDPATH='' command cd "$1" && pwd -P
else
echo "$(realpath "$(dirname "$1")")/$(basename "$1")"
fi
}

xpy=$(dirname "$(realpath "$0")")/x.py

# On Windows, `py -3` sometimes works. We need to try it first because `python3`
# sometimes tries to launch the app store on Windows.
for SEARCH_PYTHON in py python3 python python2; do
if python=$(command -v $SEARCH_PYTHON) && [ -x "$python" ]; then
if [ $SEARCH_PYTHON = py ]; then
extra_arg="-3"
else
extra_arg=""
fi
exec "$python" $extra_arg "$xpy" "$@"
fi
done
echo "$0: error: did not find python installed" >&2
exit 1

0 comments on commit 20f124a

Please sign in to comment.