diff --git a/ci_test.py b/ci_test.py index a4f233a4..067a9da1 100755 --- a/ci_test.py +++ b/ci_test.py @@ -20,14 +20,14 @@ import os -def run_command(cmd, log_file=None): +def run_command(cmd, log_file=None, cwd=None): print("Running: {}".format(cmd)) sys.stdout.flush() if log_file: file = open(log_file, "w") - p = subprocess.Popen(cmd, shell=True, stdout=file, stderr=file) + p = subprocess.Popen(cmd, shell=True, stdout=file, stderr=file, cwd=cwd) else: - p = subprocess.Popen(cmd, shell=True) + p = subprocess.Popen(cmd, shell=True, cwd=cwd) (output, err) = p.communicate() return p.wait() @@ -45,8 +45,7 @@ def get_dockerfiles(): filename = file_info['filename'] print(filename) if "Dockerfile" in filename: - file_dir = filename.replace("Dockerfile", "") - dockerfiles.append(file_dir) + dockerfiles.append(filename) return dockerfiles @@ -69,12 +68,25 @@ def main(): sys.stdout.flush() log_file = dockerfile.replace(docker_dir,"").replace("/", "_") log_file = "{}.log".format(log_file) - cmd = "docker build --no-cache=true {}".format(dockerfile) + + # Get the root directory for this collection of Dockerfiles. For example, this could be + # 'swift-ci' or 'nightly-main'. + # + # This will allow the test runner to build the Dockerfile from this directory + # which may contain additional scripts necessary for the build process. + dockerfile_path_components = dockerfile.split(os.sep) + dockerfile_root_dir = os.path.join(docker_dir, dockerfile_path_components.pop(0)) + + # Join the remaining path components as the relative path to this Dockerfile, from the root + # directory + dockerfile_relative_path = os.sep.join(dockerfile_path_components) + + cmd = "docker build --no-cache=true -f {} .".format(dockerfile_relative_path) if "buildx" in dockerfile: # if "buildx" is part of the path, we want to use the new buildx build system and build # for both amd64 and arm64. - cmd = "docker buildx build --platform linux/arm64,linux/amd64 --no-cache=true {}".format(dockerfile) - status = run_command(cmd, log_file) + cmd = "docker buildx build --platform linux/arm64,linux/amd64 --no-cache=true -f {} .".format(dockerfile_relative_path) + status = run_command(cmd, log_file, dockerfile_root_dir) results[dockerfile] = status if status != 0: suite_status = False diff --git a/swift-ci/install_node.sh b/swift-ci/install_node.sh new file mode 100755 index 00000000..3b047f6e --- /dev/null +++ b/swift-ci/install_node.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2021 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See https://swift.org/LICENSE.txt for license information +# See https://swift.org/CONTRIBUTORS.txt for Swift project authors + +# This script is used to install Node.js in the Docker containers used +# to build & qualify Swift toolchains on Linux. +# +# Node is specifically required to build swift-docc-render, the template used by swift-docc to +# render documentation archives. + +set -e + +NODE_VERSION='v14.17.4' + +ARCH=`uname -m` +if [ ${ARCH} == 'x86_64' ]; then + NODE_DISTRO='linux-x64' +elif [ ${ARCH} == 'aarch64' ]; then + NODE_DISTRO='linux-arm64' +else + echo >&2 "Unsupported CPU architecture. Unable to install required Node dependency." + exit 1 +fi + +NODE_FILENAME=node-$NODE_VERSION-$NODE_DISTRO +NODE_COMPRESSED_FILENAME=$NODE_FILENAME.tar.gz +NODE_URL=https://nodejs.org/dist/$NODE_VERSION/$NODE_COMPRESSED_FILENAME + +NODE_SHASUM_FILENAME=SHASUMS256.txt +NODE_SHASUM_URL=https://nodejs.org/dist/$NODE_VERSION/$NODE_SHASUM_FILENAME + +if [ -x "$(command -v curl)" ]; then + curl -o $NODE_COMPRESSED_FILENAME $NODE_URL + curl -o $NODE_SHASUM_FILENAME $NODE_SHASUM_URL +elif [ -x "$(command -v python)" ]; then + python -c "from urllib import urlretrieve; urlretrieve('$NODE_URL', '$NODE_COMPRESSED_FILENAME')" + python -c "from urllib import urlretrieve; urlretrieve('$NODE_SHASUM_URL', '$NODE_SHASUM_FILENAME')" +else + echo >&2 "No download command found; install curl or python." + exit 1 +fi + +if [ -x "$(command -v shasum)" ]; then + SHA_CMD="shasum" +elif [ -x "$(command -v sha256sum)" ]; then + SHA_CMD="sha256sum" +else + echo >&2 "No sha command found; install shasum or sha256sum." + exit 1 +fi + +if grep $NODE_COMPRESSED_FILENAME $NODE_SHASUM_FILENAME | $SHA_CMD -c -; then + echo "Node.js binary verified successfully." +else + echo >&2 "Node.js binary could not be verified." + exit 1 +fi + +mkdir -p /usr/local/lib/nodejs +tar -xf $NODE_COMPRESSED_FILENAME -C /usr/local/lib/nodejs +mv /usr/local/lib/nodejs/$NODE_FILENAME /usr/local/lib/nodejs/node diff --git a/swift-ci/master/amazon-linux/2/Dockerfile b/swift-ci/master/amazon-linux/2/Dockerfile index 059e3617..57f6b33b 100644 --- a/swift-ci/master/amazon-linux/2/Dockerfile +++ b/swift-ci/master/amazon-linux/2/Dockerfile @@ -40,6 +40,11 @@ RUN yum -y install \ wget \ which +# Install the current version of Node.js that swift-docc-render depends on: +COPY install_node.sh / +RUN /install_node.sh +ENV PATH "$PATH:/usr/local/lib/nodejs/node/bin" + RUN mkdir -p /usr/local/lib/python3.7/site-packages/ RUN easy_install-3.7 six diff --git a/swift-ci/master/centos/7/Dockerfile b/swift-ci/master/centos/7/Dockerfile index da12e1e5..42434043 100644 --- a/swift-ci/master/centos/7/Dockerfile +++ b/swift-ci/master/centos/7/Dockerfile @@ -39,6 +39,11 @@ RUN yum install --enablerepo=centosplus -y \ which \ zlib-devel +# Install the current version of Node.js that swift-docc-render depends on: +COPY install_node.sh / +RUN /install_node.sh +ENV PATH "$PATH:/usr/local/lib/nodejs/node/bin" + RUN echo -e ". /opt/rh/sclo-git25/enable\n. /opt/rh/llvm-toolset-7/enable\n. /opt/rh/devtoolset-8/enable\n" >> /home/build-user/.bashrc RUN sed -i -e 's/\*__block/\*__libc_block/g' /usr/include/unistd.h diff --git a/swift-ci/master/centos/8/Dockerfile b/swift-ci/master/centos/8/Dockerfile index a913bed4..5a11c417 100644 --- a/swift-ci/master/centos/8/Dockerfile +++ b/swift-ci/master/centos/8/Dockerfile @@ -37,6 +37,11 @@ RUN yum install --enablerepo=powertools -y \ tar \ which +# Install the current version of Node.js that swift-docc-render depends on +COPY install_node.sh / +RUN /install_node.sh +ENV PATH "$PATH:/usr/local/lib/nodejs/node/bin" + RUN ln -s /usr/bin/python2 /usr/bin/python USER build-user diff --git a/swift-ci/master/debian/10/Dockerfile b/swift-ci/master/debian/10/Dockerfile index 72260d91..deac00f6 100644 --- a/swift-ci/master/debian/10/Dockerfile +++ b/swift-ci/master/debian/10/Dockerfile @@ -33,6 +33,11 @@ RUN apt-get -y update && apt-get -y --no-install-recommends install \ uuid-dev \ ca-certificates +# Install the current version of Node.js that swift-docc-render depends on: +COPY install_node.sh / +RUN /install_node.sh +RUN echo "export PATH=$PATH:/usr/local/lib/nodejs/node/bin" >> /home/build-user/.bashrc + USER build-user WORKDIR /home/build-user diff --git a/swift-ci/master/debian/9/Dockerfile b/swift-ci/master/debian/9/Dockerfile index 812a16c6..56573cea 100644 --- a/swift-ci/master/debian/9/Dockerfile +++ b/swift-ci/master/debian/9/Dockerfile @@ -31,6 +31,11 @@ RUN apt-get --no-install-recommends -y install \ ca-certificates \ uuid-dev +# Install the current version of Node.js that swift-docc-render depends on: +COPY install_node.sh / +RUN /install_node.sh +RUN echo "export PATH=$PATH:/usr/local/lib/nodejs/node/bin" >> /home/build-user/.bashrc + USER build-user WORKDIR /home/build-user \ No newline at end of file diff --git a/swift-ci/master/fedora/31/Dockerfile b/swift-ci/master/fedora/31/Dockerfile index 5a7ededc..105bbf1b 100644 --- a/swift-ci/master/fedora/31/Dockerfile +++ b/swift-ci/master/fedora/31/Dockerfile @@ -26,6 +26,11 @@ RUN dnf -y update && dnf install -y \ rsync \ swig +# Install the current version of Node.js that swift-docc-render depends on: +COPY install_node.sh / +RUN /install_node.sh +ENV PATH "$PATH:/usr/local/lib/nodejs/node/bin" + USER build-user WORKDIR /home/build-user diff --git a/swift-ci/master/fedora/32/Dockerfile b/swift-ci/master/fedora/32/Dockerfile index f3218877..2697e857 100644 --- a/swift-ci/master/fedora/32/Dockerfile +++ b/swift-ci/master/fedora/32/Dockerfile @@ -26,6 +26,11 @@ RUN dnf -y update && dnf install -y \ rsync \ swig +# Install the current version of Node.js that swift-docc-render depends on: +COPY install_node.sh / +RUN /install_node.sh +ENV PATH "$PATH:/usr/local/lib/nodejs/node/bin" + USER build-user WORKDIR /home/build-user diff --git a/swift-ci/master/fedora/rawhide/Dockerfile b/swift-ci/master/fedora/rawhide/Dockerfile index 43830bd0..5df1d9c8 100644 --- a/swift-ci/master/fedora/rawhide/Dockerfile +++ b/swift-ci/master/fedora/rawhide/Dockerfile @@ -26,6 +26,11 @@ RUN dnf -y update && dnf install -y \ rsync \ swig +# Install the current version of Node.js that swift-docc-render depends on: +COPY install_node.sh / +RUN /install_node.sh +ENV PATH "$PATH:/usr/local/lib/nodejs/node/bin" + USER build-user WORKDIR /home/build-user diff --git a/swift-ci/master/ubuntu/18.04/Dockerfile b/swift-ci/master/ubuntu/18.04/Dockerfile index 61608ba6..6f337ef3 100644 --- a/swift-ci/master/ubuntu/18.04/Dockerfile +++ b/swift-ci/master/ubuntu/18.04/Dockerfile @@ -30,6 +30,11 @@ RUN apt -y update && apt -y install \ tzdata \ unzip \ uuid-dev + +# Install the current version of Node.js that swift-docc-render depends on: +COPY install_node.sh / +RUN /install_node.sh +ENV PATH "$PATH:/usr/local/lib/nodejs/node/bin" USER build-user diff --git a/swift-ci/master/ubuntu/20.04/Dockerfile b/swift-ci/master/ubuntu/20.04/Dockerfile index fcc25b4a..fac03be3 100644 --- a/swift-ci/master/ubuntu/20.04/Dockerfile +++ b/swift-ci/master/ubuntu/20.04/Dockerfile @@ -32,6 +32,11 @@ RUN apt-get -y update && apt-get -y install \ unzip \ uuid-dev +# Install the current version of Node.js that swift-docc-render depends on: +COPY install_node.sh / +RUN /install_node.sh +ENV PATH "$PATH:/usr/local/lib/nodejs/node/bin" + USER build-user WORKDIR /home/build-user