From ea40cc3cd31eee5ea2cd3b7d9261fec834ede574 Mon Sep 17 00:00:00 2001 From: Marek Wydmuch Date: Thu, 7 Sep 2023 17:42:08 +0200 Subject: [PATCH] Update cibuildwheel tests --- scripts/build_and_test.sh | 4 --- scripts/build_and_test_conda.sh | 28 +++++++++++++++++++ tests/build_test_cibuildwheel_linux.sh | 5 +++- tests/build_test_local_linux_builds.sh | 14 ++++------ .../apt+conda-based.Dockerfile | 20 +++++++++++++ .../apt-based.Dockerfile | 4 +-- .../conda-based.Dockerfile | 5 ++-- .../dnf-based.Dockerfile | 4 +-- .../apt-based.Dockerfile | 2 +- .../dnf-based.Dockerfile | 2 +- 10 files changed, 63 insertions(+), 25 deletions(-) create mode 100755 scripts/build_and_test_conda.sh create mode 100644 tests/local_builds_dockerfiles/apt+conda-based.Dockerfile diff --git a/scripts/build_and_test.sh b/scripts/build_and_test.sh index e79817d16..887b26b42 100755 --- a/scripts/build_and_test.sh +++ b/scripts/build_and_test.sh @@ -25,7 +25,3 @@ python3 -c "import vizdoom" # Run tests pytest tests -# CMake manual build -# rm CMakeCache.txt -# cmake -DCMAKE_PREFIX_PATH=$CONDA_PREFIX -DCMAKE_BUILD_TYPE=Release -DBUILD_PYTHON=ON . -# make -j diff --git a/scripts/build_and_test_conda.sh b/scripts/build_and_test_conda.sh new file mode 100755 index 000000000..743add8ca --- /dev/null +++ b/scripts/build_and_test_conda.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -e + +# Set working dir to the root of the repo +cd $( dirname "${BASH_SOURCE[0]}" )/.. + +# Report directory +ls -lha . + +# Report cmake version +cmake --version + +# Report gcc version +gcc --version + +# Report python version +python --version +python -c "import sys; print('Python', sys.version)" + +# Install +export VIZDOOM_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=/root/miniconda/" +python -m pip install .[test] + +# Test import +python -c "import vizdoom" + +# Run tests +pytest tests diff --git a/tests/build_test_cibuildwheel_linux.sh b/tests/build_test_cibuildwheel_linux.sh index ffae5e36f..2d40f49fd 100755 --- a/tests/build_test_cibuildwheel_linux.sh +++ b/tests/build_test_cibuildwheel_linux.sh @@ -9,7 +9,7 @@ DOCKERFILES_DIR=$( dirname ${BASH_SOURCE[0]} )/wheels_test_dockerfiles GENERATED_DOCKERFILES_DIR=tests/test_dockerfiles IMAGE_PREFIX="vizdoom_wheels" -# Generate and run dockerfiles + # Array in format " " DOCKERFILES_TO_BUILD_AND_RUN=( "almalinux:9 dnf-based.Dockerfile" # Python 3.9 @@ -22,6 +22,9 @@ DOCKERFILES_TO_BUILD_AND_RUN=( "continuumio/miniconda3:latest conda-based.Dockerfile" # Python 3.10 ) +# Build wheels using cibuildwheel +cibuildwheel --platform linux --arch $(uname -m) + function create_dockerfile ( ) { local all_args=("$@") local base_image=$1 diff --git a/tests/build_test_local_linux_builds.sh b/tests/build_test_local_linux_builds.sh index 23e0fe15e..45bea9bc5 100755 --- a/tests/build_test_local_linux_builds.sh +++ b/tests/build_test_local_linux_builds.sh @@ -9,6 +9,7 @@ DOCKERFILES_DIR=$( dirname ${BASH_SOURCE[0]} )/local_builds_dockerfiles GENERATED_DOCKERFILES_DIR=tests/test_dockerfiles IMAGE_PREFIX="vizdoom_local" +# Generate and run dockerfiles # Array in format " " DOCKERFILES_TO_BUILD_AND_RUN=( "almalinux:9 dnf-based.Dockerfile RUN dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb" @@ -20,15 +21,10 @@ DOCKERFILES_TO_BUILD_AND_RUN=( "ubuntu:18.04 apt-based.Dockerfile" "ubuntu:20.04 apt-based.Dockerfile" "ubuntu:22.04 apt-based.Dockerfile" - "ubuntu:latest apt-based.Dockerfile" - #"continuumio/miniconda3:latest conda-based.Dockerfile" # Does not work at the moment + "ubuntu:20.04 apt+conda-based.Dockerfile" # Ubuntu build with dependencies installed via conda + #"continuumio/miniconda3:latest conda-based.Dockerfile" # Does not work at the moment ) - -# Build wheels using cibuildwheel -cibuildwheel --platform linux --arch $(uname -m) - - # Test wheels inside docker containers function create_dockerfile ( ) { local all_args=("$@") @@ -56,8 +52,8 @@ for dockerfile_setting in "${DOCKERFILES_TO_BUILD_AND_RUN[@]}"; do tag="${without_ext}:latest" log="${dockerfile_dir}/${without_ext}.log" - docker build -t $tag -f $dockerfile . &> $log || ( echo -e "${RED}FAILED${NC}"; exit 1 ) - docker run -it $tag &>> $log || ( echo -e "${RED}FAILED${NC}"; exit 1 ) + docker build -t $tag -f $dockerfile . #&> $log || ( echo -e "${RED}FAILED${NC}"; exit 1 ) + docker run -it $tag #&>> $log || ( echo -e "${RED}FAILED${NC}"; exit 1 ) echo -e "${GREEN}OK${NC}" done diff --git a/tests/local_builds_dockerfiles/apt+conda-based.Dockerfile b/tests/local_builds_dockerfiles/apt+conda-based.Dockerfile new file mode 100644 index 000000000..25a637f89 --- /dev/null +++ b/tests/local_builds_dockerfiles/apt+conda-based.Dockerfile @@ -0,0 +1,20 @@ +FROM ubuntu:latest + +ARG DEBIAN_FRONTEND=noninteractive +ENV TZ=Europe/Warsaw + +WORKDIR vizdoom + +# Instal wget +RUN apt-get update && apt-get install -y build-essential git make cmake wget + +# Install miniconda +RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh +RUN bash ~/miniconda.sh -b -p $HOME/miniconda +ENV PATH="/root/miniconda/bin:${PATH}" + +# Install conda dependencies +RUN conda install -y -c conda-forge boost sdl2 openal-soft + +COPY . ./ +CMD ["bash", "./scripts/build_and_test_conda.sh"] diff --git a/tests/local_builds_dockerfiles/apt-based.Dockerfile b/tests/local_builds_dockerfiles/apt-based.Dockerfile index 9f63bc0b5..cc7613016 100644 --- a/tests/local_builds_dockerfiles/apt-based.Dockerfile +++ b/tests/local_builds_dockerfiles/apt-based.Dockerfile @@ -5,10 +5,8 @@ ENV TZ=Europe/Warsaw WORKDIR vizdoom -# MINIMAL +# Install minimal dependencies RUN apt update && apt install -y build-essential cmake git libboost-all-dev libsdl2-dev libopenal-dev python3-dev python3-pip - -# FULL #RUN apt update && apt install -y build-essential cmake libboost-all-dev libsdl2-dev libfreetype-dev libopenal-dev python3-dev python3-pip COPY . ./ diff --git a/tests/local_builds_dockerfiles/conda-based.Dockerfile b/tests/local_builds_dockerfiles/conda-based.Dockerfile index 128721013..39e7eff44 100644 --- a/tests/local_builds_dockerfiles/conda-based.Dockerfile +++ b/tests/local_builds_dockerfiles/conda-based.Dockerfile @@ -2,8 +2,7 @@ FROM continuumio/miniconda3:latest WORKDIR vizdoom -# MINIMAL -RUN conda install -c conda-forge c-compiler cxx-compiler make cmake boost sdl2 openal-soft +RUN conda install -y -c conda-forge gcc gxx rhash make cmake boost sdl2 openal-soft COPY . ./ -CMD ["bash", "./scripts/build_and_test.sh"] +CMD ["bash", "./scripts/build_and_test_conda.sh"] diff --git a/tests/local_builds_dockerfiles/dnf-based.Dockerfile b/tests/local_builds_dockerfiles/dnf-based.Dockerfile index 96501b243..935c67045 100644 --- a/tests/local_builds_dockerfiles/dnf-based.Dockerfile +++ b/tests/local_builds_dockerfiles/dnf-based.Dockerfile @@ -2,10 +2,8 @@ FROM fedora:latest WORKDIR vizdoom -# MINIMAL +# Install minimal dependencies RUN dnf update -y && dnf clean all && dnf install -y gcc gcc-c++ make cmake git boost-devel SDL2-devel openal-soft-devel python3-devel python3-pip - -# FULL #RUN dnf update -y && dnf clean all && dnf install -y gcc gcc-c++ make cmake boost-devel SDL2-devel freetype-devel openal-soft-devel python3-devel python3-pip COPY . ./ diff --git a/tests/wheels_test_dockerfiles/apt-based.Dockerfile b/tests/wheels_test_dockerfiles/apt-based.Dockerfile index e5f2063df..7c1df9e30 100644 --- a/tests/wheels_test_dockerfiles/apt-based.Dockerfile +++ b/tests/wheels_test_dockerfiles/apt-based.Dockerfile @@ -5,7 +5,7 @@ ENV TZ=Europe/Warsaw WORKDIR vizdoom -# MINIMAL +# Install Python and pip RUN apt update && apt install -y python3-dev python3-pip COPY . ./ diff --git a/tests/wheels_test_dockerfiles/dnf-based.Dockerfile b/tests/wheels_test_dockerfiles/dnf-based.Dockerfile index d35640ff7..7c9c9a0bc 100644 --- a/tests/wheels_test_dockerfiles/dnf-based.Dockerfile +++ b/tests/wheels_test_dockerfiles/dnf-based.Dockerfile @@ -2,7 +2,7 @@ FROM fedora:latest WORKDIR vizdoom -# MINIMAL +# Install Python and pip RUN dnf update -y && dnf clean all && dnf install -y python3-devel python3-pip COPY . ./