From 2ca1c4061c31e725f140c7ca42fe341626e47ad8 Mon Sep 17 00:00:00 2001 From: Marek Wydmuch Date: Sun, 3 Sep 2023 23:00:28 +0200 Subject: [PATCH 01/12] Update CMake's FindPython and silent some C/C++ compilation warnings" --- CMakeLists.txt | 11 +++++-- setup.py | 61 ++++++++++++++++------------------- src/lib_python/CMakeLists.txt | 29 ++++++++++------- 3 files changed, 53 insertions(+), 48 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 055904ec5..8eb5a5536 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.12) # Project + versions #----------------------------------------------------------------------------------------------------------------------- @@ -76,6 +76,9 @@ endif (APPLE) if (CMAKE_COMPILER_IS_GNUCXX) add_definitions("-fPIC") + # Explicitly silence some warnings + add_definitions("-Wno-implicit-fallthrough -Wno-shift-negative-value") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-class-memaccess -Wno-deprecated-copy") # C++ only endif () if (MSVC) @@ -98,7 +101,7 @@ set(VIZDOOM_LIB_SRC_DIR ${VIZDOOM_SRC_DIR}/lib) set(VIZDOOM_LIB_INCLUDE_DIR ${VIZDOOM_INCLUDE_DIR} ${VIZDOOM_LIB_SRC_DIR}) set(VIZDOOM_OUTPUT_NAME vizdoom) -find_package(Boost 1.65.0 COMPONENTS filesystem thread system date_time chrono regex iostreams REQUIRED) +find_package(Boost 1.53 COMPONENTS filesystem thread system date_time chrono regex iostreams REQUIRED) find_package(Threads REQUIRED) include_directories(${VIZDOOM_LIB_INCLUDE_DIR} ${Boost_INCLUDE_DIR}) @@ -159,7 +162,9 @@ set_target_properties(libvizdoom_shared # ViZDoom Engine #----------------------------------------------------------------------------------------------------------------------- -add_subdirectory(${VIZDOOM_SRC_DIR}/vizdoom) +if (BUILD_ENGINE) + add_subdirectory(${VIZDOOM_SRC_DIR}/vizdoom) +endif () # Python bindings diff --git a/setup.py b/setup.py index 349b05a9e..87377c8bb 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,8 @@ from distutils.command.build import build from multiprocessing import cpu_count -from setuptools import setup +from setuptools import setup, Distribution +from setuptools.command.install import install from wheel.bdist_wheel import bdist_wheel @@ -73,26 +74,6 @@ def get_long_description(): ) -def get_python_library(python_root_dir): - paths_to_check = [ - "libs/python{}{}.{}", # Windows Python/Anaconda - "libpython{}.{}m.{}", # Unix - "libpython{}.{}.{}", # Unix - "lib/libpython{}.{}m.{}", # Unix Anaconda - "lib/libpython{}.{}.{}", # Unix Anaconda - ] - - for path_format in paths_to_check: - path = os.path.join( - python_root_dir, - path_format.format(*python_version.split("."), library_extension), - ) - if os.path.exists(path): - return path - - return None - - class Wheel(bdist_wheel): def finalize_options(self): bdist_wheel.finalize_options(self) @@ -104,16 +85,29 @@ def get_tag(self): return python, abi, plat +class BinaryDistribution(Distribution): + def has_ext_modules(self): + return True + + def is_pure(self): + return False + + +class InstallPlatlib(install): + def finalize_options(self): + install.finalize_options(self) + if self.distribution.has_ext_modules(): + self.install_lib = self.install_platlib + + class BuildCommand(build): def run(self): cpu_cores = max(1, cpu_count() - 1) - python_executable = os.path.realpath(sys.executable) - cmake_arg_list = [ "cmake", "-DCMAKE_BUILD_TYPE=Release", "-DBUILD_PYTHON=ON", - f"-DPYTHON_EXECUTABLE={python_executable}", + f"-DBUILD_PYTHON_VERSION={python_version}" ] env_cmake_args = os.getenv("VIZDOOM_CMAKE_ARGS") @@ -123,6 +117,7 @@ def run(self): f"VIZDOOM_CMAKE_ARGS is set, the following arguments will be added to cmake command: {env_cmake_args}" ) + # Windows specific version of the libraries if platform.startswith("win"): generator = os.getenv("VIZDOOM_BUILD_GENERATOR_NAME") if not generator: @@ -160,20 +155,17 @@ def run(self): shutil.copy(sndfile_dll, build_output_path) shutil.copy(openal_dll, build_output_path) - python_standard_lib = sysconfig.get_python_lib(standard_lib=True) - python_root_dir = os.path.dirname(python_standard_lib) - python_library = get_python_library(python_root_dir) - python_include_dir = sysconfig.get_python_inc() - - if python_include_dir and os.path.exists(python_include_dir): - cmake_arg_list.append(f"-DPYTHON_INCLUDE_DIR={python_include_dir}") + #python_standard_lib = sysconfig.get_python_lib(standard_lib=True) + python_root_dir = os.path.dirname(sys.executable) - if python_library and os.path.exists(python_library): - cmake_arg_list.append(f"-DPYTHON_LIBRARY={python_library}") + if python_root_dir and os.path.exists(python_root_dir): + cmake_arg_list.append(f"-DPython_ROOT_DIR={python_root_dir}") if os.path.exists("CMakeCache.txt"): os.remove("CMakeCache.txt") + print(f"Running cmake with arguments: {cmake_arg_list}", file=sys.stderr) + try: if platform.startswith("win"): if os.path.exists("./src/lib_python/libvizdoom_python.dir"): @@ -214,7 +206,8 @@ def run(self): package_dir={"vizdoom": package_path}, package_data={"vizdoom": package_data}, include_package_data=True, - cmdclass={"bdist_wheel": Wheel, "build": BuildCommand}, + cmdclass={"bdist_wheel": Wheel, "build": BuildCommand, "install": InstallPlatlib}, + distclass=BinaryDistribution, platforms=supported_platforms, classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/src/lib_python/CMakeLists.txt b/src/lib_python/CMakeLists.txt index a720239ba..e1e2ec8ec 100644 --- a/src/lib_python/CMakeLists.txt +++ b/src/lib_python/CMakeLists.txt @@ -1,17 +1,24 @@ -find_package(PythonInterp 3 REQUIRED) +set(BUILD_PYTHON_VERSION "" CACHE STRING "Version of Python to build bindings for") -set(PYTHON_VERSION_FULL ${PYTHON_VERSION_STRING}) -STRING(REGEX REPLACE "([0-9]+\\.[0-9]+).*" "\\1" PYTHON_VERSION_MINOR ${PYTHON_VERSION_STRING}) -find_package(PythonLibs ${PYTHON_VERSION_MINOR} REQUIRED) +if("${BUILD_PYTHON_VERSION}" STREQUAL "") + # Try to find Python 3 if BUILD_PYTHON_VERSION is not set + message(STATUS "BUILD_PYTHON_VERSION is not set, trying to find Python 3, Python_ROOT_DIR is ${Python_ROOT_DIR}") + find_package(Python 3 REQUIRED) + set(BUILD_PYTHON_VERSION ${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}) +endif() + +set (PYBIND11_PYTHON_VERSION ${BUILD_PYTHON_VERSION}) -set(VIZDOOM_PYTHON_OUTPUT_DIR ${VIZDOOM_OUTPUT_DIR}/python${PYTHON_VERSION_MINOR}) +set(VIZDOOM_PYTHON_OUTPUT_DIR ${VIZDOOM_OUTPUT_DIR}/python${BUILD_PYTHON_VERSION}) set(VIZDOOM_PYTHON_SRC_DIR ${VIZDOOM_SRC_DIR}/lib_python) set(VIZDOOM_PYTHON_INCLUDE_DIR ${VIZDOOM_INCLUDE_DIR} ${VIZDOOM_PYTHON_SRC_DIR}) +message(STATUS "Specified Python version to use: ${BUILD_PYTHON_VERSION}") + if(NOT EXISTS ${VIZDOOM_PYTHON_SRC_DIR}/pybind11/CMakeLists.txt) find_package(Git QUIET) if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git") - message(STATUS "Pybind11 submodule update") + message(STATUS "pybind11 submodule update") execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} RESULT_VARIABLE GIT_SUBMOD_RESULT) @@ -34,7 +41,7 @@ include_directories(${VIZDOOM_PYTHON_INCLUDE_DIR} ${Boost_INCLUDE_DIR}) set(ViZDoom_PYTHON_INLCUDE_DIR ${VIZDOOM_PYTHON_INCLUDE_DIR}) -set(ViZDoom_PYTHON_VERSION_STR ${PYTHON_VERSION_MINOR}) +set(ViZDoom_PYTHON_VERSION_STR ${BUILD_PYTHON_VERSION}) configure_file( "${VIZDOOM_PYTHON_SRC_DIR}/__init__.py.in" @@ -58,15 +65,15 @@ set_target_properties(libvizdoom_python LIBRARY_OUTPUT_DIRECTORY_MINSIZEREL ${VIZDOOM_PYTHON_OUTPUT_DIR} LIBRARY_OUTPUT_DIRECTORY_RELWITHDEBINFO ${VIZDOOM_PYTHON_OUTPUT_DIR} OUTPUT_NAME vizdoom - PROJECT_LABEL "python${PYTHON_VERSION_MINOR} binding") + PROJECT_LABEL "python${BUILD_PYTHON_VERSION} binding") if (UNIX) add_custom_target(python_pip_package ALL - COMMAND ${VIZDOOM_SCRIPTS_DIR}/assemble_pip_package.sh ${PYTHON_VERSION_MINOR} ${VIZDOOM_OUTPUT_DIR} ${CMAKE_SOURCE_DIR} + COMMAND ${VIZDOOM_SCRIPTS_DIR}/assemble_pip_package.sh ${BUILD_PYTHON_VERSION} ${VIZDOOM_OUTPUT_DIR} ${CMAKE_SOURCE_DIR} COMMENT "Assembling pip package in ${VIZDOOM_PYTHON_OUTPUT_DIR}/pip_package") elseif (WIN32) add_custom_target(python_pip_package ALL - COMMAND ${VIZDOOM_SCRIPTS_DIR}/assemble_pip_package.bat ${PYTHON_VERSION_MINOR} ${VIZDOOM_OUTPUT_DIR} ${CMAKE_SOURCE_DIR} + COMMAND ${VIZDOOM_SCRIPTS_DIR}/assemble_pip_package.bat ${BUILD_PYTHON_VERSION} ${VIZDOOM_OUTPUT_DIR} ${CMAKE_SOURCE_DIR} COMMENT "Assembling pip package in ${VIZDOOM_PYTHON_OUTPUT_DIR}/pip_package") endif () @@ -75,4 +82,4 @@ set_target_properties(python_pip_package PROJECT_LABEL "Python module") # vizdoom, vizdoom.pk3 are needed to assemble the package -add_dependencies(python_pip_package libvizdoom_python vizdoom pk3) \ No newline at end of file +add_dependencies(python_pip_package libvizdoom_python vizdoom pk3) From 4c661a2f3e3b966221aa305114fdd039071cf8c8 Mon Sep 17 00:00:00 2001 From: Marek Wydmuch Date: Mon, 4 Sep 2023 03:29:56 +0200 Subject: [PATCH 02/12] Update required CMake versions, add support for earlier Boost versions --- CMakeLists.txt | 2 +- README.md | 2 +- docs/introduction/building.md | 6 +++--- docs/introduction/pythonQuickstart.md | 2 +- src/lib/ViZDoomController.cpp | 5 +++++ src/lib/ViZDoomController.h | 3 +++ src/vizdoom/CMakeLists.txt | 2 +- src/vizdoom/bzip2/CMakeLists.txt | 2 +- src/vizdoom/dumb/CMakeLists.txt | 2 +- src/vizdoom/game-music-emu/CMakeLists.txt | 2 +- src/vizdoom/gdtoa/CMakeLists.txt | 2 +- src/vizdoom/jpeg-6b/CMakeLists.txt | 2 +- src/vizdoom/lzma/CMakeLists.txt | 2 +- src/vizdoom/output_sdl/CMakeLists.txt | 2 +- src/vizdoom/src/CMakeLists.txt | 2 +- src/vizdoom/tools/CMakeLists.txt | 2 +- src/vizdoom/tools/fixrtext/CMakeLists.txt | 2 +- src/vizdoom/tools/lemon/CMakeLists.txt | 2 +- src/vizdoom/tools/re2c/CMakeLists.txt | 2 +- src/vizdoom/tools/updaterevision/CMakeLists.txt | 2 +- src/vizdoom/tools/zipdir/CMakeLists.txt | 2 +- src/vizdoom/wadsrc/CMakeLists.txt | 2 +- src/vizdoom/zlib/CMakeLists.txt | 2 +- 23 files changed, 31 insertions(+), 23 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8eb5a5536..671c69df3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,7 +77,7 @@ endif (APPLE) if (CMAKE_COMPILER_IS_GNUCXX) add_definitions("-fPIC") # Explicitly silence some warnings - add_definitions("-Wno-implicit-fallthrough -Wno-shift-negative-value") + add_definitions("-Wno-implicit-fallthrough -Wno-shift-negative-value -Wno-stringop-truncation") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-class-memaccess -Wno-deprecated-copy") # C++ only endif () diff --git a/README.md b/README.md index f1e5a8878..19ee1fc48 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ or ### Linux Both x86-64 and ARM64 architectures are supported. -ViZDoom requires C++11 compiler, CMake 3.4+, Boost 1.65+ SDL2, OpenAL (optional) and Python 3.8+. Below you will find instructrion how to install these dependencies. +ViZDoom requires C++11 compiler, CMake 3.12+, Boost 1.65+ SDL2, OpenAL (optional) and Python 3.8+. Below you will find instructrion how to install these dependencies. #### apt-based distros (Ubuntu, Debian, Linux Mint, etc.) diff --git a/docs/introduction/building.md b/docs/introduction/building.md index 0f79b2592..21c5f8977 100644 --- a/docs/introduction/building.md +++ b/docs/introduction/building.md @@ -20,7 +20,7 @@ Even if you plan to install ViZDoom via pip, you need to install some dependenci ### Linux -* CMake 3.4+ +* CMake 3.12+ * Make * GCC 6.0+ * Boost libraries 1.65.0+ @@ -66,7 +66,7 @@ python setup.py build && python setup.py install ### MacOS -* CMake 3.4+ +* CMake 3.12+ * Clang 5.0+ * Boost libraries 1.65.0+ * Python 3.8+ for Python binding (optional) @@ -85,7 +85,7 @@ brew install cmake boost openal-soft sdl2 ### Windows -* CMake 3.4+ +* CMake 3.12+ * Visual Studio 2012+ * Boost 1.65+ * Python 3.8+ for Python binding (optional) diff --git a/docs/introduction/pythonQuickstart.md b/docs/introduction/pythonQuickstart.md index bed565bf3..626c25b5b 100644 --- a/docs/introduction/pythonQuickstart.md +++ b/docs/introduction/pythonQuickstart.md @@ -2,7 +2,7 @@ ## Linux Both x86-64 and ARM64 architectures are supported. -ViZDoom requires C++11 compiler, CMake 3.4+, Boost 1.65+ SDL2, OpenAL (optional) and Python 3.8+. Below you will find instructrion how to install these dependencies. +ViZDoom requires C++11 compiler, CMake 3.12+, Boost 1.65+ SDL2, OpenAL (optional) and Python 3.8+. Below you will find instructrion how to install these dependencies. ### apt-based distros (Ubuntu, Debian, Linux Mint, etc.) diff --git a/src/lib/ViZDoomController.cpp b/src/lib/ViZDoomController.cpp index 69d4c08fa..3120f75e4 100644 --- a/src/lib/ViZDoomController.cpp +++ b/src/lib/ViZDoomController.cpp @@ -1130,7 +1130,12 @@ namespace vizdoom { void DoomController::handleSignals() { this->ioService = new ba::io_service(); ba::signal_set signals(*this->ioService, SIGINT, SIGABRT, SIGTERM); + + #if BOOST_VERSION >= 106000 signals.async_wait(b::bind(signalHandler, b::ref(signals), this, bpl::_1, bpl::_2)); + #else + signals.async_wait(b::bind(signalHandler, b::ref(signals), this, _1, _2)); + #endif this->ioService->run(); } diff --git a/src/lib/ViZDoomController.h b/src/lib/ViZDoomController.h index 1cabcf9c4..2591c9cba 100644 --- a/src/lib/ViZDoomController.h +++ b/src/lib/ViZDoomController.h @@ -36,13 +36,16 @@ #include #include #include +#include #include #include namespace vizdoom { namespace b = boost; +#if BOOST_VERSION >= 106000 namespace bpl = boost::placeholders; +#endif namespace ba = boost::asio; namespace bip = boost::interprocess; namespace br = boost::random; diff --git a/src/vizdoom/CMakeLists.txt b/src/vizdoom/CMakeLists.txt index 95e0273e4..0c740c121 100644 --- a/src/vizdoom/CMakeLists.txt +++ b/src/vizdoom/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required( VERSION 3.4 ) +cmake_minimum_required( VERSION 3.5 ) if( COMMAND cmake_policy ) if( POLICY CMP0011 ) diff --git a/src/vizdoom/bzip2/CMakeLists.txt b/src/vizdoom/bzip2/CMakeLists.txt index 2517e17f1..8366bd389 100644 --- a/src/vizdoom/bzip2/CMakeLists.txt +++ b/src/vizdoom/bzip2/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required( VERSION 3.4 ) +cmake_minimum_required( VERSION 3.5 ) make_release_only() diff --git a/src/vizdoom/dumb/CMakeLists.txt b/src/vizdoom/dumb/CMakeLists.txt index ad939f193..2b61d446b 100644 --- a/src/vizdoom/dumb/CMakeLists.txt +++ b/src/vizdoom/dumb/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required( VERSION 3.4 ) +cmake_minimum_required( VERSION 3.5 ) make_release_only() diff --git a/src/vizdoom/game-music-emu/CMakeLists.txt b/src/vizdoom/game-music-emu/CMakeLists.txt index d181fbde5..3bc817ad2 100644 --- a/src/vizdoom/game-music-emu/CMakeLists.txt +++ b/src/vizdoom/game-music-emu/CMakeLists.txt @@ -10,7 +10,7 @@ set(GME_VERSION 0.6.0 CACHE INTERNAL "libgme Version") # Of course, 2.4 might work, in which case you're welcome to drop # down the requirement, but I can't test that. #cmake_minimum_required(VERSION 2.6 FATAL_ERROR) -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) make_release_only() diff --git a/src/vizdoom/gdtoa/CMakeLists.txt b/src/vizdoom/gdtoa/CMakeLists.txt index efcf48d59..62df86545 100644 --- a/src/vizdoom/gdtoa/CMakeLists.txt +++ b/src/vizdoom/gdtoa/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) set( CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -D_DEBUG" ) diff --git a/src/vizdoom/jpeg-6b/CMakeLists.txt b/src/vizdoom/jpeg-6b/CMakeLists.txt index a68a33b0e..563254ae8 100644 --- a/src/vizdoom/jpeg-6b/CMakeLists.txt +++ b/src/vizdoom/jpeg-6b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) make_release_only() diff --git a/src/vizdoom/lzma/CMakeLists.txt b/src/vizdoom/lzma/CMakeLists.txt index fc2d6ba1b..b87a0672d 100644 --- a/src/vizdoom/lzma/CMakeLists.txt +++ b/src/vizdoom/lzma/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) make_release_only() diff --git a/src/vizdoom/output_sdl/CMakeLists.txt b/src/vizdoom/output_sdl/CMakeLists.txt index 5f574843d..1fdc21128 100644 --- a/src/vizdoom/output_sdl/CMakeLists.txt +++ b/src/vizdoom/output_sdl/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) if( NOT NO_FMOD AND FMOD_INCLUDE_DIR ) add_library( output_sdl MODULE output_sdl.c ) include_directories( ${FMOD_INCLUDE_DIR} ${SDL2_INCLUDE_DIR} ) diff --git a/src/vizdoom/src/CMakeLists.txt b/src/vizdoom/src/CMakeLists.txt index 92b9b713a..bb7042cb9 100644 --- a/src/vizdoom/src/CMakeLists.txt +++ b/src/vizdoom/src/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}" "${PROJECT_SOURCE_DIR}/cmake_modules") diff --git a/src/vizdoom/tools/CMakeLists.txt b/src/vizdoom/tools/CMakeLists.txt index b991d2dcc..a04ca45cb 100644 --- a/src/vizdoom/tools/CMakeLists.txt +++ b/src/vizdoom/tools/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) add_subdirectory( lemon ) add_subdirectory( re2c ) diff --git a/src/vizdoom/tools/fixrtext/CMakeLists.txt b/src/vizdoom/tools/fixrtext/CMakeLists.txt index 4abe68b78..6773ba9a7 100644 --- a/src/vizdoom/tools/fixrtext/CMakeLists.txt +++ b/src/vizdoom/tools/fixrtext/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) if( NOT CMAKE_CROSSCOMPILING ) add_executable( fixrtext fixrtext.c ) diff --git a/src/vizdoom/tools/lemon/CMakeLists.txt b/src/vizdoom/tools/lemon/CMakeLists.txt index 056dbf932..f1097ed1f 100644 --- a/src/vizdoom/tools/lemon/CMakeLists.txt +++ b/src/vizdoom/tools/lemon/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) if( NOT CMAKE_CROSSCOMPILING ) set( CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -D_DEBUG" ) diff --git a/src/vizdoom/tools/re2c/CMakeLists.txt b/src/vizdoom/tools/re2c/CMakeLists.txt index 730f6d9c9..e037bd2a3 100644 --- a/src/vizdoom/tools/re2c/CMakeLists.txt +++ b/src/vizdoom/tools/re2c/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) if( NOT CMAKE_CROSSCOMPILING ) diff --git a/src/vizdoom/tools/updaterevision/CMakeLists.txt b/src/vizdoom/tools/updaterevision/CMakeLists.txt index 3a87d2286..3d65085c0 100644 --- a/src/vizdoom/tools/updaterevision/CMakeLists.txt +++ b/src/vizdoom/tools/updaterevision/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) if( WIN32 ) if( ZD_CMAKE_COMPILER_IS_GNUC_COMPATIBLE OR ZD_CMAKE_COMPILER_IS_GNUCXX_COMPATIBLE ) diff --git a/src/vizdoom/tools/zipdir/CMakeLists.txt b/src/vizdoom/tools/zipdir/CMakeLists.txt index 83f951f3b..1f015c550 100644 --- a/src/vizdoom/tools/zipdir/CMakeLists.txt +++ b/src/vizdoom/tools/zipdir/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) if(NOT CMAKE_CROSSCOMPILING) message(STATUS "${ZLIB_INCLUDE_DIR}" ) diff --git a/src/vizdoom/wadsrc/CMakeLists.txt b/src/vizdoom/wadsrc/CMakeLists.txt index 5da5f0af3..980d45013 100644 --- a/src/vizdoom/wadsrc/CMakeLists.txt +++ b/src/vizdoom/wadsrc/CMakeLists.txt @@ -1,3 +1,3 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) add_pk3(vizdoom.pk3 ${CMAKE_CURRENT_SOURCE_DIR}/static) diff --git a/src/vizdoom/zlib/CMakeLists.txt b/src/vizdoom/zlib/CMakeLists.txt index 3543b82e4..00a53ecfc 100644 --- a/src/vizdoom/zlib/CMakeLists.txt +++ b/src/vizdoom/zlib/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) set(CMAKE_ALLOW_LOOSE_LOOP_CONSTRUCTS ON) make_release_only() From 59b119f55d1ffbbeee26d6fc0264976244e57ff3 Mon Sep 17 00:00:00 2001 From: Marek Wydmuch Date: Mon, 4 Sep 2023 03:30:28 +0200 Subject: [PATCH 03/12] Update pyproject.toml with cibuildwheel config --- pyproject.toml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c926ee83a..a3d13c4dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,6 @@ [build-system] -requires = ["cmake>=3.1.0", "setuptools", "wheel"] +requires-python = ">=3.8" +requires = ["cmake>=3.12.0", "setuptools", "wheel"] [tool.black] safe = true @@ -30,3 +31,19 @@ reportMissingImports = "none" reportGeneralTypeIssues = "none" reportPrivateUsage = "warning" reportPrivateImportUsage = "warning" + +[tool.cibuildwheel] +build = "cp{38,39,310,311}-*" +build-verbosity = 3 + +[tool.cibuildwheel.linux] +# For manylinux_2_28 we need to install the following dependencies: +before-all = "yum install -y cmake git boost-devel SDL2-devel openal-soft-devel" +repair-wheel-command = "auditwheel show {wheel} && auditwheel repair {wheel} -w {dest_dir}" + +manylinux-x86_64-image = "manylinux_2_28" +manylinux-i686-image = "manylinux_2_28" +manylinux-aarch64-image = "manylinux_2_28" + +[tool.cibuildwheel.macos] +before-all = "brew install cmake boost sdl2 openal-soft" From 7587ba8a6c1ba39cc4b5c8bcb1be6e0ad40c211e Mon Sep 17 00:00:00 2001 From: Marek Wydmuch Date: Wed, 6 Sep 2023 01:10:36 +0200 Subject: [PATCH 04/12] Update the docs on building/installing ViZDoom --- README.md | 45 ++++---- docs/introduction/building.md | 142 +++++++++++++++----------- docs/introduction/pythonQuickstart.md | 55 +++++----- 3 files changed, 129 insertions(+), 113 deletions(-) diff --git a/README.md b/README.md index 19ee1fc48..7c9c62192 100644 --- a/README.md +++ b/README.md @@ -71,57 +71,56 @@ or ## Python quick start ### Linux -Both x86-64 and ARM64 architectures are supported. -ViZDoom requires C++11 compiler, CMake 3.12+, Boost 1.65+ SDL2, OpenAL (optional) and Python 3.8+. Below you will find instructrion how to install these dependencies. +To install the latest release of ViZDoom, just run: +``` +pip install vizdoom +``` +Both x86-64 and AArch64 (ARM64) architectures are supported. + +If Python wheel is not available for your platform (Python version <3.8, distros below manylinux_2_28 standard), pip will try to install (build) ViZDoom from source. +ViZDoom requires C++11 compiler, CMake 3.12+, Boost 1.54+ SDL2, OpenAL (optional) and Python 3.7+. Below you will find instructrion how to install these dependencies. #### apt-based distros (Ubuntu, Debian, Linux Mint, etc.) -To install ViZDoom run (may take few minutes): +To build ViZDoom run (it may take few minutes): ``` apt install cmake git libboost-all-dev libsdl2-dev libopenal-dev pip install vizdoom ``` -We recommend using at least Ubuntu 18.04+ or Debian 10+ with Python 3.8+. +We recommend using at least Ubuntu 18.04+ or Debian 10+ with Python 3.7+. #### dnf/yum-based distros (Fedora, RHEL, CentOS, Alma/Rocky Linux, etc.) -To install ViZDoom run (may take few minutes): +To install ViZDoom run (it may take few minutes): ``` dnf install cmake git boost-devel SDL2-devel openal-soft-devel pip install vizdoom ``` -We recommend using at least Fedora 35+ or RHEL/CentOS/Alma/Rocky Linux 9+ with Python 3.8+. To install openal-soft-devel on RHEL/CentOS/Alma/Rocky Linux 9, one needs to use `dnf --enablerepo=crb install`. - -#### Conda-based installation -To install ViZDoom on a conda environment (no system-wide installations required): -``` -conda install -c conda-forge boost cmake sdl2 -git clone https://github.com/mwydmuch/ViZDoom.git --recurse-submodules -cd ViZDoom -python setup.py build && python setup.py install -``` -Note that `pip install vizdoom` won't work with conda install and you have to follow these steps. +We recommend using at least Fedora 35+ or RHEL/CentOS/Alma/Rocky Linux 9+ with Python 3.7+. +To install openal-soft-devel on RHEL/CentOS/Alma/Rocky Linux 9, one needs to use `dnf --enablerepo=crb install`. ### macOS -Both Intel and Apple Silicon CPUs are supported. - -To install ViZDoom on run (may take few minutes): +To install the latest release of ViZDoom just run (it may take few minutes as it will build ViZDoom from source on M1/M2 chips): ``` brew install cmake git boost openal-soft sdl2 pip install vizdoom ``` -We recommend using at least macOS High Sierra 10.13+ with Python 3.8+. -On Apple Silicon (M1 and M2), make sure you are using Python for Apple Silicon. +Both Intel and Apple Silicon CPUs are supported. +We recommend using at least macOS High Sierra 10.13+ with Python 3.7+. +On Apple Silicon (M1 and M2), make sure you are using Python/Pip for Apple Silicon. ### Windows -To install pre-build release for Windows 10 or 11 64-bit and Python 3.8+ just run (should take few seconds): +To install the latest release of ViZDoom, just run: ``` pip install vizdoom ``` +At the moment only x86-64 architecture is supported on Windows. -Please note that the Windows version is not as well-tested as Linux and macOS versions. It can be used for development and testing if you want to conduct experiments on Windows, please consider using [Docker](https://docs.docker.com/docker-for-windows/install/) or [WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10). +Please note that the Windows version is not as well-tested as Linux and macOS versions. +It can be used for development and testing but if you want to conduct serious (time and resource-extensive) experiments on Windows, +please consider using [Docker](https://docs.docker.com/docker-for-windows/install/) or [WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10) with Linux version. ### Gymnasium/Gym wrappers diff --git a/docs/introduction/building.md b/docs/introduction/building.md index 21c5f8977..b0fc8e11d 100644 --- a/docs/introduction/building.md +++ b/docs/introduction/building.md @@ -1,41 +1,37 @@ # Building from source -- [Installation and building](#installation-and-building) - - [ Dependencies](#-dependencies) - - [ Linux](#-linux) - - [ MacOS](#-macos) - - [ Building](#-building) - - [ Windows](#-windows) - - [ Installation via pip (recommended for Python users)](#-installation-via-pip-recommended-for-python-users) - - [ Building manually (not recommended)](#-building-manually-not-recommended) - - [ Linux / MacOS](#-linux--macos) - - [ Windows](#-windows-1) - - [ Compilation output](#-compilation-output) - - [ Manual installation](#-manual-installation) +Here we describe how to build ViZDoom from source. +If you want to install pre-build ViZDoom wheels for Python, see [Python quick start](./pythonQuickstart.md). -## Dependencies +## Dependencies -Even if you plan to install ViZDoom via pip, you need to install some dependencies in your system first. +To build ViZDoom (regardless of the method), you need to install some dependencies in your system first. -### Linux +### Linux + +To build ViZDoom on Linux, the following dependencies are required: * CMake 3.12+ * Make * GCC 6.0+ -* Boost libraries 1.65.0+ -* Python 3.8+ for Python binding (optional) +* Boost libraries 1.54.0+ +* Python 3.7+ for Python binding (optional) -Additionally, [ZDoom dependencies](http://zdoom.org/wiki/Compile_ZDoom_on_Linux) are needed. +Also some of additionally [ZDoom dependencies](http://zdoom.org/wiki/Compile_ZDoom_on_Linux) are needed. -To get all dependencies on apt-based Linux (Ubuntu, Debian, Linux Mint, etc.) execute the following commands in the shell (might require root access). -```bash -# All ZDoom dependencies (most are optional) -apt install build-essential zlib1g-dev libsdl2-dev libjpeg-dev \ -tar libbz2-dev libgtk2.0-dev cmake git libfluidsynth-dev libgme-dev \ -libopenal-dev timidity libwildmidi-dev unzip libboost-all-dev +#### apt-based distros (Ubuntu, Debian, Linux Mint, etc.) -# Only essential ZDoom dependencies +To get all dependencies on apt-based Linux (Ubuntu, Debian, Linux Mint, etc.) execute the following commands in the shell (might require root access). +```sh +# All possible ViZDoom dependencies, +# most are optional and required only to support alternative sound and music backends in the engine +# other can replace libraries that are included in the ViZDoom repository +apt install build-essential cmake git libsdl2-dev libboost-all-dev libopenal-dev \ +zlib1g-dev libjpeg-dev tar libbz2-dev libgtk2.0-dev libfluidsynth-dev libgme-dev \ +timidity libwildmidi-dev unzip + +# Only essential ViZDoom dependencies apt install build-essential cmake git libboost-all-dev libsdl2-dev libopenal-dev # Python 3 dependencies (alternatively Anaconda 3 installed) @@ -43,8 +39,10 @@ apt install python3-dev python3-pip # or install Anaconda 3 and add it to PATH ``` +#### dnf/yum-based distros (Fedora, RHEL, CentOS, Alma/Rocky Linux, etc.) + To get all dependencies on dnf/yum-based Linux (Fedora, RHEL, CentOS, Alma/Rocky Linux, etc.) execute the following commands in the shell (might require root access). -```bash +```sh # Essential ZDoom dependencies dnf install cmake git boost-devel SDL2-devel openal-soft-devel @@ -52,8 +50,11 @@ dnf install cmake git boost-devel SDL2-devel openal-soft-devel dnf install python3-devel python3-pip ``` + +#### Anaconda/Miniconda + If you do not have a root access, you can use a conda (e.g. [miniconda](https://docs.conda.io/en/latest/miniconda.html)) environment to install dependencies to your environment only: -``` +```sh conda install -c conda-forge boost cmake gtk2 sdl2 ``` @@ -65,61 +66,75 @@ python setup.py build && python setup.py install ``` -### MacOS +### MacOS + +To build ViZDoom on MacOS, the following dependencies are required: * CMake 3.12+ * Clang 5.0+ -* Boost libraries 1.65.0+ -* Python 3.8+ for Python binding (optional) -## Building +* Boost libraries 1.54.0+ +* Python 3.7+ for Python binding (optional) -Additionally, [ZDoom dependencies](http://zdoom.org/wiki/Compile_ZDoom_on_Mac_OS_X) are needed. - -To get dependencies install [homebrew](https://brew.sh/) +Also some of additionally [ZDoom dependencies](http://zdoom.org/wiki/Compile_ZDoom_on_Mac_OS_X) are needed. +To get all the dependencies install [homebrew](https://brew.sh/) first, than execute the following commands in the shell: ```sh -# ZDoom dependencies and Boost libraries -brew install cmake boost openal-soft sdl2 - -# You can use system python or install Anaconda 3 and add it to PATH +brew install cmake boost sdl2 openal-soft ``` -### Windows +### Windows * CMake 3.12+ * Visual Studio 2012+ -* Boost 1.65+ -* Python 3.8+ for Python binding (optional) +* Boost libraries 1.54.0+ +* Python 3.7+ for Python binding (optional) Additionally, [ZDoom dependencies](http://zdoom.org/wiki/Compile_ZDoom_on_Windows) are needed. Most of them (except Boost) are gathered in this repository: [ViZDoomWinDepBin](https://github.com/mwydmuch/ViZDoomWinDepBin). You can download Boost from [here](https://www.boost.org/users/download). -## Installation via pip (recommended for Python users) +## Building via pip (recommended for Python users) -ViZDoom for Python can be installed via **pip** on Linux, MacOS and Windows, and it is strongly recommended. -However you will still need to install **[Linux](#linux_deps)/[MacOS](#macos_deps) dependencies**, as it will be build locally from source. -For Windows 10 or 11 64-bit and Python 3.8+ we provide pre-build wheels (binary packages). +ViZDoom for Python can be build via **pip** on Linux, MacOS and Windows, and it is strongly recommended. +Even when building using pip you still need to install dependencies. - -To install the most stable official release from [PyPI](https://pypi.python.org/pypi): -```bash -pip install vizdoom +To build the newest version from the repository run: +```sh +pip install git+https://github.com/mwydmuch/ViZDoom.git +``` +or +```sh +git clone https://github.com/mwydmuch/ViZDoom.git +cd ViZDoom +pip install . ``` -To install the newest version from the repository (only Linux and MacOS): -```bash -pip install git+https://github.com/mwydmuch/ViZDoom.git +On Linux and MacOS dependencies should be found automatically. +On Windows you need to manually set following environment variables: +* `BOOST_ROOT` - the path to the directory with Boost libraries (e.g. `C:\boost_1_76_0`), +* `VIZDOOM_BUILD_GENERATOR_NAME` - generator name (e.g. `Visual Studio 16 2019`), +* `VIZDOOM_WIN_DEPS_ROOT` - the path to the directory with ZDoom dependencies (e.g. `C:\ViZDoomWinDepBin`). + +The process of building ViZDoom this way on Windows is demonstarted in [scripts/windows_build_wheels.bat](https://github.com/Farama-Foundation/ViZDoom/tree/master/scripts/windows_build_wheels.bat). + + +## Building manylinux wheels + +To build manylinux wheels you need to install docker and cibuildwheel. Then on Linux and MacOS run in ViZDoom root directory: +```sh +cibuildwheel --platform linux ``` +The binary ViZDoom wheels will be placed in `wheelhouse` directory. +In case of building using cibuildwheel, the dependencies are installed automatically inside the docker container, so you do not need to install them manually in your system. -## Building manually (not recommended) -Instructions below can be used to build ViZDoom manually. +## Building manually (not recommended) -### Linux / MacOS +Instructions below can be used to build ViZDoom manually. +We recommend doing it only if you want to use C++ API, work on the ViZDoom, or if you have problems with pip installation. ->>> Using [pip](#pypi) is the recommended way to install ViZDoom, please try it first unless you are sure you want to compile the package by hand. +### Linux / MacOS In ViZDoom's root directory: ```bash @@ -131,9 +146,10 @@ make where `-DBUILD_ENGINE=ON` and `-DBUILD_PYTHON=ON` CMake options are optional (default ON). -### Windows -Run CMake GUI, select ViZDoom root directory and set paths to: +### Windows + +1. Run CMake GUI or cmake command in cmd/powershell in ViZDoom root directory with the following paths provided: * BOOST_ROOT * BOOST_INCLUDEDIR * BOOST_LIBRARYDIR @@ -141,12 +157,14 @@ Run CMake GUI, select ViZDoom root directory and set paths to: * PYTHON_LIBRARY (optional, for Python/Anaconda bindings) * ZDoom dependencies paths -In configuration select `DBUILD_ENGINE` and `DBUILD_PYTHON` (optional, default ON). +2. In configuration select `DBUILD_ENGINE` and `DBUILD_PYTHON` (optional, default ON). + +3. Use generated Visual Studio solution to build all parts of ViZDoom environment. -Use generated Visual Studio solution to build all parts of ViZDoom environment. +The process of building ViZDoom this way on Windows is demonstarted in [scripts/windows_build_cmake.bat](https://github.com/Farama-Foundation/ViZDoom/tree/master/scripts/windows_build_cmake.bat) script. -### Compilation output +### Compilation output Compilation output will be placed in `build/bin` and it should contain the following files. * `bin/vizdoom / vizdoom.exe` - ViZDoom executable @@ -157,6 +175,6 @@ Compilation output will be placed in `build/bin` and it should contain the follo * `bin/pythonX.X/pip_package` - complete ViZDoom Python X.X package -### Manual installation +### Manual installation To manually install Python package copy `vizdoom_root_dir/build/bin/pythonX.X/pip_package` contents to `python_root_dir/lib/pythonX.X/site-packages/site-packages/vizdoom`. diff --git a/docs/introduction/pythonQuickstart.md b/docs/introduction/pythonQuickstart.md index 626c25b5b..bbe76751a 100644 --- a/docs/introduction/pythonQuickstart.md +++ b/docs/introduction/pythonQuickstart.md @@ -1,54 +1,53 @@ # Python quick start ## Linux -Both x86-64 and ARM64 architectures are supported. -ViZDoom requires C++11 compiler, CMake 3.12+, Boost 1.65+ SDL2, OpenAL (optional) and Python 3.8+. Below you will find instructrion how to install these dependencies. +To install the latest release of ViZDoom, just run: +```sh +pip install vizdoom +``` +Both x86-64 and AArch64 (ARM64) architectures are supported. + +If Python wheel is not available for your platform (Python version <3.8, distros below manylinux_2_28 standard), pip will try to install (build) ViZDoom from source. +ViZDoom requires C++11 compiler, CMake 3.12+, Boost 1.54+ SDL2, OpenAL (optional) and Python 3.7+. Below you will find instructrion how to install these dependencies. ### apt-based distros (Ubuntu, Debian, Linux Mint, etc.) -To install ViZDoom run (may take few minutes): -``` +To build ViZDoom run (it may take few minutes): +```sh apt install cmake git libboost-all-dev libsdl2-dev libopenal-dev pip install vizdoom ``` -We recommend using at least Ubuntu 18.04+ or Debian 10+ with Python 3.8+. +We recommend using at least Ubuntu 18.04+ or Debian 10+ with Python 3.7+. ### dnf/yum-based distros (Fedora, RHEL, CentOS, Alma/Rocky Linux, etc.) -To install ViZDoom run (may take few minutes): -``` +To install ViZDoom run (it may take few minutes): +```sh dnf install cmake git boost-devel SDL2-devel openal-soft-devel pip install vizdoom ``` -We recommend using at least Fedora 35+ or RHEL/CentOS/Alma/Rocky Linux 9+ with Python 3.8+. To install openal-soft-devel on RHEL/CentOS/Alma/Rocky Linux 9, one needs to use `dnf --enablerepo=crb install`. - -### Conda-based installation -To install ViZDoom on a conda environment (no system-wide installations required): -``` -conda install -c conda-forge boost cmake sdl2 -git clone https://github.com/mwydmuch/ViZDoom.git --recurse-submodules -cd ViZDoom -python setup.py build && python setup.py install -``` -Note that `pip install vizdoom` won't work with conda install and you have to follow these steps. +We recommend using at least Fedora 35+ or RHEL/CentOS/Alma/Rocky Linux 9+ with Python 3.7+. +To install openal-soft-devel on RHEL/CentOS/Alma/Rocky Linux 9, one needs to use `dnf --enablerepo=crb install`. ## macOS -Both Intel and Apple Silicon CPUs are supported. - -To install ViZDoom on run (may take few minutes): -``` -brew install cmake git boost openal-soft sdl2 +To install the latest release of ViZDoom just run (may take few minutes as it will build ViZDoom from source on M1/M2 chips): +```sh +brew install cmake boost sdl2 openal-soft pip install vizdoom ``` -We recommend using at least macOS High Sierra 10.13+ with Python 3.8+. -On Apple Silicon (M1 and M2), make sure you are using Python for Apple Silicon. +Both Intel and Apple Silicon CPUs are supported. +We recommend using at least macOS High Sierra 10.13+ with Python 3.7+. +On Apple Silicon (M1 and M2), make sure you are using Python/Pip for Apple Silicon. ## Windows -To install pre-build release for Windows 10 or 11 64-bit and Python 3.8+ just run (should take few seconds): -``` +To install the latest release of ViZDoom, just run: +```sh pip install vizdoom ``` +At the moment only x86-64 architecture is supported on Windows. -Please note that the Windows version is not as well-tested as Linux and macOS versions. It can be used for development and testing if you want to conduct experiments on Windows, please consider using [Docker](https://docs.docker.com/docker-for-windows/install/) or [WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10). +Please note that the Windows version is not as well-tested as Linux and macOS versions. +It can be used for development and testing but if you want to conduct serious (time and resource-extensive) experiments on Windows, +please consider using [Docker](https://docs.docker.com/docker-for-windows/install/) or [WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10) with Linux version. From ad7242195d10f13340e9c97fdfc44bb7c0fca323 Mon Sep 17 00:00:00 2001 From: Marek Wydmuch Date: Wed, 6 Sep 2023 03:14:39 +0200 Subject: [PATCH 05/12] Update cibuildwheel setting --- .gitignore | 23 ++--------------------- CMakeLists.txt | 4 ++-- pyproject.toml | 12 +++++++----- setup.py | 10 +++++----- 4 files changed, 16 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index 2fd378716..908fcad78 100644 --- a/.gitignore +++ b/.gitignore @@ -4,10 +4,10 @@ ViZDoomVersion.h viz_version.h Makefile -!docs/Makefile ImportExecutables.cmake **/link-make cmake-build-* +src/boost_* *~ *.o @@ -47,20 +47,10 @@ cmake-build-* examples/python/*.cfg examples/python/*.ini dist -build venv .eggs vizdoom.egg-info - -# Lua -examples/lua/*.cfg -examples/lua/*.ini -examples/lua/vizdoom -*.rock - -# Java -*.jar -*.class +wheelhouse # Copied from the original ZDoom repository (and modified) *.cbp @@ -119,14 +109,5 @@ src/vizdoom/zlib/x64/ # CLion & PyCharm **/.idea -# Visual Studio Code -.vscode - # MacOS .DS_Store - -# Dockerfiles -tmp_dockerfiles - -# Documentation -docs_html diff --git a/CMakeLists.txt b/CMakeLists.txt index 671c69df3..f4443762f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ set(ViZDoom_VERSION_ID ${ViZDoom_VERSION_MAJOR}${ViZDoom_VERSION_MINOR}${ViZDoom #----------------------------------------------------------------------------------------------------------------------- option(BUILD_PYTHON "Build ViZDoom Python (3) binding/module" ON) -option(BUILD_ENGINE "Build ViZDoom Engine" ON) +option(BUILD_ENGINE "Build ViZDoom Engine (required to build Python package)" ON) # CMake options @@ -76,7 +76,7 @@ endif (APPLE) if (CMAKE_COMPILER_IS_GNUCXX) add_definitions("-fPIC") - # Explicitly silence some warnings + # Explicitly silence some warnings add_definitions("-Wno-implicit-fallthrough -Wno-shift-negative-value -Wno-stringop-truncation") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-class-memaccess -Wno-deprecated-copy") # C++ only endif () diff --git a/pyproject.toml b/pyproject.toml index a3d13c4dd..ecd0d9096 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,4 @@ [build-system] -requires-python = ">=3.8" requires = ["cmake>=3.12.0", "setuptools", "wheel"] [tool.black] @@ -33,16 +32,19 @@ reportPrivateUsage = "warning" reportPrivateImportUsage = "warning" [tool.cibuildwheel] +# We need to build for the following Python versions: build = "cp{38,39,310,311}-*" -build-verbosity = 3 [tool.cibuildwheel.linux] -# For manylinux_2_28 we need to install the following dependencies: +# Only manylinux is supported (no musl) +build = "cp{38,39,310,311}-manylinux*" + +# For manylinux_2_28 we need to install the following dependencies using yum: before-all = "yum install -y cmake git boost-devel SDL2-devel openal-soft-devel" -repair-wheel-command = "auditwheel show {wheel} && auditwheel repair {wheel} -w {dest_dir}" +# Only build for x86_64 and aarch64 are officially supported +archs = "x86_64 aarch64" manylinux-x86_64-image = "manylinux_2_28" -manylinux-i686-image = "manylinux_2_28" manylinux-aarch64-image = "manylinux_2_28" [tool.cibuildwheel.macos] diff --git a/setup.py b/setup.py index 87377c8bb..d0071d4bc 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ from distutils.command.build import build from multiprocessing import cpu_count -from setuptools import setup, Distribution +from setuptools import Distribution, setup from setuptools.command.install import install from wheel.bdist_wheel import bdist_wheel @@ -88,10 +88,10 @@ def get_tag(self): class BinaryDistribution(Distribution): def has_ext_modules(self): return True - + def is_pure(self): return False - + class InstallPlatlib(install): def finalize_options(self): @@ -107,7 +107,7 @@ def run(self): "cmake", "-DCMAKE_BUILD_TYPE=Release", "-DBUILD_PYTHON=ON", - f"-DBUILD_PYTHON_VERSION={python_version}" + f"-DBUILD_PYTHON_VERSION={python_version}", ] env_cmake_args = os.getenv("VIZDOOM_CMAKE_ARGS") @@ -155,7 +155,7 @@ def run(self): shutil.copy(sndfile_dll, build_output_path) shutil.copy(openal_dll, build_output_path) - #python_standard_lib = sysconfig.get_python_lib(standard_lib=True) + # python_standard_lib = sysconfig.get_python_lib(standard_lib=True) python_root_dir = os.path.dirname(sys.executable) if python_root_dir and os.path.exists(python_root_dir): From 3cecc226e1fefc8bb1e93733d53f0e9e7bd69b66 Mon Sep 17 00:00:00 2001 From: Marek Wydmuch Date: Wed, 6 Sep 2023 03:15:59 +0200 Subject: [PATCH 06/12] Add a workflow for building wheels and publishing them on PyPI, change names in the old workflows --- .../{build.yml => build-and-test.yml} | 11 +-- .github/workflows/build-wheels,yml | 92 +++++++++++++++++++ .github/workflows/pre-commit.yml | 3 +- 3 files changed, 98 insertions(+), 8 deletions(-) rename .github/workflows/{build.yml => build-and-test.yml} (83%) create mode 100644 .github/workflows/build-wheels,yml diff --git a/.github/workflows/build.yml b/.github/workflows/build-and-test.yml similarity index 83% rename from .github/workflows/build.yml rename to .github/workflows/build-and-test.yml index 98784997b..2fa5092c9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build-and-test.yml @@ -1,4 +1,4 @@ -name: Build +name: Build and test on: push: @@ -11,8 +11,7 @@ on: - 'CMakeLists.txt' - 'setup.py' - 'pyproject.toml' - branches: - - master + branches: [master] pull_request: paths: - '.github/workflows/**' @@ -23,8 +22,7 @@ on: - 'CMakeLists.txt' - 'setup.py' - 'pyproject.toml' - branches: - - master + branches: [master] env: HOMEBREW_NO_AUTO_UPDATE: 1 @@ -36,7 +34,6 @@ jobs: matrix: os: [ubuntu-20.04, ubuntu-22.04, macos-12] python-version: ['3.8', '3.9', '3.10', '3.11'] - runs-on: ${{ matrix.os }} steps: @@ -46,7 +43,7 @@ jobs: if: runner.os == 'Linux' run: | sudo apt update - sudo apt install -y cmake libboost-all-dev libsdl2-dev libfreetype6-dev libgl1-mesa-dev libglu1-mesa-dev libpng-dev libjpeg-dev libbz2-dev libfluidsynth-dev libgme-dev libopenal-dev zlib1g-dev timidity tar nasm + sudo apt install -y cmake git libboost-all-dev libsdl2-dev libopenal-dev - name: Apt report if: runner.os == 'Linux' diff --git a/.github/workflows/build-wheels,yml b/.github/workflows/build-wheels,yml new file mode 100644 index 000000000..3dbac6855 --- /dev/null +++ b/.github/workflows/build-wheels,yml @@ -0,0 +1,92 @@ +name: Build Python wheels and make release + +on: + workflow_dispatch: + pull_request: + push: + paths: + - '.github/workflows/**' + - 'include/**' + - 'scripts/**' + - 'src/**' + - 'CMakeLists.txt' + - 'setup.py' + - 'pyproject.toml' + branches: [master] + release: + types: [published] + +jobs: + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-22.04, macos-11] + + steps: + - uses: actions/checkout@v3 + + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v2 + with: + platforms: all + + - name: Build wheels + uses: pypa/cibuildwheel@v2.15.0 + env: + # Configure cibuildwheel to build native archs, and some emulated ones + CIBW_ARCHS_LINUX: x86_64 aarch64 + CIBW_ARCHS_MACOS: x86_64 + CIBW_BUILD_VERBOSITY: 3 + CIBW_REPAIR_WHEEL_COMMAND_LINUX: > + auditwheel show {wheel} && auditwheel repair -w {dest_dir} {wheel} + + - name: Report builded wheels + run: | + ls -l ./wheelhouse/*.whl + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl + + build_sdist: + name: Build source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Build sdist + run: pipx run build --sdist + + - name: Upload sdist + uses: actions/upload-artifact@v3 + with: + path: dist/*.tar.gz + + upload_pypi: + needs: [build_wheels, build_sdist] + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + if: github.event_name == 'release' && github.event.action == 'published' + steps: + - name: Download all dists + uses: actions/download-artifact@v3 + with: + # Unpacks default artifact into dist/ + # If `name: artifact` is omitted, the action will create extra parent dir + name: artifact + path: dist + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} + # To test: + # with: + # repository_url: https://test.pypi.org/legacy/ + diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 094aff310..f79d0fc2a 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -1,6 +1,6 @@ # https://pre-commit.com # This GitHub Action assumes that the repo contains a valid .pre-commit-config.yaml file. -name: Pre-commit +name: Pre-commit checks on: pull_request: push: @@ -11,6 +11,7 @@ permissions: jobs: pre-commit: + name: Pre-commit checks runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 88205098f17ab2ccfa4e882c3ec0bf0291837822 Mon Sep 17 00:00:00 2001 From: Marek Wydmuch Date: Wed, 6 Sep 2023 03:23:55 +0200 Subject: [PATCH 07/12] Fix pre-commit failed checks --- .github/workflows/build-wheels,yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-wheels,yml b/.github/workflows/build-wheels,yml index 3dbac6855..48eb3705d 100644 --- a/.github/workflows/build-wheels,yml +++ b/.github/workflows/build-wheels,yml @@ -37,13 +37,13 @@ jobs: uses: pypa/cibuildwheel@v2.15.0 env: # Configure cibuildwheel to build native archs, and some emulated ones - CIBW_ARCHS_LINUX: x86_64 aarch64 + CIBW_ARCHS_LINUX: x86_64 aarch64 CIBW_ARCHS_MACOS: x86_64 CIBW_BUILD_VERBOSITY: 3 CIBW_REPAIR_WHEEL_COMMAND_LINUX: > auditwheel show {wheel} && auditwheel repair -w {dest_dir} {wheel} - - name: Report builded wheels + - name: Report built wheels run: | ls -l ./wheelhouse/*.whl @@ -89,4 +89,3 @@ jobs: # To test: # with: # repository_url: https://test.pypi.org/legacy/ - From fb1f4a725feb1f78c1c219c7911c32488e146e6a Mon Sep 17 00:00:00 2001 From: Marek Wydmuch Date: Thu, 7 Sep 2023 02:24:16 +0200 Subject: [PATCH 08/12] Add tests for wheels built using cibuildwheel --- .gitignore | 3 + scripts/install_and_test_wheel.sh | 29 ++++++++++ tests/README.md | 5 ++ tests/build_test_cibuildwheel_linux.sh | 55 ++++++++++++++++++ ...ly.sh => build_test_local_linux_builds.sh} | 57 +++++++++---------- .../apt-based.Dockerfile | 0 .../conda-based.Dockerfile | 0 .../dnf-based.Dockerfile | 0 .../apt-based.Dockerfile | 12 ++++ .../conda-based.Dockerfile | 6 ++ .../dnf-based.Dockerfile | 9 +++ 11 files changed, 147 insertions(+), 29 deletions(-) create mode 100644 scripts/install_and_test_wheel.sh create mode 100644 tests/README.md create mode 100644 tests/build_test_cibuildwheel_linux.sh rename tests/{dockerfiles/run_all_localy.sh => build_test_local_linux_builds.sh} (65%) rename tests/{dockerfiles => local_builds_dockerfiles}/apt-based.Dockerfile (100%) rename tests/{dockerfiles => local_builds_dockerfiles}/conda-based.Dockerfile (100%) rename tests/{dockerfiles => local_builds_dockerfiles}/dnf-based.Dockerfile (100%) create mode 100644 tests/wheels_test_dockerfiles/apt-based.Dockerfile create mode 100644 tests/wheels_test_dockerfiles/conda-based.Dockerfile create mode 100644 tests/wheels_test_dockerfiles/dnf-based.Dockerfile diff --git a/.gitignore b/.gitignore index 908fcad78..d9b17428b 100644 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,9 @@ venv vizdoom.egg-info wheelhouse +# Tests +test_dockerfiles + # Copied from the original ZDoom repository (and modified) *.cbp *.ncb diff --git a/scripts/install_and_test_wheel.sh b/scripts/install_and_test_wheel.sh new file mode 100644 index 000000000..ff64ba6d4 --- /dev/null +++ b/scripts/install_and_test_wheel.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -e + +# Set working dir to the root of the repo +cd $( dirname "${BASH_SOURCE[0]}" )/.. + +# Report directory +ls -lha . + +# Report python version +python3 --version +python3 -c "import sys; print('Python', sys.version)" + +# Find matching wheel file in wheelhouse +PYTHON_VERSION=$(python3 -c "import sys; print('{}{}'.format(sys.version_info.major, sys.version_info.minor))") +PYTHON_WHEEL=$(ls wheelhouse/vizdoom-*-cp${PYTHON_VERSION}-cp${PYTHON_VERSION}*.whl) + +# Updgrade pip and install test deps +python3 -m pip install --upgrade pip +python3 -m pip install pytest psutil + +# Install wheel +python3 -m pip install ${PYTHON_WHEEL} + +# Test import +python3 -c "import vizdoom" + +# Run tests +pytest tests diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..f4fc33b42 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,5 @@ +# Tests + +This directory contains the tests for the project that can be run with pytest or by running the `test_*.py` or `manual_test_*.py` files directly. +Manual tests require significant amount of time, so they are not run by default by CI/CD. +The `build_test_*.sh` scripts test the build process of the project under different distrubutions and enviroments. To run them docker and cibuildwheels is required. diff --git a/tests/build_test_cibuildwheel_linux.sh b/tests/build_test_cibuildwheel_linux.sh new file mode 100644 index 000000000..ffae5e36f --- /dev/null +++ b/tests/build_test_cibuildwheel_linux.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -e + +NC='\033[0m' +RED='\033[0;31m' +GREEN='\033[0;32m' + +DOCKERFILES_DIR=$( dirname ${BASH_SOURCE[0]} )/wheels_test_dockerfiles +GENERATED_DOCKERFILES_DIR=tests/test_dockerfiles +IMAGE_PREFIX="vizdoom_wheels" + +# Generate and run dockerfiles +# Array in format " " +DOCKERFILES_TO_BUILD_AND_RUN=( + "almalinux:9 dnf-based.Dockerfile" # Python 3.9 + "fedora:36 dnf-based.Dockerfile" # Python 3.10 + "fedora:37 dnf-based.Dockerfile" # Python 3.11 + "rockylinux:9 dnf-based.Dockerfile" # Python 3.9 + "debian:11.6 apt-based.Dockerfile ENV LANG C.UTF-8" # Python 3.9 + "ubuntu:20.04 apt-based.Dockerfile" # Python 3.8 + "ubuntu:22.04 apt-based.Dockerfile" # Python 3.10 + "continuumio/miniconda3:latest conda-based.Dockerfile" # Python 3.10 +) + +function create_dockerfile ( ) { + local all_args=("$@") + local base_image=$1 + local base_name=$( basename "$( echo ${base_image} | tr ':' '_' )" ) + local base_dockerfile=$2 + local add_commands=("${all_args[@]:2}") + + mkdir -p $GENERATED_DOCKERFILES_DIR + dockerfile=${GENERATED_DOCKERFILES_DIR}/${IMAGE_PREFIX}_${base_name}.Dockerfile + + echo "FROM $base_image" > $dockerfile + echo "" >> $dockerfile + echo -e "${add_commands[@]}" >> $dockerfile + cat ${DOCKERFILES_DIR}/$base_dockerfile | tail -n +2 >> $dockerfile +} + +for dockerfile_setting in "${DOCKERFILES_TO_BUILD_AND_RUN[@]}"; do + create_dockerfile $dockerfile_setting + + echo -n "Building and running $dockerfile, saving output to $dockerfile.log ... " + filename=$( basename "$dockerfile" ) + dockerfile_dir=$( dirname "$dockerfile" ) + without_ext="${filename%.*}" + tag="${without_ext}:latest" + log="${dockerfile_dir}/${without_ext}.log" + + docker build -t $tag -f $dockerfile . &> $log || ( echo -e "${RED}FAILED${NC}"; exit 1 ) + docker run -it $tag &>> $log || ( echo -e "${RED}FAILED${NC}"; exit 1 ) + + echo -e "${GREEN}OK${NC}" +done diff --git a/tests/dockerfiles/run_all_localy.sh b/tests/build_test_local_linux_builds.sh similarity index 65% rename from tests/dockerfiles/run_all_localy.sh rename to tests/build_test_local_linux_builds.sh index d2edcea85..23e0fe15e 100755 --- a/tests/dockerfiles/run_all_localy.sh +++ b/tests/build_test_local_linux_builds.sh @@ -5,9 +5,31 @@ NC='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' -DOCKERFILES_DIR=$( dirname "${BASH_SOURCE[0]}" ) -GENERATED_DOCKERFILES_DIR=tmp_dockerfiles +DOCKERFILES_DIR=$( dirname ${BASH_SOURCE[0]} )/local_builds_dockerfiles +GENERATED_DOCKERFILES_DIR=tests/test_dockerfiles +IMAGE_PREFIX="vizdoom_local" +# Array in format " " +DOCKERFILES_TO_BUILD_AND_RUN=( + "almalinux:9 dnf-based.Dockerfile RUN dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb" + "tgagor/centos:stream9 dnf-based.Dockerfile RUN dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb" + "fedora:36 dnf-based.Dockerfile" + "fedora:37 dnf-based.Dockerfile" + "rockylinux:9 dnf-based.Dockerfile RUN dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb" + "debian:11.6 apt-based.Dockerfile ENV LANG C.UTF-8" + "ubuntu:18.04 apt-based.Dockerfile" + "ubuntu:20.04 apt-based.Dockerfile" + "ubuntu:22.04 apt-based.Dockerfile" + "ubuntu:latest apt-based.Dockerfile" + #"continuumio/miniconda3:latest conda-based.Dockerfile" # Does not work at the moment +) + + +# Build wheels using cibuildwheel +cibuildwheel --platform linux --arch $(uname -m) + + +# Test wheels inside docker containers function create_dockerfile ( ) { local all_args=("$@") local base_image=$1 @@ -16,7 +38,7 @@ function create_dockerfile ( ) { local add_commands=("${all_args[@]:2}") mkdir -p $GENERATED_DOCKERFILES_DIR - dockerfile=${GENERATED_DOCKERFILES_DIR}/${base_name}.Dockerfile + dockerfile=${GENERATED_DOCKERFILES_DIR}/${IMAGE_PREFIX}_${base_name}.Dockerfile echo "FROM $base_image" > $dockerfile echo "" >> $dockerfile @@ -24,41 +46,18 @@ function create_dockerfile ( ) { cat ${DOCKERFILES_DIR}/$base_dockerfile | tail -n +2 >> $dockerfile } -# Generate and run dockerfiles -# Array in florma " " -DOCKERFILES_TO_BUILD_AND_RUN=( - #"almalinux:8 dnf-based.Dockerfile RUN dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools" - "almalinux:9 dnf-based.Dockerfile RUN dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb" - #"tgagor/centos:stream8 dnf-based.Dockerfile RUN dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools" - "tgagor/centos:stream9 dnf-based.Dockerfile RUN dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb" - #"fedora:34 dnf-based.Dockerfile" # EOL - #"fedora:35 dnf-based.Dockerfile" # EOL - "fedora:36 dnf-based.Dockerfile" - "fedora:37 dnf-based.Dockerfile" - #"rockylinux:8 dnf-based.Dockerfile RUN dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled devel" - "rockylinux:9 dnf-based.Dockerfile RUN dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb" - #"debian:10.13 apt-based.Dockerfile ENV LANG C.UTF-8" # EOL - "debian:11.6 apt-based.Dockerfile ENV LANG C.UTF-8" - #"ubuntu:18.04 apt-based.Dockerfile" # EOL - "ubuntu:20.04 apt-based.Dockerfile" - "ubuntu:22.04 apt-based.Dockerfile" - #"continuumio/miniconda3:latest conda-based.Dockerfile" # Does not work at the moment -) - for dockerfile_setting in "${DOCKERFILES_TO_BUILD_AND_RUN[@]}"; do create_dockerfile $dockerfile_setting echo -n "Building and running $dockerfile, saving output to $dockerfile.log ... " filename=$( basename "$dockerfile" ) + dockerfile_dir=$( dirname "$dockerfile" ) without_ext="${filename%.*}" - tag="vizdoom_${without_ext}:latest" - log="vizdoom_${without_ext}.log" + tag="${without_ext}:latest" + log="${dockerfile_dir}/${without_ext}.log" docker build -t $tag -f $dockerfile . &> $log || ( echo -e "${RED}FAILED${NC}"; exit 1 ) docker run -it $tag &>> $log || ( echo -e "${RED}FAILED${NC}"; exit 1 ) echo -e "${GREEN}OK${NC}" - done - -rm -rf $GENERATED_DOCKERFILES_DIR diff --git a/tests/dockerfiles/apt-based.Dockerfile b/tests/local_builds_dockerfiles/apt-based.Dockerfile similarity index 100% rename from tests/dockerfiles/apt-based.Dockerfile rename to tests/local_builds_dockerfiles/apt-based.Dockerfile diff --git a/tests/dockerfiles/conda-based.Dockerfile b/tests/local_builds_dockerfiles/conda-based.Dockerfile similarity index 100% rename from tests/dockerfiles/conda-based.Dockerfile rename to tests/local_builds_dockerfiles/conda-based.Dockerfile diff --git a/tests/dockerfiles/dnf-based.Dockerfile b/tests/local_builds_dockerfiles/dnf-based.Dockerfile similarity index 100% rename from tests/dockerfiles/dnf-based.Dockerfile rename to tests/local_builds_dockerfiles/dnf-based.Dockerfile diff --git a/tests/wheels_test_dockerfiles/apt-based.Dockerfile b/tests/wheels_test_dockerfiles/apt-based.Dockerfile new file mode 100644 index 000000000..e5f2063df --- /dev/null +++ b/tests/wheels_test_dockerfiles/apt-based.Dockerfile @@ -0,0 +1,12 @@ +FROM ubuntu:latest + +ARG DEBIAN_FRONTEND=noninteractive +ENV TZ=Europe/Warsaw + +WORKDIR vizdoom + +# MINIMAL +RUN apt update && apt install -y python3-dev python3-pip + +COPY . ./ +CMD ["bash", "./scripts/install_and_test_wheel.sh"] diff --git a/tests/wheels_test_dockerfiles/conda-based.Dockerfile b/tests/wheels_test_dockerfiles/conda-based.Dockerfile new file mode 100644 index 000000000..9b4cdff3c --- /dev/null +++ b/tests/wheels_test_dockerfiles/conda-based.Dockerfile @@ -0,0 +1,6 @@ +FROM continuumio/miniconda3:latest + +WORKDIR vizdoom + +COPY . ./ +CMD ["bash", "./scripts/install_and_test_wheel.sh"] diff --git a/tests/wheels_test_dockerfiles/dnf-based.Dockerfile b/tests/wheels_test_dockerfiles/dnf-based.Dockerfile new file mode 100644 index 000000000..d35640ff7 --- /dev/null +++ b/tests/wheels_test_dockerfiles/dnf-based.Dockerfile @@ -0,0 +1,9 @@ +FROM fedora:latest + +WORKDIR vizdoom + +# MINIMAL +RUN dnf update -y && dnf clean all && dnf install -y python3-devel python3-pip + +COPY . ./ +CMD ["bash", "./scripts/install_and_test_wheel.sh"] From 4dffb9da7adc66e192572c58603ed57e6a01ada5 Mon Sep 17 00:00:00 2001 From: Marek Wydmuch Date: Thu, 7 Sep 2023 02:29:04 +0200 Subject: [PATCH 09/12] Fix pre-commit failed checks --- scripts/install_and_test_wheel.sh | 2 +- tests/README.md | 2 +- tests/build_test_cibuildwheel_linux.sh | 0 3 files changed, 2 insertions(+), 2 deletions(-) mode change 100644 => 100755 scripts/install_and_test_wheel.sh mode change 100644 => 100755 tests/build_test_cibuildwheel_linux.sh diff --git a/scripts/install_and_test_wheel.sh b/scripts/install_and_test_wheel.sh old mode 100644 new mode 100755 index ff64ba6d4..70032079f --- a/scripts/install_and_test_wheel.sh +++ b/scripts/install_and_test_wheel.sh @@ -15,7 +15,7 @@ python3 -c "import sys; print('Python', sys.version)" PYTHON_VERSION=$(python3 -c "import sys; print('{}{}'.format(sys.version_info.major, sys.version_info.minor))") PYTHON_WHEEL=$(ls wheelhouse/vizdoom-*-cp${PYTHON_VERSION}-cp${PYTHON_VERSION}*.whl) -# Updgrade pip and install test deps +# Updgrad pip and install test deps python3 -m pip install --upgrade pip python3 -m pip install pytest psutil diff --git a/tests/README.md b/tests/README.md index f4fc33b42..9e4156bad 100644 --- a/tests/README.md +++ b/tests/README.md @@ -2,4 +2,4 @@ This directory contains the tests for the project that can be run with pytest or by running the `test_*.py` or `manual_test_*.py` files directly. Manual tests require significant amount of time, so they are not run by default by CI/CD. -The `build_test_*.sh` scripts test the build process of the project under different distrubutions and enviroments. To run them docker and cibuildwheels is required. +The `build_test_*.sh` scripts test the build process of the project under different distributions and environments. To run them docker and cibuildwheels is required. diff --git a/tests/build_test_cibuildwheel_linux.sh b/tests/build_test_cibuildwheel_linux.sh old mode 100644 new mode 100755 From ea40cc3cd31eee5ea2cd3b7d9261fec834ede574 Mon Sep 17 00:00:00 2001 From: Marek Wydmuch Date: Thu, 7 Sep 2023 17:42:08 +0200 Subject: [PATCH 10/12] Update cibuildwheel tests --- scripts/build_and_test.sh | 4 --- scripts/build_and_test_conda.sh | 28 +++++++++++++++++++ tests/build_test_cibuildwheel_linux.sh | 5 +++- tests/build_test_local_linux_builds.sh | 14 ++++------ .../apt+conda-based.Dockerfile | 20 +++++++++++++ .../apt-based.Dockerfile | 4 +-- .../conda-based.Dockerfile | 5 ++-- .../dnf-based.Dockerfile | 4 +-- .../apt-based.Dockerfile | 2 +- .../dnf-based.Dockerfile | 2 +- 10 files changed, 63 insertions(+), 25 deletions(-) create mode 100755 scripts/build_and_test_conda.sh create mode 100644 tests/local_builds_dockerfiles/apt+conda-based.Dockerfile diff --git a/scripts/build_and_test.sh b/scripts/build_and_test.sh index e79817d16..887b26b42 100755 --- a/scripts/build_and_test.sh +++ b/scripts/build_and_test.sh @@ -25,7 +25,3 @@ python3 -c "import vizdoom" # Run tests pytest tests -# CMake manual build -# rm CMakeCache.txt -# cmake -DCMAKE_PREFIX_PATH=$CONDA_PREFIX -DCMAKE_BUILD_TYPE=Release -DBUILD_PYTHON=ON . -# make -j diff --git a/scripts/build_and_test_conda.sh b/scripts/build_and_test_conda.sh new file mode 100755 index 000000000..743add8ca --- /dev/null +++ b/scripts/build_and_test_conda.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -e + +# Set working dir to the root of the repo +cd $( dirname "${BASH_SOURCE[0]}" )/.. + +# Report directory +ls -lha . + +# Report cmake version +cmake --version + +# Report gcc version +gcc --version + +# Report python version +python --version +python -c "import sys; print('Python', sys.version)" + +# Install +export VIZDOOM_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=/root/miniconda/" +python -m pip install .[test] + +# Test import +python -c "import vizdoom" + +# Run tests +pytest tests diff --git a/tests/build_test_cibuildwheel_linux.sh b/tests/build_test_cibuildwheel_linux.sh index ffae5e36f..2d40f49fd 100755 --- a/tests/build_test_cibuildwheel_linux.sh +++ b/tests/build_test_cibuildwheel_linux.sh @@ -9,7 +9,7 @@ DOCKERFILES_DIR=$( dirname ${BASH_SOURCE[0]} )/wheels_test_dockerfiles GENERATED_DOCKERFILES_DIR=tests/test_dockerfiles IMAGE_PREFIX="vizdoom_wheels" -# Generate and run dockerfiles + # Array in format " " DOCKERFILES_TO_BUILD_AND_RUN=( "almalinux:9 dnf-based.Dockerfile" # Python 3.9 @@ -22,6 +22,9 @@ DOCKERFILES_TO_BUILD_AND_RUN=( "continuumio/miniconda3:latest conda-based.Dockerfile" # Python 3.10 ) +# Build wheels using cibuildwheel +cibuildwheel --platform linux --arch $(uname -m) + function create_dockerfile ( ) { local all_args=("$@") local base_image=$1 diff --git a/tests/build_test_local_linux_builds.sh b/tests/build_test_local_linux_builds.sh index 23e0fe15e..45bea9bc5 100755 --- a/tests/build_test_local_linux_builds.sh +++ b/tests/build_test_local_linux_builds.sh @@ -9,6 +9,7 @@ DOCKERFILES_DIR=$( dirname ${BASH_SOURCE[0]} )/local_builds_dockerfiles GENERATED_DOCKERFILES_DIR=tests/test_dockerfiles IMAGE_PREFIX="vizdoom_local" +# Generate and run dockerfiles # Array in format " " DOCKERFILES_TO_BUILD_AND_RUN=( "almalinux:9 dnf-based.Dockerfile RUN dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb" @@ -20,15 +21,10 @@ DOCKERFILES_TO_BUILD_AND_RUN=( "ubuntu:18.04 apt-based.Dockerfile" "ubuntu:20.04 apt-based.Dockerfile" "ubuntu:22.04 apt-based.Dockerfile" - "ubuntu:latest apt-based.Dockerfile" - #"continuumio/miniconda3:latest conda-based.Dockerfile" # Does not work at the moment + "ubuntu:20.04 apt+conda-based.Dockerfile" # Ubuntu build with dependencies installed via conda + #"continuumio/miniconda3:latest conda-based.Dockerfile" # Does not work at the moment ) - -# Build wheels using cibuildwheel -cibuildwheel --platform linux --arch $(uname -m) - - # Test wheels inside docker containers function create_dockerfile ( ) { local all_args=("$@") @@ -56,8 +52,8 @@ for dockerfile_setting in "${DOCKERFILES_TO_BUILD_AND_RUN[@]}"; do tag="${without_ext}:latest" log="${dockerfile_dir}/${without_ext}.log" - docker build -t $tag -f $dockerfile . &> $log || ( echo -e "${RED}FAILED${NC}"; exit 1 ) - docker run -it $tag &>> $log || ( echo -e "${RED}FAILED${NC}"; exit 1 ) + docker build -t $tag -f $dockerfile . #&> $log || ( echo -e "${RED}FAILED${NC}"; exit 1 ) + docker run -it $tag #&>> $log || ( echo -e "${RED}FAILED${NC}"; exit 1 ) echo -e "${GREEN}OK${NC}" done diff --git a/tests/local_builds_dockerfiles/apt+conda-based.Dockerfile b/tests/local_builds_dockerfiles/apt+conda-based.Dockerfile new file mode 100644 index 000000000..25a637f89 --- /dev/null +++ b/tests/local_builds_dockerfiles/apt+conda-based.Dockerfile @@ -0,0 +1,20 @@ +FROM ubuntu:latest + +ARG DEBIAN_FRONTEND=noninteractive +ENV TZ=Europe/Warsaw + +WORKDIR vizdoom + +# Instal wget +RUN apt-get update && apt-get install -y build-essential git make cmake wget + +# Install miniconda +RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh +RUN bash ~/miniconda.sh -b -p $HOME/miniconda +ENV PATH="/root/miniconda/bin:${PATH}" + +# Install conda dependencies +RUN conda install -y -c conda-forge boost sdl2 openal-soft + +COPY . ./ +CMD ["bash", "./scripts/build_and_test_conda.sh"] diff --git a/tests/local_builds_dockerfiles/apt-based.Dockerfile b/tests/local_builds_dockerfiles/apt-based.Dockerfile index 9f63bc0b5..cc7613016 100644 --- a/tests/local_builds_dockerfiles/apt-based.Dockerfile +++ b/tests/local_builds_dockerfiles/apt-based.Dockerfile @@ -5,10 +5,8 @@ ENV TZ=Europe/Warsaw WORKDIR vizdoom -# MINIMAL +# Install minimal dependencies RUN apt update && apt install -y build-essential cmake git libboost-all-dev libsdl2-dev libopenal-dev python3-dev python3-pip - -# FULL #RUN apt update && apt install -y build-essential cmake libboost-all-dev libsdl2-dev libfreetype-dev libopenal-dev python3-dev python3-pip COPY . ./ diff --git a/tests/local_builds_dockerfiles/conda-based.Dockerfile b/tests/local_builds_dockerfiles/conda-based.Dockerfile index 128721013..39e7eff44 100644 --- a/tests/local_builds_dockerfiles/conda-based.Dockerfile +++ b/tests/local_builds_dockerfiles/conda-based.Dockerfile @@ -2,8 +2,7 @@ FROM continuumio/miniconda3:latest WORKDIR vizdoom -# MINIMAL -RUN conda install -c conda-forge c-compiler cxx-compiler make cmake boost sdl2 openal-soft +RUN conda install -y -c conda-forge gcc gxx rhash make cmake boost sdl2 openal-soft COPY . ./ -CMD ["bash", "./scripts/build_and_test.sh"] +CMD ["bash", "./scripts/build_and_test_conda.sh"] diff --git a/tests/local_builds_dockerfiles/dnf-based.Dockerfile b/tests/local_builds_dockerfiles/dnf-based.Dockerfile index 96501b243..935c67045 100644 --- a/tests/local_builds_dockerfiles/dnf-based.Dockerfile +++ b/tests/local_builds_dockerfiles/dnf-based.Dockerfile @@ -2,10 +2,8 @@ FROM fedora:latest WORKDIR vizdoom -# MINIMAL +# Install minimal dependencies RUN dnf update -y && dnf clean all && dnf install -y gcc gcc-c++ make cmake git boost-devel SDL2-devel openal-soft-devel python3-devel python3-pip - -# FULL #RUN dnf update -y && dnf clean all && dnf install -y gcc gcc-c++ make cmake boost-devel SDL2-devel freetype-devel openal-soft-devel python3-devel python3-pip COPY . ./ diff --git a/tests/wheels_test_dockerfiles/apt-based.Dockerfile b/tests/wheels_test_dockerfiles/apt-based.Dockerfile index e5f2063df..7c1df9e30 100644 --- a/tests/wheels_test_dockerfiles/apt-based.Dockerfile +++ b/tests/wheels_test_dockerfiles/apt-based.Dockerfile @@ -5,7 +5,7 @@ ENV TZ=Europe/Warsaw WORKDIR vizdoom -# MINIMAL +# Install Python and pip RUN apt update && apt install -y python3-dev python3-pip COPY . ./ diff --git a/tests/wheels_test_dockerfiles/dnf-based.Dockerfile b/tests/wheels_test_dockerfiles/dnf-based.Dockerfile index d35640ff7..7c9c9a0bc 100644 --- a/tests/wheels_test_dockerfiles/dnf-based.Dockerfile +++ b/tests/wheels_test_dockerfiles/dnf-based.Dockerfile @@ -2,7 +2,7 @@ FROM fedora:latest WORKDIR vizdoom -# MINIMAL +# Install Python and pip RUN dnf update -y && dnf clean all && dnf install -y python3-devel python3-pip COPY . ./ From 2b3dd19533cfa8597fe48fd55da6f8fdec0e79b0 Mon Sep 17 00:00:00 2001 From: Marek Wydmuch Date: Thu, 7 Sep 2023 17:46:52 +0200 Subject: [PATCH 11/12] Fix pre-commit failed checks once again --- scripts/build_and_test.sh | 1 - tests/local_builds_dockerfiles/apt+conda-based.Dockerfile | 4 ++-- tests/local_builds_dockerfiles/conda-based.Dockerfile | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/build_and_test.sh b/scripts/build_and_test.sh index 887b26b42..c00b7e01d 100755 --- a/scripts/build_and_test.sh +++ b/scripts/build_and_test.sh @@ -24,4 +24,3 @@ python3 -c "import vizdoom" # Run tests pytest tests - diff --git a/tests/local_builds_dockerfiles/apt+conda-based.Dockerfile b/tests/local_builds_dockerfiles/apt+conda-based.Dockerfile index 25a637f89..c34c019e4 100644 --- a/tests/local_builds_dockerfiles/apt+conda-based.Dockerfile +++ b/tests/local_builds_dockerfiles/apt+conda-based.Dockerfile @@ -5,7 +5,7 @@ ENV TZ=Europe/Warsaw WORKDIR vizdoom -# Instal wget +# Install wget RUN apt-get update && apt-get install -y build-essential git make cmake wget # Install miniconda @@ -14,7 +14,7 @@ RUN bash ~/miniconda.sh -b -p $HOME/miniconda ENV PATH="/root/miniconda/bin:${PATH}" # Install conda dependencies -RUN conda install -y -c conda-forge boost sdl2 openal-soft +RUN conda install -y -c conda-forge boost sdl2 openal-soft COPY . ./ CMD ["bash", "./scripts/build_and_test_conda.sh"] diff --git a/tests/local_builds_dockerfiles/conda-based.Dockerfile b/tests/local_builds_dockerfiles/conda-based.Dockerfile index 39e7eff44..a37a9fc43 100644 --- a/tests/local_builds_dockerfiles/conda-based.Dockerfile +++ b/tests/local_builds_dockerfiles/conda-based.Dockerfile @@ -2,7 +2,7 @@ FROM continuumio/miniconda3:latest WORKDIR vizdoom -RUN conda install -y -c conda-forge gcc gxx rhash make cmake boost sdl2 openal-soft +RUN conda install -y -c conda-forge gcc gxx rhash make cmake boost sdl2 openal-soft COPY . ./ CMD ["bash", "./scripts/build_and_test_conda.sh"] From 11b129a261dc273638f58c01ed0cb70a99600a7c Mon Sep 17 00:00:00 2001 From: Marek Wydmuch Date: Thu, 7 Sep 2023 17:53:11 +0200 Subject: [PATCH 12/12] Update one of the tests --- tests/build_test_local_linux_builds.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/build_test_local_linux_builds.sh b/tests/build_test_local_linux_builds.sh index 45bea9bc5..235e6e17c 100755 --- a/tests/build_test_local_linux_builds.sh +++ b/tests/build_test_local_linux_builds.sh @@ -52,8 +52,8 @@ for dockerfile_setting in "${DOCKERFILES_TO_BUILD_AND_RUN[@]}"; do tag="${without_ext}:latest" log="${dockerfile_dir}/${without_ext}.log" - docker build -t $tag -f $dockerfile . #&> $log || ( echo -e "${RED}FAILED${NC}"; exit 1 ) - docker run -it $tag #&>> $log || ( echo -e "${RED}FAILED${NC}"; exit 1 ) + docker build -t $tag -f $dockerfile . &> $log || ( echo -e "${RED}FAILED${NC}"; exit 1 ) + docker run -it $tag &>> $log || ( echo -e "${RED}FAILED${NC}"; exit 1 ) echo -e "${GREEN}OK${NC}" done