From 84848761fafc154f9d83d0f9cbf2c46728a4a413 Mon Sep 17 00:00:00 2001 From: regro-cf-autotick-bot <36490558+regro-cf-autotick-bot@users.noreply.github.com> Date: Thu, 16 Nov 2023 14:25:06 +0000 Subject: [PATCH 1/2] Rebuild for CUDA 12 w/arch + Windows support The transition to CUDA 12 SDK includes new packages for all CUDA libraries and build tools. Notably, the cudatoolkit package no longer exists, and packages should depend directly on the specific CUDA libraries (libcublas, libcusolver, etc) as needed. For an in-depth overview of the changes and to report problems [see this issue]( https://github.com/conda-forge/conda-forge.github.io/issues/1963 ). Please feel free to raise any issues encountered there. Thank you! :pray: --- .ci_support/migrations/cuda120.yaml | 22 +++++++++++----------- recipe/meta.yaml | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.ci_support/migrations/cuda120.yaml b/.ci_support/migrations/cuda120.yaml index 49da33d..abfb9cb 100644 --- a/.ci_support/migrations/cuda120.yaml +++ b/.ci_support/migrations/cuda120.yaml @@ -3,7 +3,7 @@ __migrator: kind: version migration_number: - 2 + 3 build_number: 1 paused: false @@ -47,13 +47,13 @@ __migrator: - quay.io/condaforge/linux-anvil-cos7-x86_64 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] cuda_compiler_version: - None - - 10.2 # [(linux64 or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - - 11.0 # [(linux64 or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - - 11.1 # [(linux64 or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - - 11.2 # [(linux or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - - 12.0 # [(linux or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 10.2 # [(linux64 or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11.0 # [(linux64 or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11.1 # [(linux64 or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11.2 # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 12.0 # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] commit_message: | - Rebuild for CUDA 12 w/arch support + Rebuild for CUDA 12 w/arch + Windows support The transition to CUDA 12 SDK includes new packages for all CUDA libraries and build tools. Notably, the cudatoolkit package no longer exists, and packages @@ -62,11 +62,11 @@ __migrator: [see this issue]( https://github.com/conda-forge/conda-forge.github.io/issues/1963 ). Please feel free to raise any issues encountered there. Thank you! :pray: -cuda_compiler: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - - cuda-nvcc # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] +cuda_compiler: # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - cuda-nvcc # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] -cuda_compiler_version: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - - 12.0 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] +cuda_compiler_version: # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 12.0 # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] c_compiler_version: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - 12 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 8361533..95b1686 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -23,7 +23,7 @@ source: sha256: e362e1fbf6b101ad71aa5314f21d00a4353b90efbe5cdedc5fcfb0f3b68cd791 # [win and cuda_major == "12"] build: - number: 3 + number: 4 # TODO: Enable Windows with CUDA 12 when it is supported in CI skip: True # [win32 or osx or (win and cuda_major == "12")] # Disable binary relocation to workaround patchelf issue From d2a9391e2fa7d2b4b1f1c8c3754383ad1aceb170 Mon Sep 17 00:00:00 2001 From: regro-cf-autotick-bot <36490558+regro-cf-autotick-bot@users.noreply.github.com> Date: Thu, 16 Nov 2023 14:25:27 +0000 Subject: [PATCH 2/2] MNT: Re-rendered with conda-build 3.27.0, conda-smithy 3.29.0, and conda-forge-pinning 2023.11.16.09.46.51 --- .azure-pipelines/azure-pipelines-linux.yml | 3 + .azure-pipelines/azure-pipelines-win.yml | 67 +++------- ...lercuda-nvcccuda_compiler_version12.0.yaml | 17 +++ ...ompilernvcccuda_compiler_version11.2.yaml} | 0 ...ompilernvcccuda_compiler_version11.8.yaml} | 0 .scripts/build_steps.sh | 22 +++- .scripts/logging_utils.sh | 4 +- .scripts/run_docker_build.sh | 3 + .scripts/run_win_build.bat | 120 ++++++++++++++++++ .travis.yml | 3 + README.md | 17 ++- 11 files changed, 193 insertions(+), 63 deletions(-) create mode 100644 .ci_support/win_64_cuda_compilercuda-nvcccuda_compiler_version12.0.yaml rename .ci_support/{win_64_cuda_compiler_version11.2.yaml => win_64_cuda_compilernvcccuda_compiler_version11.2.yaml} (100%) rename .ci_support/{win_64_cuda_compiler_version11.8.yaml => win_64_cuda_compilernvcccuda_compiler_version11.8.yaml} (100%) create mode 100755 .scripts/run_win_build.bat diff --git a/.azure-pipelines/azure-pipelines-linux.yml b/.azure-pipelines/azure-pipelines-linux.yml index 5e0efa9..005b39f 100755 --- a/.azure-pipelines/azure-pipelines-linux.yml +++ b/.azure-pipelines/azure-pipelines-linux.yml @@ -45,6 +45,9 @@ jobs: - script: | export CI=azure + export flow_run_id=azure_$(Build.BuildNumber).$(System.JobAttempt) + export remote_url=$(Build.Repository.Uri) + export sha=$(Build.SourceVersion) export GIT_BRANCH=$BUILD_SOURCEBRANCHNAME export FEEDSTOCK_NAME=$(basename ${BUILD_REPOSITORY_NAME}) if [[ "${BUILD_REASON:-}" == "PullRequest" ]]; then diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml index 742460e..d9780d0 100755 --- a/.azure-pipelines/azure-pipelines-win.yml +++ b/.azure-pipelines/azure-pipelines-win.yml @@ -8,11 +8,14 @@ jobs: vmImage: windows-2022 strategy: matrix: - win_64_cuda_compiler_version11.2: - CONFIG: win_64_cuda_compiler_version11.2 + win_64_cuda_compilercuda-nvcccuda_compiler_version12.0: + CONFIG: win_64_cuda_compilercuda-nvcccuda_compiler_version12.0 UPLOAD_PACKAGES: 'True' - win_64_cuda_compiler_version11.8: - CONFIG: win_64_cuda_compiler_version11.8 + win_64_cuda_compilernvcccuda_compiler_version11.2: + CONFIG: win_64_cuda_compilernvcccuda_compiler_version11.2 + UPLOAD_PACKAGES: 'True' + win_64_cuda_compilernvcccuda_compiler_version11.8: + CONFIG: win_64_cuda_compilernvcccuda_compiler_version11.8 UPLOAD_PACKAGES: 'True' timeoutInMinutes: 360 variables: @@ -20,6 +23,7 @@ jobs: UPLOAD_TEMP: D:\\tmp steps: + - task: PythonScript@0 displayName: 'Download Miniforge' inputs: @@ -38,52 +42,17 @@ jobs: displayName: Add conda to PATH - script: | - call activate base - mamba.exe install "python=3.10" conda-build conda pip boa conda-forge-ci-setup=3 -c conda-forge --strict-channel-priority --yes - displayName: Install conda-build - - - script: set PYTHONUNBUFFERED=1 - displayName: Set PYTHONUNBUFFERED - - # Configure the VM - - script: | - call activate base - setup_conda_rc .\ ".\recipe" .\.ci_support\%CONFIG%.yaml - displayName: conda-forge CI setup - - # Configure the VM. - - script: | - set "CI=azure" - call activate base - run_conda_forge_build_setup - displayName: conda-forge build setup - - - script: | - call activate base - if EXIST LICENSE.txt ( - copy LICENSE.txt "recipe\\recipe-scripts-license.txt" - ) - conda.exe mambabuild "recipe" -m .ci_support\%CONFIG%.yaml --suppress-variables %EXTRA_CB_OPTIONS% - displayName: Build recipe + call ".scripts\run_win_build.bat" + displayName: Run Windows build env: PYTHONUNBUFFERED: 1 - - script: | - set "FEEDSTOCK_NAME=%BUILD_REPOSITORY_NAME:*/=%" - call activate base - validate_recipe_outputs "%FEEDSTOCK_NAME%" - displayName: Validate Recipe Outputs - - - script: | - set "GIT_BRANCH=%BUILD_SOURCEBRANCHNAME%" - set "FEEDSTOCK_NAME=%BUILD_REPOSITORY_NAME:*/=%" - set "TEMP=$(UPLOAD_TEMP)" - if not exist "%TEMP%\" md "%TEMP%" - set "TMP=%TEMP%" - call activate base - upload_package --validate --feedstock-name="%FEEDSTOCK_NAME%" .\ ".\recipe" .ci_support\%CONFIG%.yaml - displayName: Upload package - env: + CONFIG: $(CONFIG) + CI: azure + flow_run_id: azure_$(Build.BuildNumber).$(System.JobAttempt) + remote_url: $(Build.Repository.Uri) + sha: $(Build.SourceVersion) + UPLOAD_PACKAGES: $(UPLOAD_PACKAGES) + UPLOAD_TEMP: $(UPLOAD_TEMP) BINSTAR_TOKEN: $(BINSTAR_TOKEN) FEEDSTOCK_TOKEN: $(FEEDSTOCK_TOKEN) - STAGING_BINSTAR_TOKEN: $(STAGING_BINSTAR_TOKEN) - condition: and(succeeded(), not(eq(variables['UPLOAD_PACKAGES'], 'False')), not(eq(variables['Build.Reason'], 'PullRequest'))) \ No newline at end of file + STAGING_BINSTAR_TOKEN: $(STAGING_BINSTAR_TOKEN) \ No newline at end of file diff --git a/.ci_support/win_64_cuda_compilercuda-nvcccuda_compiler_version12.0.yaml b/.ci_support/win_64_cuda_compilercuda-nvcccuda_compiler_version12.0.yaml new file mode 100644 index 0000000..67e5a06 --- /dev/null +++ b/.ci_support/win_64_cuda_compilercuda-nvcccuda_compiler_version12.0.yaml @@ -0,0 +1,17 @@ +c_compiler: +- vs2019 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler: +- cuda-nvcc +cuda_compiler_version: +- '12.0' +cxx_compiler: +- vs2019 +target_platform: +- win-64 +zip_keys: +- - cuda_compiler + - cuda_compiler_version diff --git a/.ci_support/win_64_cuda_compiler_version11.2.yaml b/.ci_support/win_64_cuda_compilernvcccuda_compiler_version11.2.yaml similarity index 100% rename from .ci_support/win_64_cuda_compiler_version11.2.yaml rename to .ci_support/win_64_cuda_compilernvcccuda_compiler_version11.2.yaml diff --git a/.ci_support/win_64_cuda_compiler_version11.8.yaml b/.ci_support/win_64_cuda_compilernvcccuda_compiler_version11.8.yaml similarity index 100% rename from .ci_support/win_64_cuda_compiler_version11.8.yaml rename to .ci_support/win_64_cuda_compilernvcccuda_compiler_version11.8.yaml diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index 595f8b5..ab7bc8f 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -28,14 +28,15 @@ conda-build: pkgs_dirs: - ${FEEDSTOCK_ROOT}/build_artifacts/pkg_cache - /opt/conda/pkgs +solver: libmamba CONDARC +export CONDA_LIBMAMBA_SOLVER_NO_CHANNELS_FROM_INSTALLED=1 - -mamba install --update-specs --yes --quiet --channel conda-forge \ - conda-build pip boa conda-forge-ci-setup=3 -mamba update --update-specs --yes --quiet --channel conda-forge \ - conda-build pip boa conda-forge-ci-setup=3 +mamba install --update-specs --yes --quiet --channel conda-forge --strict-channel-priority \ + pip mamba conda-build conda-forge-ci-setup=4 +mamba update --update-specs --yes --quiet --channel conda-forge --strict-channel-priority \ + pip mamba conda-build conda-forge-ci-setup=4 # set up the condarc setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" @@ -53,6 +54,12 @@ if [[ -f "${FEEDSTOCK_ROOT}/LICENSE.txt" ]]; then cp "${FEEDSTOCK_ROOT}/LICENSE.txt" "${RECIPE_ROOT}/recipe-scripts-license.txt" fi +if [[ "${sha:-}" == "" ]]; then + pushd ${FEEDSTOCK_ROOT} + sha=$(git rev-parse HEAD) + popd +fi + if [[ "${BUILD_WITH_CONDA_DEBUG:-0}" == 1 ]]; then if [[ "x${BUILD_OUTPUT_ID:-}" != "x" ]]; then EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --output-id ${BUILD_OUTPUT_ID}" @@ -64,9 +71,10 @@ if [[ "${BUILD_WITH_CONDA_DEBUG:-0}" == 1 ]]; then # Drop into an interactive shell /bin/bash else - conda mambabuild "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ + conda build "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ --suppress-variables ${EXTRA_CB_OPTIONS:-} \ - --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" + --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" \ + --extra-meta flow_run_id="${flow_run_id:-}" remote_url="${remote_url:-}" sha="${sha:-}" ( startgroup "Validating outputs" ) 2> /dev/null validate_recipe_outputs "${FEEDSTOCK_NAME}" diff --git a/.scripts/logging_utils.sh b/.scripts/logging_utils.sh index 57bc95c..aff009f 100644 --- a/.scripts/logging_utils.sh +++ b/.scripts/logging_utils.sh @@ -12,7 +12,7 @@ function startgroup { echo "##[group]$1";; travis ) echo "$1" - echo -en 'travis_fold:start:'"${1// /}"'\\r';; + echo -en 'travis_fold:start:'"${1// /}"'\r';; github_actions ) echo "::group::$1";; * ) @@ -28,7 +28,7 @@ function endgroup { azure ) echo "##[endgroup]";; travis ) - echo -en 'travis_fold:end:'"${1// /}"'\\r';; + echo -en 'travis_fold:end:'"${1// /}"'\r';; github_actions ) echo "::endgroup::";; esac diff --git a/.scripts/run_docker_build.sh b/.scripts/run_docker_build.sh index 9236239..b70ef01 100755 --- a/.scripts/run_docker_build.sh +++ b/.scripts/run_docker_build.sh @@ -91,6 +91,9 @@ docker run ${DOCKER_RUN_ARGS} \ -e CPU_COUNT \ -e BUILD_WITH_CONDA_DEBUG \ -e BUILD_OUTPUT_ID \ + -e flow_run_id \ + -e remote_url \ + -e sha \ -e BINSTAR_TOKEN \ -e FEEDSTOCK_TOKEN \ -e STAGING_BINSTAR_TOKEN \ diff --git a/.scripts/run_win_build.bat b/.scripts/run_win_build.bat new file mode 100755 index 0000000..5f648db --- /dev/null +++ b/.scripts/run_win_build.bat @@ -0,0 +1,120 @@ +:: PLEASE NOTE: This script has been automatically generated by conda-smithy. Any changes here +:: will be lost next time ``conda smithy rerender`` is run. If you would like to make permanent +:: changes to this script, consider a proposal to conda-smithy so that other feedstocks can also +:: benefit from the improvement. + +:: Note: we assume a Miniforge installation is available + +:: INPUTS (required environment variables) +:: CONFIG: name of the .ci_support/*.yaml file for this job +:: CI: azure, github_actions, or unset +:: UPLOAD_PACKAGES: true or false +:: UPLOAD_ON_BRANCH: true or false + +setlocal enableextensions enabledelayedexpansion + +call :start_group "Configuring conda" + +:: Activate the base conda environment +call activate base +:: Configure the solver +set "CONDA_SOLVER=libmamba" +if !errorlevel! neq 0 exit /b !errorlevel! +set "CONDA_LIBMAMBA_SOLVER_NO_CHANNELS_FROM_INSTALLED=1" + +:: Provision the necessary dependencies to build the recipe later +echo Installing dependencies +mamba.exe install "python=3.10" pip mamba conda-build conda-forge-ci-setup=4 -c conda-forge --strict-channel-priority --yes +if !errorlevel! neq 0 exit /b !errorlevel! + +:: Set basic configuration +echo Setting up configuration +setup_conda_rc .\ ".\recipe" .\.ci_support\%CONFIG%.yaml +if !errorlevel! neq 0 exit /b !errorlevel! +echo Running build setup +CALL run_conda_forge_build_setup + + +if !errorlevel! neq 0 exit /b !errorlevel! + +if EXIST LICENSE.txt ( + echo Copying feedstock license + copy LICENSE.txt "recipe\\recipe-scripts-license.txt" +) + +if NOT [%flow_run_id%] == [] ( + set "EXTRA_CB_OPTIONS=%EXTRA_CB_OPTIONS% --extra-meta flow_run_id=%flow_run_id% remote_url=%remote_url% sha=%sha%" +) + +call :end_group + +:: Build the recipe +echo Building recipe +conda.exe build "recipe" -m .ci_support\%CONFIG%.yaml --suppress-variables %EXTRA_CB_OPTIONS% +if !errorlevel! neq 0 exit /b !errorlevel! + +:: Prepare some environment variables for the upload step +if /i "%CI%" == "github_actions" ( + set "FEEDSTOCK_NAME=%GITHUB_REPOSITORY:*/=%" + set "GIT_BRANCH=%GITHUB_REF:refs/heads/=%" + if /i "%GITHUB_EVENT_NAME%" == "pull_request" ( + set "IS_PR_BUILD=True" + ) else ( + set "IS_PR_BUILD=False" + ) + set "TEMP=%RUNNER_TEMP%" +) +if /i "%CI%" == "azure" ( + set "FEEDSTOCK_NAME=%BUILD_REPOSITORY_NAME:*/=%" + set "GIT_BRANCH=%BUILD_SOURCEBRANCHNAME%" + if /i "%BUILD_REASON%" == "PullRequest" ( + set "IS_PR_BUILD=True" + ) else ( + set "IS_PR_BUILD=False" + ) + set "TEMP=%UPLOAD_TEMP%" +) + +:: Validate +call :start_group "Validating outputs" +validate_recipe_outputs "%FEEDSTOCK_NAME%" +if !errorlevel! neq 0 exit /b !errorlevel! +call :end_group + +if /i "%UPLOAD_PACKAGES%" == "true" ( + if /i "%IS_PR_BUILD%" == "false" ( + call :start_group "Uploading packages" + if not exist "%TEMP%\" md "%TEMP%" + set "TMP=%TEMP%" + upload_package --validate --feedstock-name="%FEEDSTOCK_NAME%" .\ ".\recipe" .ci_support\%CONFIG%.yaml + if !errorlevel! neq 0 exit /b !errorlevel! + call :end_group + ) +) + +exit + +:: Logging subroutines + +:start_group +if /i "%CI%" == "github_actions" ( + echo ::group::%~1 + exit /b +) +if /i "%CI%" == "azure" ( + echo ##[group]%~1 + exit /b +) +echo %~1 +exit /b + +:end_group +if /i "%CI%" == "github_actions" ( + echo ::endgroup:: + exit /b +) +if /i "%CI%" == "azure" ( + echo ##[endgroup] + exit /b +) +exit /b \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 9a10a93..5dd4ea6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,9 @@ matrix: script: - export CI=travis - export GIT_BRANCH="$TRAVIS_BRANCH" + - export flow_run_id="travis_$TRAVIS_JOB_ID" + - export sha="$TRAVIS_COMMIT" + - export remote_url="https://github.com/$TRAVIS_REPO_SLUG" - export FEEDSTOCK_NAME=$(basename ${TRAVIS_REPO_SLUG}) - if [[ "${TRAVIS_PULL_REQUEST:-}" == "false" ]]; then export IS_PR_BUILD="False"; else export IS_PR_BUILD="True"; fi diff --git a/README.md b/README.md index 299bb36..28b56f5 100644 --- a/README.md +++ b/README.md @@ -111,17 +111,24 @@ Current build status - win_64_cuda_compiler_version11.2 + win_64_cuda_compilercuda-nvcccuda_compiler_version12.0 - variant + variant - win_64_cuda_compiler_version11.8 + win_64_cuda_compilernvcccuda_compiler_version11.2 - variant + variant + + + + win_64_cuda_compilernvcccuda_compiler_version11.8 + + + variant @@ -205,7 +212,7 @@ available continuous integration services. Thanks to the awesome service provide [CircleCI](https://circleci.com/), [AppVeyor](https://www.appveyor.com/), [Drone](https://cloud.drone.io/welcome), and [TravisCI](https://travis-ci.com/) it is possible to build and upload installable packages to the -[conda-forge](https://anaconda.org/conda-forge) [Anaconda-Cloud](https://anaconda.org/) +[conda-forge](https://anaconda.org/conda-forge) [anaconda.org](https://anaconda.org/) channel for Linux, Windows and OSX respectively. To manage the continuous integration and simplify feedstock maintenance