[CI] Artifactory for Python

.gitlab-ci.yml

@@ -168,7 +168,9 @@ variables:
   ARTIFACTORY_TOKEN_SSM_NAME: ci.datadog-agent.artifactory_token
   ARTIFACTORY_URL: datadog.jfrog.io
   ARTIFACTORY_GEMS_PATH: artifactory/api/gems/agent-gems
+  ARTIFACTORY_PYPI_PATH: artifactory/api/pypi/agent-pypi/simple
   USE_CACHING_PROXY_RUBY: "true"
+  USE_CACHING_PROXY_PYTHON: "true"
   CLANG_LLVM_VER: 12.0.1
 
 #

.gitlab/maintenance_jobs/docker.yml

@@ -68,6 +68,7 @@ delete_docker_tag:
   before_script:
     - DOCKER_REGISTRY_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_LOGIN_SSM_KEY --with-decryption --query "Parameter.Value" --out text)
     - PASS=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_PWD_SSM_KEY --with-decryption --query "Parameter.Value" --out text)
+    - !reference [.setup_python_mirror_linux]
     - python3 -m pip install -r requirements.txt
     - |
       export DOCKER_TOKEN=`curl -s -H "Content-Type: application/json" -X POST -d '{"username": "'$DOCKER_REGISTRY_LOGIN'", "password": "'$PASS'"}' https://hub.docker.com/v2/users/login/ | python -c 'import sys, json; print(json.load(sys.stdin)["token"].strip())'`

.gitlab/notify.yml

@@ -25,6 +25,7 @@ notify:
   script:
     - set +x
     - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_read_api_token --with-decryption --query "Parameter.Value" --out text)
+    - !reference [.setup_python_mirror_linux]
     - python3 -m pip install -r tasks/libs/requirements-notifications.txt
     - |
       # Do not send notifications if this is a child pipeline of another repo
@@ -51,5 +52,6 @@ send_pipeline_stats:
     - set +x
     - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_read_api_token --with-decryption --query "Parameter.Value" --out text)
     - export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text)
+    - !reference [.setup_python_mirror_linux]
     - python3 -m pip install -r tasks/libs/requirements-stats.txt
     - invoke -e pipeline.send-stats

.gitlab/package_build/deb.yml

@@ -10,6 +10,7 @@
   script:
     - echo "About to build for $RELEASE_VERSION"
     - !reference [.setup_ruby_mirror_linux]
+    - !reference [.setup_python_mirror_linux]
     - !reference [.retrieve_linux_go_deps]
     # remove artifacts from previous pipelines that may come from the cache
     - rm -rf $OMNIBUS_PACKAGE_DIR/*
@@ -131,6 +132,7 @@ agent_deb-arm64-a7:
 .iot_agent_build_common_deb:
   script:
     - !reference [.setup_ruby_mirror_linux]
+    - !reference [.setup_python_mirror_linux]
     - !reference [.retrieve_linux_go_deps]
     - echo "About to build for $RELEASE_VERSION_7"
     - echo "Detected host architecture $(uname -m)"

.gitlab/package_build/dmg.yml

@@ -8,6 +8,7 @@
     - export GITHUB_KEY_B64=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.macos_github_key_b64 --with-decryption --query "Parameter.Value" --out text)
     - export GITHUB_APP_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.macos_github_app_id --with-decryption --query "Parameter.Value" --out text)
     - export GITHUB_INSTALLATION_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.macos_github_installation_id --with-decryption --query "Parameter.Value" --out text)
+    - !reference [.setup_python_mirror_linux]
     - python3 -m pip install -r tasks/libs/requirements-github.txt
     - inv -e github.trigger-macos-build --datadog-agent-ref "$CI_COMMIT_SHA" --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --python-runtimes "$PYTHON_RUNTIMES" --destination "$OMNIBUS_PACKAGE_DIR"
     - !reference [.upload_sbom_artifacts]

.gitlab/package_build/rpm.yml

@@ -3,6 +3,7 @@
   script:
     - echo "About to build for $RELEASE_VERSION"
     - !reference [.setup_ruby_mirror_linux]
+    - !reference [.setup_python_mirror_linux]
     - !reference [.retrieve_linux_go_deps]
     # remove artifacts from previous pipelines that may come from the cache
     - rm -rf $OMNIBUS_PACKAGE_DIR/*
@@ -124,6 +125,7 @@ agent_rpm-arm64-a7:
   script:
     - echo "About to build iot agent for $RELEASE_VERSION_7"
     - !reference [.setup_ruby_mirror_linux]
+    - !reference [.setup_python_mirror_linux]
     - !reference [.retrieve_linux_go_deps]
     - echo "Detected host architecture $(uname -m)"
     # $DD_TARGET_ARCH is only set by Arm build images, so assume amd64 if not present

.gitlab/package_build/suse_rpm.yml

@@ -3,6 +3,7 @@
   script:
     - echo "About to build for $RELEASE_VERSION"
     - !reference [.setup_ruby_mirror_linux]
+    - !reference [.setup_python_mirror_linux]
     - !reference [.retrieve_linux_go_deps]
     # remove artifacts from previous pipelines that may come from the cache
     - rm -rf $OMNIBUS_PACKAGE_DIR_SUSE/*
@@ -100,6 +101,7 @@ iot_agent_suse-x64:
   script:
     - echo "About to build iot agent for $RELEASE_VERSION_7"
     - !reference [.setup_ruby_mirror_linux]
+    - !reference [.setup_python_mirror_linux]
     - !reference [.retrieve_linux_go_deps]
     - echo "Detected host architecture $(uname -m)"
     # $DD_TARGET_ARCH is only set by Arm build images, so assume amd64 if not present

.gitlab/package_build/windows.yml

@@ -10,7 +10,8 @@
     - mkdir omnibus\pkg
     - $vcpkgBlobSaSUrl = (aws ssm get-parameter --region us-east-1 --name ci.datadog-agent-buildimages.vcpkg_blob_sas_url --with-decryption --query "Parameter.Value" --out text)
     - !reference [.setup_ruby_mirror_win]
-    - docker run --rm -m 4096M -v "$(Get-Location):c:\mnt" -e CI_JOB_ID=${CI_JOB_ID} -e CI_PIPELINE_ID=${CI_PIPELINE_ID} -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} -e OMNIBUS_TARGET=${OMNIBUS_TARGET} -e WINDOWS_BUILDER=true -e RELEASE_VERSION="$RELEASE_VERSION" -e MAJOR_VERSION="$AGENT_MAJOR_VERSION" -e PY_RUNTIMES="$PYTHON_RUNTIMES" -e INTEGRATIONS_CORE_VERSION="$INTEGRATIONS_CORE_VERSION" -e GOMODCACHE="c:\modcache" -e AWS_NETWORKING=true -e SIGN_WINDOWS=true -e TARGET_ARCH="$ARCH" -e DEBUG_CUSTOMACTION="$DEBUG_CUSTOMACTION" -e BUCKET_BRANCH="$BUCKET_BRANCH" -e S3_OMNIBUS_CACHE_BUCKET="$S3_OMNIBUS_CACHE_BUCKET" -e USE_S3_CACHING="$USE_S3_CACHING" -e INTEGRATION_WHEELS_CACHE_BUCKET="$INTEGRATION_WHEELS_CACHE_BUCKET" -e VCPKG_BINARY_SOURCES="clear;x-azblob,${vcpkgBlobSaSUrl}" -e GO_VERSION_CHECK="true" -e BUNDLE_MIRROR__RUBYGEMS__ORG=${BUNDLE_MIRROR__RUBYGEMS__ORG} 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}:${Env:DATADOG_AGENT_WINBUILDIMAGES} c:\mnt\tasks\winbuildscripts\buildwin.bat
+    - !reference [.setup_python_mirror_win]
+    - docker run --rm -m 4096M -v "$(Get-Location):c:\mnt" -e CI_JOB_ID=${CI_JOB_ID} -e CI_PIPELINE_ID=${CI_PIPELINE_ID} -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} -e OMNIBUS_TARGET=${OMNIBUS_TARGET} -e WINDOWS_BUILDER=true -e RELEASE_VERSION="$RELEASE_VERSION" -e MAJOR_VERSION="$AGENT_MAJOR_VERSION" -e PY_RUNTIMES="$PYTHON_RUNTIMES" -e INTEGRATIONS_CORE_VERSION="$INTEGRATIONS_CORE_VERSION" -e GOMODCACHE="c:\modcache" -e AWS_NETWORKING=true -e SIGN_WINDOWS=true -e TARGET_ARCH="$ARCH" -e DEBUG_CUSTOMACTION="$DEBUG_CUSTOMACTION" -e BUCKET_BRANCH="$BUCKET_BRANCH" -e S3_OMNIBUS_CACHE_BUCKET="$S3_OMNIBUS_CACHE_BUCKET" -e USE_S3_CACHING="$USE_S3_CACHING" -e INTEGRATION_WHEELS_CACHE_BUCKET="$INTEGRATION_WHEELS_CACHE_BUCKET" -e VCPKG_BINARY_SOURCES="clear;x-azblob,${vcpkgBlobSaSUrl}" -e GO_VERSION_CHECK="true" -e BUNDLE_MIRROR__RUBYGEMS__ORG=${BUNDLE_MIRROR__RUBYGEMS__ORG} -e PIP_INDEX_URL=${PIP_INDEX_URL} 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}:${Env:DATADOG_AGENT_WINBUILDIMAGES} c:\mnt\tasks\winbuildscripts\buildwin.bat
     - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
     - get-childitem omnibus\pkg
     - !reference [.upload_sbom_artifacts_windows]
@@ -68,7 +69,8 @@ windows_zip_agent_binaries_x64-a7:
     - if (Test-Path omnibus\pkg) { remove-item -recurse -force omnibus\pkg }
     - mkdir omnibus\pkg
     - !reference [.setup_ruby_mirror_win]
-    - docker run --rm -m 4096M -v "$(Get-Location):c:\mnt" -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} -e OMNIBUS_TARGET=${OMNIBUS_TARGET} -e WINDOWS_BUILDER=true -e RELEASE_VERSION="$RELEASE_VERSION" -e MAJOR_VERSION="$AGENT_MAJOR_VERSION" -e INTEGRATIONS_CORE_VERSION="$INTEGRATIONS_CORE_VERSION" -e PY_RUNTIMES="$PYTHON_RUNTIMES" -e GOMODCACHE="c:\modcache" -e AWS_NETWORKING=true -e SIGN_WINDOWS=true -e BUCKET_BRANCH="$BUCKET_BRANCH" -e INTEGRATION_WHEELS_CACHE_BUCKET="$INTEGRATION_WHEELS_CACHE_BUCKET" -e S3_OMNIBUS_CACHE_BUCKET="$S3_OMNIBUS_CACHE_BUCKET" -e USE_S3_CACHING="$USE_S3_CACHING" -e VCPKG_BINARY_SOURCES="clear;x-azblob,${vcpkgBlobSaSUrl}" -e BUNDLE_MIRROR__RUBYGEMS__ORG=${BUNDLE_MIRROR__RUBYGEMS__ORG} 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}:${Env:DATADOG_AGENT_WINBUILDIMAGES} c:\mnt\tasks\winbuildscripts\buildwin.bat
+    - !reference [.setup_python_mirror_win]
+    - docker run --rm -m 4096M -v "$(Get-Location):c:\mnt" -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} -e OMNIBUS_TARGET=${OMNIBUS_TARGET} -e WINDOWS_BUILDER=true -e RELEASE_VERSION="$RELEASE_VERSION" -e MAJOR_VERSION="$AGENT_MAJOR_VERSION" -e INTEGRATIONS_CORE_VERSION="$INTEGRATIONS_CORE_VERSION" -e PY_RUNTIMES="$PYTHON_RUNTIMES" -e GOMODCACHE="c:\modcache" -e AWS_NETWORKING=true -e SIGN_WINDOWS=true -e BUCKET_BRANCH="$BUCKET_BRANCH" -e INTEGRATION_WHEELS_CACHE_BUCKET="$INTEGRATION_WHEELS_CACHE_BUCKET" -e S3_OMNIBUS_CACHE_BUCKET="$S3_OMNIBUS_CACHE_BUCKET" -e USE_S3_CACHING="$USE_S3_CACHING" -e VCPKG_BINARY_SOURCES="clear;x-azblob,${vcpkgBlobSaSUrl}" -e BUNDLE_MIRROR__RUBYGEMS__ORG=${BUNDLE_MIRROR__RUBYGEMS__ORG} -e PIP_INDEX_URL=${PIP_INDEX_URL} 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}:${Env:DATADOG_AGENT_WINBUILDIMAGES} c:\mnt\tasks\winbuildscripts\buildwin.bat
     - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
     - get-childitem omnibus\pkg
     - !reference [.upload_sbom_artifacts_windows]

.gitlab/shared.yml

@@ -15,3 +15,13 @@
 .setup_ruby_mirror_win:
   - *get_artifactory_token_win
   - if ($Env:USE_CACHING_PROXY_RUBY -eq "true") { $BUNDLE_MIRROR__RUBYGEMS__ORG="https://${Env:ARTIFACTORY_USERNAME}:${ARTIFACTORY_TOKEN}@${Env:ARTIFACTORY_URL}/${Env:ARTIFACTORY_GEMS_PATH}" }
+
+.setup_python_mirror_linux:
+  - set +x
+  - *get_artifactory_token_linux
+  - if [ "${USE_CACHING_PROXY_PYTHON}" = "true" ]; then export PIP_INDEX_URL=https://${ARTIFACTORY_USERNAME}:${ARTIFACTORY_TOKEN}@${ARTIFACTORY_URL}/${ARTIFACTORY_PYPI_PATH}; fi
+  - set -x
+
+.setup_python_mirror_win:
+  - *get_artifactory_token_win
+  - if ($Env:USE_CACHING_PROXY_PYTHON -eq "true") { $PIP_INDEX_URL="https://${Env:ARTIFACTORY_USERNAME}:${ARTIFACTORY_TOKEN}@${Env:ARTIFACTORY_URL}/${Env:ARTIFACTORY_PYPI_PATH}" }

.gitlab/source_test/ebpf.yml

@@ -58,7 +58,8 @@
   script:
     - $ErrorActionPreference = "Stop"
     - '$_instance_id = (iwr  -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"'
-    - docker run --rm -m 8192M -v "$(Get-Location):c:\mnt" -e AWS_NETWORKING=true -e SIGN_WINDOWS=true -e PY_RUNTIMES="$PYTHON_RUNTIMES" -e GOMODCACHE="c:\modcache" 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}:$Env:DATADOG_AGENT_WINBUILDIMAGES c:\mnt\tasks\winbuildscripts\sysprobe.bat
+    - !reference [.setup_python_mirror_win]
+    - docker run --rm -m 8192M -v "$(Get-Location):c:\mnt" -e AWS_NETWORKING=true -e SIGN_WINDOWS=true -e PY_RUNTIMES="$PYTHON_RUNTIMES" -e GOMODCACHE="c:\modcache" -e PIP_INDEX_URL=${PIP_INDEX_URL} 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}:$Env:DATADOG_AGENT_WINBUILDIMAGES c:\mnt\tasks\winbuildscripts\sysprobe.bat
     - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
   artifacts:
     when: always

.gitlab/source_test/macos.yml

@@ -12,6 +12,7 @@ tests_macos:
     - export GITHUB_KEY_B64=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.macos_github_key_b64 --with-decryption --query "Parameter.Value" --out text)
     - export GITHUB_APP_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.macos_github_app_id --with-decryption --query "Parameter.Value" --out text)
     - export GITHUB_INSTALLATION_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.macos_github_installation_id --with-decryption --query "Parameter.Value" --out text)
+    - !reference [.setup_python_mirror_linux]
     - python3 -m pip install -r tasks/libs/requirements-github.txt
     - inv -e github.trigger-macos-test --datadog-agent-ref "$CI_COMMIT_SHA" --python-runtimes "$PYTHON_RUNTIMES"
     - inv -e junit-macos-repack --infile junit-tests_macos.tgz --outfile junit-tests_macos-repacked.tgz

.gitlab/source_test/windows.yml

@@ -10,7 +10,8 @@
     - '$_instance_id = (iwr  -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"'
     # we pass in CI_JOB_URL and CI_JOB_NAME so that they can be added to additional tags
     # inside JUNIT_TAR and then later used by datadog-ci
-    - docker run --rm -m 8192M -v "$(Get-Location):c:\mnt" -e CI_JOB_URL="${CI_JOB_URL}" -e CI_JOB_NAME="${CI_JOB_NAME}" -e AWS_NETWORKING=true -e SIGN_WINDOWS=true -e PY_RUNTIMES="$PYTHON_RUNTIMES" -e GOMODCACHE="c:\modcache" -e JUNIT_TAR="c:\mnt\junit-${CI_JOB_NAME}.tgz" -e VCPKG_BINARY_SOURCES="clear;x-azblob,${vcpkgBlobSaSUrl}" 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}:$Env:DATADOG_AGENT_WINBUILDIMAGES c:\mnt\tasks\winbuildscripts\unittests.bat
+    - !reference [.setup_python_mirror_win]
+    - docker run --rm -m 8192M -v "$(Get-Location):c:\mnt" -e CI_JOB_URL="${CI_JOB_URL}" -e CI_JOB_NAME="${CI_JOB_NAME}" -e AWS_NETWORKING=true -e SIGN_WINDOWS=true -e PY_RUNTIMES="$PYTHON_RUNTIMES" -e GOMODCACHE="c:\modcache" -e JUNIT_TAR="c:\mnt\junit-${CI_JOB_NAME}.tgz" -e VCPKG_BINARY_SOURCES="clear;x-azblob,${vcpkgBlobSaSUrl}" -e PIP_INDEX_URL=${PIP_INDEX_URL} 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}:$Env:DATADOG_AGENT_WINBUILDIMAGES c:\mnt\tasks\winbuildscripts\unittests.bat
     - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
   artifacts:
     expire_in: 2 weeks

omnibus/config/software/datadog-agent-integrations-py2.rb

@@ -119,6 +119,13 @@
     python = "#{install_dir}/embedded/bin/python2"
   end
 
+  # If a python_mirror was set, it's passed through a pip config file so that we're not leaking the API key in the CI Output
+  # Else the pip config file so pip will act casually
+  pip_config_file = ENV['PIP_CONFIG_FILE']
+  pre_build_env = {
+    "PIP_CONFIG_FILE" => "#{pip_config_file}"
+  }
+
   # Install the checks along with their dependencies
   block do
     if windows?
@@ -133,14 +140,15 @@
     # Prepare the build env, these dependencies are only needed to build and
     # install the core integrations.
     #
-    command "#{pip} download --dest #{build_deps_dir} hatchling==0.25.1"
-    command "#{pip} download --dest #{build_deps_dir} setuptools==40.9.0" # Version from ./setuptools2.rb
-    command "#{pip} install wheel==0.34.1"
-    command "#{pip} install setuptools-scm==5.0.2" # Pin to the last version that supports Python 2
-    command "#{pip} install pip-tools==5.4.0"
+    command "#{pip} download --dest #{build_deps_dir} hatchling==0.25.1", :env => pre_build_env
+    command "#{pip} download --dest #{build_deps_dir} setuptools==40.9.0", :env => pre_build_env # Version from ./setuptools2.rb
+    command "#{pip} install wheel==0.34.1", :env => pre_build_env
+    command "#{pip} install setuptools-scm==5.0.2", :env => pre_build_env # Pin to the last version that supports Python 2
+    command "#{pip} install pip-tools==5.4.0", :env => pre_build_env
     uninstall_buildtime_deps = ['rtloader', 'click', 'first', 'pip-tools']
     nix_build_env = {
       "PIP_FIND_LINKS" => "#{build_deps_dir}",
+      "PIP_CONFIG_FILE" => "#{pip_config_file}",
       "CFLAGS" => "-I#{install_dir}/embedded/include -I/opt/mqm/inc",
       "CXXFLAGS" => "-I#{install_dir}/embedded/include -I/opt/mqm/inc",
       "LDFLAGS" => "-L#{install_dir}/embedded/lib -L/opt/mqm/lib64 -L/opt/mqm/lib",
@@ -149,6 +157,7 @@
     }
     win_build_env = {
       "PIP_FIND_LINKS" => "#{build_deps_dir}",
+      "PIP_CONFIG_FILE" => "#{pip_config_file}",
     }
     # Some libraries (looking at you, aerospike-client-python) need EXT_CFLAGS instead of CFLAGS.
     specific_build_env = {

omnibus/config/software/datadog-agent-integrations-py3.rb

@@ -124,6 +124,12 @@
     python = "#{install_dir}/embedded/bin/python3"
   end
 
+  # If a python_mirror is set, it is set in a pip config file so that we do not leak the token in the CI output
+  pip_config_file = ENV['PIP_CONFIG_FILE']
+  pre_build_env = {
+    "PIP_CONFIG_FILE" => "#{pip_config_file}"
+  }
+
   # Install the checks along with their dependencies
   block do
     if windows?
@@ -138,13 +144,14 @@
     # Prepare the build env, these dependencies are only needed to build and
     # install the core integrations.
     #
-    command "#{pip} download --dest #{build_deps_dir} hatchling==0.25.1"
-    command "#{pip} download --dest #{build_deps_dir} setuptools==40.9.0" # Version from ./setuptools3.rb
-    command "#{pip} install wheel==0.34.1"
-    command "#{pip} install pip-tools==6.4.0"
+    command "#{pip} download --dest #{build_deps_dir} hatchling==0.25.1", :env => pre_build_env
+    command "#{pip} download --dest #{build_deps_dir} setuptools==40.9.0", :env => pre_build_env # Version from ./setuptools3.rb
+    command "#{pip} install wheel==0.34.1", :env => pre_build_env
+    command "#{pip} install pip-tools==6.4.0", :env => pre_build_env
     uninstall_buildtime_deps = ['rtloader', 'click', 'first', 'pip-tools']
     nix_build_env = {
       "PIP_FIND_LINKS" => "#{build_deps_dir}",
+      "PIP_CONFIG_FILE" => "#{pip_config_file}",
       # Specify C99 standard explicitly to avoid issues while building some
       # wheels (eg. ddtrace)
       "CFLAGS" => "-I#{install_dir}/embedded/include -I/opt/mqm/inc",
@@ -155,6 +162,7 @@
     }
     win_build_env = {
       "PIP_FIND_LINKS" => "#{build_deps_dir}",
+      "PIP_CONFIG_FILE" => "#{pip_config_file}",
     }
     # Some libraries (looking at you, aerospike-client-python) need EXT_CFLAGS instead of CFLAGS.
     specific_build_env = {

omnibus/config/software/pylint2.rb

@@ -17,14 +17,21 @@
     python2 = "#{install_dir}/embedded/bin/python2"
   end
 
+  # If a python_mirror was set, it's passed through a pip config file so that we're not leaking the API key in the CI Output
+  # Else the pip config file so pip will act casually
+  pip_config_file = ENV['PIP_CONFIG_FILE']
+  build_env = {
+    "PIP_CONFIG_FILE" => "#{pip_config_file}"
+  }
+
   # pin 2 dependencies of pylint:
   # - configparser: later versions (up to v3.7.1) are broken
   # - lazy-object-proxy 1.7.0 broken on python 2 https://github.com/ionelmc/python-lazy-object-proxy/issues/61
   if windows?
-    command "#{python2} -m pip install configparser==3.5.0 lazy-object-proxy==1.6.0"
-    command "#{python2} -m pip install pylint==#{version}"
+    command "#{python2} -m pip install configparser==3.5.0 lazy-object-proxy==1.6.0", :env => build_env
+    command "#{python2} -m pip install pylint==#{version}", :env => build_env
   else
-    command "#{pip2} install configparser==3.5.0 lazy-object-proxy==1.6.0"
-    command "#{pip2} install pylint==#{version}"
+    command "#{pip2} install configparser==3.5.0 lazy-object-proxy==1.6.0", :env => build_env
+    command "#{pip2} install pylint==#{version}", :env => build_env
   end
 end