[omnibus] Upgrade setuptools to 66.1.1, pip to 22.3.1 in Python 3 embedded environment

omnibus/config/software/datadog-agent-integrations-py3.rb

@@ -10,6 +10,7 @@
 
 dependency 'datadog-agent'
 dependency 'pip3'
+dependency 'setuptools3'
 
 dependency 'snowflake-connector-python-py3'
 
@@ -117,10 +118,8 @@
 
   # aliases for pip
   if windows?
-    pip = "#{windows_safe_path(python_3_embedded)}\\Scripts\\pip.exe"
     python = "#{windows_safe_path(python_3_embedded)}\\python.exe"
   else
-    pip = "#{install_dir}/embedded/bin/pip3"
     python = "#{install_dir}/embedded/bin/python3"
   end
 
@@ -144,10 +143,10 @@
     # Prepare the build env, these dependencies are only needed to build and
     # install the core integrations.
     #
-    command "#{pip} download --dest #{build_deps_dir} hatchling==0.25.1", :env => pre_build_env
-    command "#{pip} download --dest #{build_deps_dir} setuptools==40.9.0", :env => pre_build_env # Version from ./setuptools3.rb
-    command "#{pip} install wheel==0.38.4", :env => pre_build_env
-    command "#{pip} install pip-tools==6.4.0", :env => pre_build_env
+    command "#{python} -m pip download --dest #{build_deps_dir} hatchling==0.25.1", :env => pre_build_env
+    command "#{python} -m pip download --dest #{build_deps_dir} setuptools==66.1.1", :env => pre_build_env # Version from ./setuptools3.rb
+    command "#{python} -m pip install wheel==0.38.4", :env => pre_build_env
+    command "#{python} -m pip install pip-tools==6.12.1", :env => pre_build_env
     uninstall_buildtime_deps = ['rtloader', 'click', 'first', 'pip-tools']
     nix_build_env = {
       "PIP_FIND_LINKS" => "#{build_deps_dir}",
@@ -267,10 +266,10 @@
         "--pip-args \"--retries #{pip_max_retries} --timeout #{pip_timeout}\"", :env => env
       end
     else
-      command "#{pip} wheel . --no-deps --no-index --wheel-dir=#{wheel_build_dir}", :env => nix_build_env, :cwd => "#{project_dir}/datadog_checks_base"
-      command "#{pip} install datadog_checks_base --no-deps --no-index --find-links=#{wheel_build_dir}"
-      command "#{pip} wheel . --no-deps --no-index --wheel-dir=#{wheel_build_dir}", :env => nix_build_env, :cwd => "#{project_dir}/datadog_checks_downloader"
-      command "#{pip} install datadog_checks_downloader --no-deps --no-index --find-links=#{wheel_build_dir}"
+      command "#{python} -m pip wheel . --no-deps --no-index --wheel-dir=#{wheel_build_dir}", :env => nix_build_env, :cwd => "#{project_dir}/datadog_checks_base"
+      command "#{python} -m pip install datadog_checks_base --no-deps --no-index --find-links=#{wheel_build_dir}"
+      command "#{python} -m pip wheel . --no-deps --no-index --wheel-dir=#{wheel_build_dir}", :env => nix_build_env, :cwd => "#{project_dir}/datadog_checks_downloader"
+      command "#{python} -m pip install datadog_checks_downloader --no-deps --no-index --find-links=#{wheel_build_dir}"
       command "#{python} -m piptools compile --generate-hashes --output-file #{install_dir}/#{agent_requirements_file} #{static_reqs_out_file} " \
         "--pip-args \"--retries #{pip_max_retries} --timeout #{pip_timeout}\"", :env => nix_build_env
       # Pip-compiling seperately each lib that needs a custom build installation
@@ -296,7 +295,7 @@
         command "#{python} -m pip install --no-deps --require-hashes -r #{install_dir}/agent_#{lib}_requirements-py3.txt", :env => env
       end
       # Then we install the rest (already installed libraries will be ignored) with the main flags
-      command "#{pip} install --no-deps --require-hashes -r #{install_dir}/#{agent_requirements_file}", :env => nix_build_env
+      command "#{python} -m pip install --no-deps --require-hashes -r #{install_dir}/#{agent_requirements_file}", :env => nix_build_env
     end
 
     #
@@ -305,7 +304,7 @@
 
     # Create a constraint file after installing all the core dependencies and before any integration
     # This is then used as a constraint file by the integration command to avoid messing with the agent's python environment
-    command "#{pip} freeze > #{install_dir}/#{final_constraints_file}"
+    command "#{python} -m pip freeze > #{install_dir}/#{final_constraints_file}"
 
     if windows?
         cached_wheels_dir = "#{windows_safe_path(wheel_build_dir)}\\.cached"
@@ -374,7 +373,7 @@
         command "#{python} -m pip install --no-deps --no-index " \
           " --find-links #{windows_safe_path(cached_wheels_dir)} -r #{windows_safe_path(cached_wheels_dir)}\\found.txt"
       else
-        command "#{pip} install --no-deps --no-index " \
+        command "#{python} -m pip install --no-deps --no-index " \
           "--find-links #{cached_wheels_dir} -r #{cached_wheels_dir}/found.txt"
       end
     end
@@ -386,11 +385,7 @@
       # get list of integration wheels already installed from cache
       installed_list = Array.new
       if cache_bucket != ''
-        if windows?
-          installed_out = `#{python} -m pip list --format json`
-        else
-          installed_out = `#{pip} list --format json`
-        end
+        installed_out = `#{python} -m pip list --format json`
         if $?.exitstatus == 0
           installed = JSON.parse(installed_out)
           installed.each do |package|
@@ -454,11 +449,10 @@
 
         if windows?
           command "#{python} -m pip wheel . --no-deps --no-index --wheel-dir=#{wheel_build_dir}", :env => win_build_env, :cwd => "#{windows_safe_path(project_dir)}\\#{check}"
-          command "#{python} -m pip install datadog-#{check} --no-deps --no-index --find-links=#{wheel_build_dir}"
         else
-          command "#{pip} wheel . --no-deps --no-index --wheel-dir=#{wheel_build_dir}", :env => nix_build_env, :cwd => "#{project_dir}/#{check}"
-          command "#{pip} install datadog-#{check} --no-deps --no-index --find-links=#{wheel_build_dir}"
+          command "#{python} -m pip wheel . --no-deps --no-index --wheel-dir=#{wheel_build_dir}", :env => nix_build_env, :cwd => "#{project_dir}/#{check}"
         end
+        command "#{python} -m pip install datadog-#{check} --no-deps --no-index --find-links=#{wheel_build_dir}"
         if cache_bucket != '' && ENV.fetch('INTEGRATION_WHEELS_SKIP_CACHE_UPLOAD', '') == '' && cache_branch != nil
           command "inv -e agent.upload-integration-to-cache " \
             "--python 3 --bucket #{cache_bucket} " \
@@ -473,11 +467,7 @@
 
       # From now on we don't need piptools anymore, uninstall its deps so we don't include them in the final artifact
       uninstall_buildtime_deps.each do |dep|
-        if windows?
-          command "#{python} -m pip uninstall -y #{dep}"
-        else
-          command "#{pip} uninstall -y #{dep}"
-        end
+        command "#{python} -m pip uninstall -y #{dep}"
       end
     end
 
@@ -493,11 +483,7 @@
       end
 
       # Run pip check to make sure the agent's python environment is clean, all the dependencies are compatible
-      if windows?
-        command "#{python} -m pip check"
-      else
-        command "#{pip} check"
-      end
+      command "#{python} -m pip check"
     end
   end
 

omnibus/config/software/pip3.rb

@@ -1,10 +1,15 @@
 name "pip3"
-default_version "21.3.1"
 
-dependency "setuptools3"
+# The version of pip used must be at least equal to the one bundled with the Python version we use
+# Python 3.8.16 bundles pip 22.0.4
+default_version "22.3.1"
+
+skip_transitive_dependency_licensing true
+
+dependency "python3"
 
 source :url => "https://github.com/pypa/pip/archive/#{version}.tar.gz",
-       :sha256 => "cbfb6a0b5bc2d1e4b4647729ee5b944bb313c8ffd9ff83b9d2e0f727f0c79714",
+       :sha256 => "8d9f7cd8ad0d6f0c70e71704fd3f0f6538d70930454f1f21bbc2f8e94f6964ee",
        :extract => :seven_zip
 
 relative_path "pip-#{version}"
@@ -14,14 +19,12 @@
   license_file "https://raw.githubusercontent.com/pypa/pip/main/LICENSE.txt"
 
   if ohai["platform"] == "windows"
-    python_bin = "#{windows_safe_path(python_3_embedded)}\\python.exe"
-    python_prefix = "#{windows_safe_path(python_3_embedded)}"
+    python = "#{windows_safe_path(python_3_embedded)}\\python.exe"
   else
-    python_bin = "#{install_dir}/embedded/bin/python3"
-    python_prefix = "#{install_dir}/embedded"
+    python = "#{install_dir}/embedded/bin/python3"
   end
 
-  command "#{python_bin} setup.py install --prefix=#{python_prefix}"
+  command "#{python} -m pip install ."
 
   if ohai["platform"] != "windows"
     block do

omnibus/config/software/python3.rb

@@ -22,7 +22,7 @@
   python_configure = ["./configure",
                       "--prefix=#{install_dir}/embedded",
                       "--with-ssl=#{install_dir}/embedded",
-                      "--with-ensurepip=no"] # pip is installed separately by its own software def
+                      "--with-ensurepip=yes"] # We upgrade pip later, in the pip3 software definition
 
   if mac_os_x?
     python_configure.push("--enable-ipv6",
@@ -88,5 +88,9 @@
 
     command "XCOPY /YEHIR *.* \"#{windows_safe_path(python_3_embedded)}\""
     command "copy /y \"#{windows_safe_path(vcrt140_root)}\\*.dll\" \"#{windows_safe_path(python_3_embedded)}\""
+
+    # Install pip
+    python = "#{windows_safe_path(python_3_embedded)}\\python.exe"
+    command "#{python} -m ensurepip"
   end
 end

omnibus/config/software/setuptools3.rb

@@ -1,30 +1,30 @@
 name "setuptools3"
-default_version "40.9.0"
+
+# The version of setuptools used must be at least equal to the one bundled with the Python version we use
+# Python 3.8.16 bundles setuptools 56.0.0
+default_version "66.1.1"
 
 skip_transitive_dependency_licensing true
 
-dependency "python3"
+dependency "pip3"
 
 relative_path "setuptools-#{version}"
 
 source :url => "https://github.com/pypa/setuptools/archive/v#{version}.tar.gz",
-       :sha256 => "9ef6623c057d6e46ada8156bb48dc72ef6dbe721768720cc66966cca4097061c",
+       :sha256 => "081209b1c080b30ca78013dde35600a3070508fcce745a939498f4d76e05b6a6",
        :extract => :seven_zip
 
 build do
   # 2.0 is the license version here, not the python version
   license "Python-2.0"
 
   if ohai["platform"] == "windows"
-    python_bin = "#{windows_safe_path(python_3_embedded)}\\python.exe"
-    python_prefix = "#{windows_safe_path(python_3_embedded)}"
+    python = "#{windows_safe_path(python_3_embedded)}\\python.exe"
   else
-    python_bin = "#{install_dir}/embedded/bin/python3"
-    python_prefix = "#{install_dir}/embedded"
+    python = "#{install_dir}/embedded/bin/python3"
   end
 
-  command "#{python_bin} bootstrap.py"
-  command "#{python_bin} setup.py install --prefix=#{python_prefix}"
+  command "#{python} -m pip install ."
 
   if ohai["platform"] != "windows"
     block do

releasenotes/notes/update-setuptools-pip-1b049dc6f774ac39.yaml

@@ -0,0 +1,15 @@
+# Each section from every release note are combined when the
+# CHANGELOG.rst is rendered. So the text needs to be worded so that
+# it does not depend on any information only available in another
+# section. This may mean repeating some details, but each section
+# must be readable independently of the other.
+#
+# Each section note must be formatted as reStructuredText.
+---
+other:
+  - |
+    The ``setuptools`` package bundled in the Agent's Python 3 environment
+    has been upgraded to 66.1.1.
+  - |
+    The ``pip`` package bundled in the Agent's Python 3 environment
+    has been upgraded to 22.3.1.