Update python builds to latest CI image (ubuntu 24.04 build) (#35534)

* Switch python builds to latest CI image

* Switch logic for pip updates and wheel installation: do not hardcode paths

* More fixes for new python, where pip cannot just be used

Author: andy31415

.github/workflows/build.yaml

@@ -283,7 +283,7 @@ jobs:
         if: github.actor != 'restyled-io[bot]'
 
         container:
-            image: ghcr.io/project-chip/chip-build:54
+            image: ghcr.io/project-chip/chip-build:74
             volumes:
                 - "/:/runner-root-volume"
                 - "/tmp/log_output:/tmp/test_logs"
@@ -316,9 +316,10 @@ jobs:
               run: |
                   scripts/run_in_build_env.sh 'virtualenv pyenv'
                   source pyenv/bin/activate
-                  pip3 install ./out/controller/python/chip_core-0.0-cp37-abi3-linux_x86_64.whl
-                  pip3 install ./out/controller/python/chip_clusters-0.0-py3-none-any.whl
-                  pip3 install ./out/controller/python/chip_repl-0.0-py3-none-any.whl
+                  python -m ensurepip --upgrade
+                  python -m pip install ./out/controller/python/chip_core-0.0-cp37-abi3-linux_x86_64.whl
+                  python -m pip install ./out/controller/python/chip_clusters-0.0-py3-none-any.whl
+                  python -m pip install ./out/controller/python/chip_repl-0.0-py3-none-any.whl
 
             - name: Run Python tests
               shell: bash
@@ -334,7 +335,8 @@ jobs:
                   scripts/run_in_build_env.sh 'scripts/examples/gn_build_example.sh examples/chip-tool out/'
                   scripts/run_in_build_env.sh 'virtualenv pyenv'
                   source pyenv/bin/activate
-                  pip3 install -r scripts/setup/requirements.setuppayload.txt
+                  python -m ensurepip --upgrade
+                  python -m pip install -r scripts/setup/requirements.setuppayload.txt
                   python3 src/setup_payload/tests/run_python_setup_payload_test.py out/chip-tool
 
     build_linux_python_lighting_device:
@@ -344,7 +346,7 @@ jobs:
         if: github.actor != 'restyled-io[bot]'
 
         container:
-            image: ghcr.io/project-chip/chip-build:54
+            image: ghcr.io/project-chip/chip-build:74
             volumes:
                 - "/:/runner-root-volume"
                 - "/tmp/log_output:/tmp/test_logs"

scripts/build_python.sh

@@ -217,8 +217,8 @@ if [ -n "$install_virtual_env" ]; then
     fi
 
     source "$ENVIRONMENT_ROOT"/bin/activate
-    "$ENVIRONMENT_ROOT"/bin/python -m pip install --upgrade pip
-    "$ENVIRONMENT_ROOT"/bin/pip install --upgrade "${WHEEL[@]}"
+    "$ENVIRONMENT_ROOT"/bin/python -m ensurepip --upgrade
+    "$ENVIRONMENT_ROOT"/bin/python -m pip install --upgrade "${WHEEL[@]}"
 
     if [ "$install_pytest_requirements" = "yes" ]; then
         YAMLTESTS_GN_LABEL="//scripts:matter_yamltests_distribution._build_wheel"

scripts/build_python_device.sh

@@ -104,8 +104,8 @@ virtualenv --clear "$ENVIRONMENT_ROOT"
 WHEEL=("$OUTPUT_ROOT"/controller/python/chip_core*.whl)
 
 source "$ENVIRONMENT_ROOT"/bin/activate
-"$ENVIRONMENT_ROOT"/bin/python -m pip install --upgrade pip
-"$ENVIRONMENT_ROOT"/bin/pip install --upgrade --force-reinstall --no-cache-dir "${WHEEL[@]}"
+"$ENVIRONMENT_ROOT"/bin/python -m ensurepip --upgrade
+"$ENVIRONMENT_ROOT"/bin/python -m pip install --upgrade --force-reinstall --no-cache-dir "${WHEEL[@]}"
 
 echo ""
 echo_green "Compilation completed and WHL package installed in: "