[python] Split Python wheel in two separate wheels (#20054)

* Move package information into manifest file

* Specify scripts in manifest as well

* Make build-chip-wheel.py independent of build variant

Pass library name via command line arguments. This makes
build-chip-wheel.py more generic.

* Move package requirements to manifest file as well

* Split Python into library and repl package

* Introduce common GN template chip_python_wheel_action

* Add runtime dependency between chip-repl and chip-library

* Split chip-library into chip-core and chip-clusters

* Apply restyled

* Install all wheels for unit tests

* Use new chip-repl build target

* Fix Cirque failures

* Fix wheel names for chip_clusters/chip_repl

Author: agners

.github/workflows/build.yaml

@@ -316,7 +316,9 @@ jobs:
             - name: Run Python library specific unit tests
               timeout-minutes: 5
               run: |
-                  scripts/run_in_build_env.sh 'pip3 install ./out/controller/python/chip-0.0-cp37-abi3-linux_x86_64.whl'
+                  scripts/run_in_build_env.sh 'pip3 install ./out/controller/python/chip_core-0.0-cp37-abi3-linux_x86_64.whl'
+                  scripts/run_in_build_env.sh 'pip3 install ./out/controller/python/chip_clusters-0.0-py3-none-any.whl'
+                  scripts/run_in_build_env.sh 'pip3 install ./out/controller/python/chip_repl-0.0-py3-none-any.whl'
                   scripts/run_in_build_env.sh '(cd src/controller/python/test/unit_tests/ && python3 -m unittest -v)'
 
     build_darwin:

BUILD.gn

@@ -140,7 +140,7 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
         if (enable_pylib) {
           deps += [ "${chip_root}/src/pybindings/pycontroller" ]
         }
-        deps += [ "${chip_root}/src/controller/python" ]
+        deps += [ "${chip_root}/src/controller/python:chip-repl" ]
       }
     }
 
@@ -166,7 +166,7 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
         if (enable_pylib) {
           data_deps += [ "${chip_root}/src/pybindings/pycontroller" ]
         }
-        data_deps += [ "${chip_root}/src/controller/python" ]
+        data_deps += [ "${chip_root}/src/controller/python:chip-repl" ]
       }
 
       write_runtime_deps = "${root_out_dir}/certification.runtime_deps"

scripts/build/builders/host.py

@@ -278,7 +278,7 @@ def __init__(self, root, runner, app: HostApp, board=HostBoard.NATIVE,
         elif app == HostApp.PYTHON_BINDINGS:
             self.extra_gn_options.append('enable_rtti=false')
             self.extra_gn_options.append('chip_project_config_include_dirs=["//config/python"]')
-            self.build_command = 'python'
+            self.build_command = 'chip-repl'
 
     def GnBuildArgs(self):
         if self.board == HostBoard.NATIVE:

scripts/build/testdata/build_linux_on_x64.txt

@@ -374,7 +374,7 @@ ninja -C {out}/linux-arm64-ota-requestor
 ninja -C {out}/linux-arm64-ota-requestor-ipv6only
 
 # Building linux-arm64-python-bindings
-ninja -C {out}/linux-arm64-python-bindings python
+ninja -C {out}/linux-arm64-python-bindings chip-repl
 
 # Building linux-arm64-shell
 ninja -C {out}/linux-arm64-shell
@@ -491,7 +491,7 @@ ninja -C {out}/linux-x64-ota-requestor
 ninja -C {out}/linux-x64-ota-requestor-ipv6only
 
 # Building linux-x64-python-bindings
-ninja -C {out}/linux-x64-python-bindings python
+ninja -C {out}/linux-x64-python-bindings chip-repl
 
 # Building linux-x64-rpc-console
 ninja -C {out}/linux-x64-rpc-console

scripts/build_python.sh

@@ -120,13 +120,13 @@ gn --root="$CHIP_ROOT" gen "$OUTPUT_ROOT" --args="chip_detail_logging=$chip_deta
 if [ "$enable_pybindings" == true ]; then
     ninja -C "$OUTPUT_ROOT" pycontroller
 else
-    ninja -C "$OUTPUT_ROOT" python
+    ninja -C "$OUTPUT_ROOT" chip-repl
 fi
 
 if [ "$enable_pybindings" == true ]; then
-    WHEEL=$(ls "$OUTPUT_ROOT"/pybindings/pycontroller/pychip-*.whl | head -n 1)
+    WHEEL=("$OUTPUT_ROOT"/pybindings/pycontroller/pychip-*.whl)
 else
-    WHEEL=$(ls "$OUTPUT_ROOT"/controller/python/chip-*.whl | head -n 1)
+    WHEEL=("$OUTPUT_ROOT"/controller/python/chip*.whl)
 fi
 
 if [ "$install_wheel" = "no" ]; then
@@ -137,7 +137,7 @@ elif [ "$install_wheel" = "separate" ]; then
 
     source "$ENVIRONMENT_ROOT"/bin/activate
     "$ENVIRONMENT_ROOT"/bin/python -m pip install --upgrade pip
-    "$ENVIRONMENT_ROOT"/bin/pip install --upgrade --force-reinstall --no-cache-dir "$WHEEL"
+    "$ENVIRONMENT_ROOT"/bin/pip install --upgrade --force-reinstall --no-cache-dir "${WHEEL[@]}"
 
     echo ""
     echo_green "Compilation completed and WHL package installed in: "
@@ -146,7 +146,7 @@ elif [ "$install_wheel" = "separate" ]; then
     echo_green "To use please run:"
     echo_bold_white "  source $ENVIRONMENT_ROOT/bin/activate"
 elif [ "$install_wheel" = "build-env" ]; then
-    pip install --force-reinstall "$WHEEL"
+    pip install --force-reinstall "${WHEEL[@]}"
 
     echo ""
     echo_green "Compilation completed and WHL package installed in virtualenv for building sdk"

scripts/build_python_device.sh

@@ -104,7 +104,7 @@ gn --root="$CHIP_ROOT" gen "$OUTPUT_ROOT" --args="chip_detail_logging=$chip_deta
 if [ "$enable_pybindings" == true ]; then
     ninja -v -C "$OUTPUT_ROOT" pycontroller
 else
-    ninja -v -C "$OUTPUT_ROOT" python
+    ninja -v -C "$OUTPUT_ROOT" chip-library
 fi
 
 # Create a virtual environment that has access to the built python tools
@@ -113,14 +113,14 @@ virtualenv --clear "$ENVIRONMENT_ROOT"
 # Activate the new environment to register the python WHL
 
 if [ "$enable_pybindings" == true ]; then
-    WHEEL=$(ls "$OUTPUT_ROOT"/pybindings/pycontroller/pychip-*.whl | head -n 1)
+    WHEEL=("$OUTPUT_ROOT"/pybindings/pycontroller/pychip-*.whl)
 else
-    WHEEL=$(ls "$OUTPUT_ROOT"/controller/python/chip-*.whl | head -n 1)
+    WHEEL=("$OUTPUT_ROOT"/controller/python/chip_library*.whl)
 fi
 
 source "$ENVIRONMENT_ROOT"/bin/activate
 "$ENVIRONMENT_ROOT"/bin/python -m pip install --upgrade pip
-"$ENVIRONMENT_ROOT"/bin/pip install --upgrade --force-reinstall --no-cache-dir "$WHEEL"
+"$ENVIRONMENT_ROOT"/bin/pip install --upgrade --force-reinstall --no-cache-dir "${WHEEL[@]}"
 
 echo ""
 echo_green "Compilation completed and WHL package installed in: "