Avoid usage of a separate build environment for compile time codegen (#23861)

* Try to disable virtual env usage for codegen

* Fix IDL paths. also add missing callbackstubsource.jinja from the list for change detection

* Restyle

* Update chip script: need to ensure build environment is set up

* Restyle

* Fix typo

* Fix cirque typo

* Switch bootstrap with activate

Author: andy31415

build/chip/chip_codegen.gni

@@ -17,6 +17,7 @@ import("//build_overrides/chip.gni")
 import("//build_overrides/pigweed.gni")
 
 import("$dir_pw_build/python.gni")
+import("${chip_root}/scripts/idl/files.gni")
 
 declare_args() {
   # Location where code has been pre-generated
@@ -37,6 +38,9 @@ template("_chip_build_time_codegen") {
   pw_python_action("${_name}_codegen") {
     script = "${chip_root}/scripts/codegen.py"
 
+    # TODO: this seems to touch internals. Is this ok? speeds up builds!
+    _pw_internal_run_in_venv = false
+
     _idl_file = invoker.input
     _expected_outputs = "${target_gen_dir}/${_name}.expected.outputs"
 
@@ -52,12 +56,14 @@ template("_chip_build_time_codegen") {
       rebase_path(_idl_file, root_build_dir),
     ]
 
-    deps = [ "${chip_root}/scripts/idl" ]
-
     inputs = [
       _idl_file,
       _expected_outputs,
     ]
+
+    # ensure any change in codegen files will result in a rebuild
+    inputs += matter_idl_generator_files
+
     sources = [ _idl_file ]
 
     outputs = []

scripts/build/gn_gen_cirque.sh

@@ -27,10 +27,16 @@ set -x
 
 env
 
-echo "Build: GN configure"
+cd "$ROOT_PATH"
 
-gn --root="$CHIP_ROOT" gen --check --fail-on-unused-args "$CHIP_ROOT/out/debug" --args='target_os="all"'"chip_build_tests=false chip_enable_wifi=false chip_im_force_fabric_quota_check=true enable_default_builds=false enable_host_gcc_build=true enable_standalone_chip_tool_build=true enable_linux_all_clusters_app_build=true enable_linux_lighting_app_build=true"
+echo "Ensure submodules for Linux builds are checked out"
+./scripts/checkout_submodules.py --shallow --platform linux
 
-echo "Build: Ninja build"
+echo "Setup build environment"
+source "./scripts/activate.sh"
 
-time ninja -C "$CHIP_ROOT/out/debug" all check
+echo "Build: GN configure"
+gn --root="$CHIP_ROOT" gen --check --fail-on-unused-args out/debug --args='target_os="all"'"chip_build_tests=false chip_enable_wifi=false chip_im_force_fabric_quota_check=true enable_default_builds=false enable_host_gcc_build=true enable_standalone_chip_tool_build=true enable_linux_all_clusters_app_build=true enable_linux_lighting_app_build=true"
+
+echo "Build: Ninja build"
+time ninja -C out/debug all check

scripts/idl/BUILD.gn

@@ -18,20 +18,15 @@ import("//build_overrides/chip.gni")
 import("//build_overrides/pigweed.gni")
 import("$dir_pw_build/python.gni")
 
+import("${chip_root}/scripts/idl/files.gni")
+
 pw_python_package("idl") {
   setup = [ "setup.py" ]
-  inputs = [
+  inputs = matter_idl_generator_templates
+  inputs += [
     # Dependency grammar
     "matter_grammar.lark",
 
-    # Templates used for generation
-    "generators/bridge/BridgeClustersCpp.jinja",
-    "generators/bridge/BridgeClustersCommon.jinja",
-    "generators/bridge/BridgeClustersGlobalStructs.jinja",
-    "generators/java/ChipClustersCpp.jinja",
-    "generators/java/ChipClustersRead.jinja",
-    "generators/cpp/application/PluginApplicationCallbacksHeader.jinja",
-
     # Unit test data
     "tests/available_tests.yaml",
     "tests/inputs/cluster_struct_attribute.matter",
@@ -70,25 +65,7 @@ pw_python_package("idl") {
     "tests/outputs/simple_attribute/jni/MyClusterClient-InvokeSubscribeImpl.cpp",
   ]
 
-  sources = [
-    "__init__.py",
-    "generators/__init__.py",
-    "generators/bridge/__init__.py",
-    "generators/cpp/__init__.py",
-    "generators/cpp/application/__init__.py",
-    "generators/filters.py",
-    "generators/java/__init__.py",
-    "generators/types.py",
-    "matter_idl_parser.py",
-    "matter_idl_types.py",
-    "xml_parser.py",
-    "zapxml/__init__.py",
-    "zapxml/handlers/__init__.py",
-    "zapxml/handlers/base.py",
-    "zapxml/handlers/context.py",
-    "zapxml/handlers/handlers.py",
-    "zapxml/handlers/parsing.py",
-  ]
+  sources = matter_idl_generator_sources
 
   tests = [
     "test_matter_idl_parser.py",

scripts/idl/files.gni

@@ -0,0 +1,37 @@
+import("//build_overrides/build.gni")
+import("//build_overrides/chip.gni")
+
+# Templates used for generation
+matter_idl_generator_templates = [
+  "${chip_root}/scripts/idl/generators/bridge/BridgeClustersCpp.jinja",
+  "${chip_root}/scripts/idl/generators/bridge/BridgeClustersCommon.jinja",
+  "${chip_root}/scripts/idl/generators/bridge/BridgeClustersGlobalStructs.jinja",
+  "${chip_root}/scripts/idl/generators/java/ChipClustersCpp.jinja",
+  "${chip_root}/scripts/idl/generators/java/ChipClustersRead.jinja",
+  "${chip_root}/scripts/idl/generators/cpp/application/CallbackStubSource.jinja",
+  "${chip_root}/scripts/idl/generators/cpp/application/PluginApplicationCallbacksHeader.jinja",
+]
+
+matter_idl_generator_sources = [
+  "${chip_root}/scripts/idl/__init__.py",
+  "${chip_root}/scripts/idl/generators/__init__.py",
+  "${chip_root}/scripts/idl/generators/bridge/__init__.py",
+  "${chip_root}/scripts/idl/generators/cpp/__init__.py",
+  "${chip_root}/scripts/idl/generators/cpp/application/__init__.py",
+  "${chip_root}/scripts/idl/generators/filters.py",
+  "${chip_root}/scripts/idl/generators/java/__init__.py",
+  "${chip_root}/scripts/idl/generators/types.py",
+  "${chip_root}/scripts/idl/matter_idl_parser.py",
+  "${chip_root}/scripts/idl/matter_idl_types.py",
+  "${chip_root}/scripts/idl/xml_parser.py",
+  "${chip_root}/scripts/idl/zapxml/__init__.py",
+  "${chip_root}/scripts/idl/zapxml/handlers/__init__.py",
+  "${chip_root}/scripts/idl/zapxml/handlers/base.py",
+  "${chip_root}/scripts/idl/zapxml/handlers/context.py",
+  "${chip_root}/scripts/idl/zapxml/handlers/handlers.py",
+  "${chip_root}/scripts/idl/zapxml/handlers/parsing.py",
+]
+
+# All the files that the matter idl infrastructure will use
+matter_idl_generator_files =
+    matter_idl_generator_templates + matter_idl_generator_sources