Allow code pre-generation to accept files outside of chip-root (#25970)

* Start supporting separate root pregeneration

* Update documentation

* Restyle

* Fix typos

* Fix pregen logic

* Update path handling to support full and relative paths, codepregen of external roots now works

* Restyle

* Update docs/code_generation.md

Co-authored-by: Boris Zbarsky <[email protected]>

* Update docs/code_generation.md

Co-authored-by: Boris Zbarsky <[email protected]>

* Update docs/code_generation.md

Co-authored-by: Boris Zbarsky <[email protected]>

* Slight rephrase

* Another rephrase

* Restyle

* Normalize paths to not be off by one on generation path lenghts

---------

Co-authored-by: Andrei Litvin <[email protected]>
Co-authored-by: Boris Zbarsky <[email protected]>

Author: 3 people

docs/code_generation.md

@@ -234,9 +234,48 @@ scripts/codepregen.py ${OUTPUT_DIRECTORY:-./zzz_pregenerated/}
 
 # To generate a single output you can use `--input-glob`:
 
-scripts/codepregen.py --input-glob "*all-clusters*" ${OUTPUT_DIRECTORY:-./zzz_pregenerated/}
+scripts/codepregen.py --input-glob "*all-clusters*" --input-glob "*controller*" ${OUTPUT_DIRECTORY:-./zzz_pregenerated/}
 ```
 
+### External applications/zap files
+
+#### Ensure you have a `.matter` file
+
+Code generation generally will use both `.zap` or `.matter` files. If you only
+have a `.zap` file, you can create the corresponding `.matter` file via:
+
+```bash
+scripts/tools/zap/generate.py ${ZAP_FILE_PATH}
+```
+
+The above will use the template `src/app/zap-templates/matter-idl.json` to
+generate a `.matter` file corresponding to the input `.zap` file.
+
+`.matter` files are designed to be human readable. It is recommended to take a
+look at the generated file and see if it contains what is expected and also lint
+it. If anything seems wrong, the `.zap` file should be fixed (`.matter`
+represents the content of `.zap`). To lint use:
+
+```bash
+scripts/idl_lint.py ${MATTER_FILE_PATH}
+```
+
+#### Running pre-generation
+
+If you have zap files outside the CHIP repository (i.e. not in `src` or
+`examples`) you should provide the root of your application source.
+
+```bash
+scripts/codepregen.py --external-root ${PATH_TO_SOURCE_ROOT} ${OUTPUT_DIRECTORY:-./zzz_pregenerated/}
+```
+
+NOTE: `$PATH_TO_SOURCE_ROOT` should be a top-level directory containing
+zap/matter files as the code pre-generation will generate files based on the
+path inside the root:
+
+-   if files are `$PATH_TO_SOURCE_ROOT/some/path/foo.zap` this will generate
+    files into `$OUTPUT_DIRECTORY/some/path/foo/...`
+
 ### Using pre-generated code
 
 Instead of generating code at compile time, the chip build system accepts usage

scripts/codepregen.py

@@ -85,8 +85,13 @@ def _ParallelGenerateOne(arg):
     '--sdk-root',
     default=None,
     help='Path to the SDK root (where .zap/.matter files exist).')
+@click.option(
+    '--external-root',
+    default=None,
+    multiple=True,
+    help='Path to an external app root (where .zap/.matter files exist).')
 @click.argument('output_dir')
-def main(log_level, parallel, dry_run, generator, input_glob, sdk_root, output_dir):
+def main(log_level, parallel, dry_run, generator, input_glob, sdk_root, external_root, output_dir):
     if _has_coloredlogs:
         coloredlogs.install(level=__LOG_LEVELS__[
                             log_level], fmt='%(asctime)s %(levelname)-7s %(message)s')
@@ -122,7 +127,7 @@ def main(log_level, parallel, dry_run, generator, input_glob, sdk_root, output_d
     elif generator == 'codegen':
         filter.file_type = IdlFileType.MATTER
 
-    targets = FindPregenerationTargets(sdk_root, filter, runner)
+    targets = FindPregenerationTargets(sdk_root, external_root, filter, runner)
 
     runner.ensure_directory_exists(output_dir)
     if parallel:

scripts/pregenerate/__init__.py

@@ -25,7 +25,20 @@
 from .using_zap import ZapApplicationPregenerator
 
 
-def FindAllIdls(sdk_root: str) -> Iterator[InputIdlFile]:
+def _IdlsInDirectory(top_directory_name: str, truncate_length: int):
+    for root, dirs, files in os.walk(top_directory_name):
+        for file in files:
+            if file.endswith('.zap'):
+                yield InputIdlFile(file_type=IdlFileType.ZAP,
+                                   full_path=os.path.join(root, file),
+                                   relative_path=os.path.join(root[truncate_length:], file))
+            if file.endswith('.matter'):
+                yield InputIdlFile(file_type=IdlFileType.MATTER,
+                                   full_path=os.path.join(root, file),
+                                   relative_path=os.path.join(root[truncate_length:], file))
+
+
+def _FindAllIdls(sdk_root: str, external_roots: Optional[List[str]]) -> Iterator[InputIdlFile]:
     relevant_subdirs = [
         'examples',  # all example apps
         'src',      # realistically only controller/data_model
@@ -35,17 +48,19 @@ def FindAllIdls(sdk_root: str) -> Iterator[InputIdlFile]:
         sdk_root = sdk_root[:-1]
     sdk_root_length = len(sdk_root)
 
+    # first go over SDK items
     for subdir_name in relevant_subdirs:
         top_directory_name = os.path.join(sdk_root, subdir_name)
         logging.debug(f"Searching {top_directory_name}")
-        for root, dirs, files in os.walk(top_directory_name):
-            for file in files:
-                if file.endswith('.zap'):
-                    yield InputIdlFile(file_type=IdlFileType.ZAP,
-                                       relative_path=os.path.join(root[sdk_root_length+1:], file))
-                if file.endswith('.matter'):
-                    yield InputIdlFile(file_type=IdlFileType.MATTER,
-                                       relative_path=os.path.join(root[sdk_root_length+1:], file))
+        for idl in _IdlsInDirectory(top_directory_name, sdk_root_length+1):
+            yield idl
+
+    # next external roots
+    if external_roots:
+        for root in external_roots:
+            root = os.path.normpath(root)
+            for idl in _IdlsInDirectory(root, len(root) + 1):
+                yield idl
 
 
 @dataclass
@@ -67,7 +82,7 @@ def matches(self, s: str):
         return fnmatch.fnmatch(s, self.pattern)
 
 
-def FindPregenerationTargets(sdk_root: str, filter: TargetFilter, runner):
+def FindPregenerationTargets(sdk_root: str, external_roots: Optional[List[str]], filter: TargetFilter, runner):
     """Finds all relevand pre-generation targets in the given
        SDK root.
 
@@ -88,7 +103,7 @@ def FindPregenerationTargets(sdk_root: str, filter: TargetFilter, runner):
 
     path_matchers = [GlobMatcher(pattern) for pattern in filter.path_glob]
 
-    for idl in FindAllIdls(sdk_root):
+    for idl in _FindAllIdls(sdk_root, external_roots):
         if filter.file_type is not None:
             if idl.file_type != filter.file_type:
                 logging.debug(f"Will not process file of type {idl.file_type}: {idl.relative_path}")

scripts/pregenerate/types.py

@@ -26,6 +26,7 @@ class IdlFileType(Enum):
 class InputIdlFile:
     file_type: IdlFileType
     relative_path: str
+    full_path: str
 
     @property
     def pregen_subdir(self):

scripts/pregenerate/using_codegen.py

@@ -44,14 +44,14 @@ def Generate(self, output_root: str):
             output_root, self.idl.pregen_subdir, self.generator)
 
         logging.info(
-            f"Generating: {self.generator}:{self.idl.relative_path} into {output_dir}")
+            f"Generating: {self.generator}:{self.idl.full_path} into {output_dir}")
 
         cmd = [
             CODEGEN_PY_PATH,
             '--log-level', 'fatal',
             '--generator', self.generator,
             '--output-dir', output_dir,
-            os.path.join(self.sdk_root, self.idl.relative_path)
+            self.idl.full_path
         ]
 
         logging.debug(f"Executing {cmd}")

scripts/pregenerate/using_zap.py

@@ -56,16 +56,21 @@ def Generate(self, output_root: str):
 
         output_dir = os.path.join(output_root, self.idl.pregen_subdir, self.generation_type.subdir)
 
-        logging.info(f"Generating: {self.generation_type}:{self.idl.relative_path} into {output_dir}")
+        logging.info(f"Generating: {self.generation_type}:{self.idl.full_path} into {output_dir}")
 
         self.runner.ensure_directory_exists(output_dir)
 
+        if self.idl.full_path.startswith(self.sdk_root):
+            idl_path = self.idl.relative_path
+        else:
+            idl_path = self.idl.full_path
+
         cmd = [
             ZAP_GENERATE_PATH,
             '--templates', self.generation_type.generation_template,
             '--output-dir', output_dir,
             '--parallel',
-            self.idl.relative_path
+            idl_path
         ]
         logging.debug(f"Executing {cmd}")
         self.runner.run(cmd, cwd=self.sdk_root)