Adding chip-repl into runner script (#25900)

* Adding chip-repl into runner script

* Restyle

* Fix CI

* Redirect lower level logs away from stdout

* Address PR comment

.flake8

@@ -90,6 +90,7 @@ exclude = third_party
           src/controller/python/chip/yaml/__init__.py
           src/controller/python/chip/yaml/format_converter.py
           src/controller/python/chip/yaml/runner.py
+          src/controller/python/py_matter_yamltest_repl_adapter/matter_yamltest_repl_adapter/runner.py
           src/lib/asn1/gen_asn1oid.py
           src/pybindings/pycontroller/build-chip-wheel.py
           src/pybindings/pycontroller/pychip/__init__.py

scripts/py_matter_yamltests/matter_yamltests/runner.py

@@ -175,7 +175,9 @@ async def _run(self, parser: TestParser, config: TestRunnerConfig):
                 if config.pseudo_clusters.supports(request):
                     responses, logs = await config.pseudo_clusters.execute(request)
                 else:
-                    responses, logs = config.adapter.decode(await self.execute(config.adapter.encode(request)))
+                    encoded_request = config.adapter.encode(request)
+                    encoded_response = await self.execute(encoded_request)
+                    responses, logs = config.adapter.decode(encoded_response)
                 duration = round((time.time() - start) * 1000, 2)
                 test_duration += duration
 

scripts/tests/chiptest/yamltest_with_chip_repl_tester.py

@@ -14,6 +14,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
+import asyncio
 import atexit
 import logging
 import os
@@ -54,9 +55,27 @@
     os.path.join(_DEFAULT_CHIP_ROOT, "src/app/zap-templates/zcl/data-model/"))
 
 
-def StackShutdown():
-    certificateAuthorityManager.Shutdown()
-    builtins.chipStack.Shutdown()
+async def execute_test(yaml, runner):
+    # Executing and validating test
+    for test_step in yaml.tests:
+        if not test_step.is_pics_enabled:
+            continue
+        test_action = runner.encode(test_step)
+        if test_action is None:
+            raise Exception(
+                f'Failed to encode test step {test_step.label}')
+
+        response = await runner.execute(test_action)
+        decoded_response = runner.decode(response)
+        post_processing_result = test_step.post_process_response(
+            decoded_response)
+        if not post_processing_result.is_success():
+            logging.warning(f"Test step failure in 'test_step.label'")
+            for entry in post_processing_result.entries:
+                if entry.state == PostProcessCheckStatus.SUCCESS:
+                    continue
+                logging.warning("%s: %s", entry.state, entry.message)
+            raise Exception(f'Test step failed {test_step.label}')
 
 
 @click.command()
@@ -118,27 +137,8 @@ def _StackShutDown():
             runner = ReplTestRunner(
                 clusters_definitions, certificate_authority_manager, dev_ctrl)
 
-            # Executing and validating test
-            for test_step in yaml.tests:
-                if not test_step.is_pics_enabled:
-                    continue
-                test_action = runner.encode(test_step)
-                # TODO if test_action is None we should see if it is a pseudo cluster.
-                if test_action is None:
-                    raise Exception(
-                        f'Failed to encode test step {test_step.label}')
-
-                response = runner.execute(test_action)
-                decoded_response = runner.decode(response)
-                post_processing_result = test_step.post_process_response(
-                    decoded_response)
-                if not post_processing_result.is_success():
-                    logging.warning(f"Test step failure in 'test_step.label'")
-                    for entry in post_processing_result.entries:
-                        if entry.state == PostProcessCheckStatus.SUCCESS:
-                            continue
-                        logging.warning("%s: %s", entry.state, entry.message)
-                    raise Exception(f'Test step failed {test_step.label}')
+            asyncio.run(execute_test(yaml, runner))
+
         except Exception:
             print(traceback.format_exc())
             exit(-2)

scripts/tests/yaml/relative_importer.py

@@ -21,6 +21,7 @@
     os.path.join(os.path.dirname(__file__), '..', '..', '..'))
 SCRIPT_PATH = os.path.join(DEFAULT_CHIP_ROOT, 'scripts')
 EXAMPLES_PATH = os.path.join(DEFAULT_CHIP_ROOT, 'examples')
+REPL_PATH = os.path.join(DEFAULT_CHIP_ROOT, 'src', 'controller', 'python')
 
 try:
     import matter_idl  # noqa: F401
@@ -41,3 +42,8 @@
     import matter_placeholder_adapter  # noqa: F401
 except ModuleNotFoundError:
     sys.path.append(os.path.join(EXAMPLES_PATH, 'placeholder', 'py_matter_placeholder_adapter'))
+
+try:
+    import matter_yamltest_repl_adapter  # noqa: F401
+except ModuleNotFoundError:
+    sys.path.append(os.path.join(REPL_PATH, 'py_matter_yamltest_repl_adapter'))

scripts/tests/yaml/runner.py

@@ -88,6 +88,16 @@ def websocket_runner_options(f):
     return f
 
 
+def chip_repl_runner_options(f):
+    f = click.option('--repl_storage_path', type=str, default='/tmp/repl-storage.json',
+                     help='Path to persistent storage configuration file.')(f)
+    f = click.option('--commission_on_network_dut', type=bool, default=False,
+                     help='Prior to running test should we try to commission DUT on network.')(f)
+    f = click.option('--runner', type=str, default=None, show_default=True,
+                     help='The runner to run the test with.')(f)
+    return f
+
+
 @dataclass
 class ParserGroup:
     builder_config: TestParserBuilderConfig
@@ -202,6 +212,10 @@ def __add_custom_params(self, ctx):
             'server_name': 'chip-app2',
             'server_arguments': '--interactive',
         },
+        'chip-repl': {
+            'adapter': 'matter_yamltest_repl_adapter.adapter',
+            'runner': 'matter_yamltest_repl_adapter.runner',
+        },
     },
     max_content_width=120,
 )
@@ -281,6 +295,21 @@ def websocket(parser_group: ParserGroup, adapter: str, stop_on_error: bool, stop
     return runner.run(parser_group.builder_config, runner_config)
 
 
+@runner_base.command()
+@test_runner_options
+@chip_repl_runner_options
+@pass_parser_group
+def chip_repl(parser_group: ParserGroup, adapter: str, stop_on_error: bool, stop_on_warning: bool, stop_at_number: int, show_adapter_logs: bool, show_adapter_logs_on_error: bool, runner: str, repl_storage_path: str, commission_on_network_dut: bool):
+    """Run the test suite using chip-repl."""
+    adapter = __import__(adapter, fromlist=[None]).Adapter(parser_group.builder_config.parser_config.definitions)
+    runner_options = TestRunnerOptions(stop_on_error, stop_on_warning, stop_at_number)
+    runner_hooks = TestRunnerLogger(show_adapter_logs, show_adapter_logs_on_error)
+    runner_config = TestRunnerConfig(adapter, parser_group.pseudo_clusters, runner_options, runner_hooks)
+
+    runner = __import__(runner, fromlist=[None]).Runner(repl_storage_path, commission_on_network_dut)
+    return runner.run(parser_group.builder_config, runner_config)
+
+
 @runner_base.command()
 @test_runner_options
 @websocket_runner_options

src/controller/python/chip/yaml/runner.py

@@ -15,7 +15,6 @@
 #    limitations under the License.
 #
 
-import asyncio as asyncio
 import logging
 import queue
 from abc import ABC, abstractmethod
@@ -116,7 +115,7 @@ def pics_enabled(self):
         return self._pics_enabled
 
     @abstractmethod
-    def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
+    async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
         pass
 
 
@@ -127,8 +126,8 @@ def __init__(self, test_step):
         if not _PSEUDO_CLUSTERS.supports(test_step):
             raise ActionCreationError(f'Default cluster {test_step.cluster} {test_step.command}, not supported')
 
-    def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
-        resp = asyncio.run(_PSEUDO_CLUSTERS.execute(self._test_step))
+    async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
+        resp = await _PSEUDO_CLUSTERS.execute(self._test_step)
         return _ActionResult(status=_ActionStatus.SUCCESS, response=None)
 
 
@@ -186,17 +185,17 @@ def __init__(self, test_step, cluster: str, context: _ExecutionContext):
         else:
             self._request_object = command_object
 
-    def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
+    async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
         try:
             if self._group_id:
                 resp = dev_ctrl.SendGroupCommand(
                     self._group_id, self._request_object,
                     busyWaitMs=self._busy_wait_ms)
             else:
-                resp = asyncio.run(dev_ctrl.SendCommand(
+                resp = await dev_ctrl.SendCommand(
                     self._node_id, self._endpoint, self._request_object,
                     timedRequestTimeoutMs=self._interation_timeout_ms,
-                    busyWaitMs=self._busy_wait_ms))
+                    busyWaitMs=self._busy_wait_ms)
         except chip.interaction_model.InteractionModelError as error:
             return _ActionResult(status=_ActionStatus.ERROR, response=error)
 
@@ -249,11 +248,11 @@ def __init__(self, test_step, cluster: str, context: _ExecutionContext):
             raise UnexpectedActionCreationError(
                 f'ReadAttribute doesnt have valid attribute_type. {self.label}')
 
-    def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
+    async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
         try:
-            raw_resp = asyncio.run(dev_ctrl.ReadAttribute(self._node_id,
-                                                          [(self._endpoint, self._request_object)],
-                                                          fabricFiltered=self._fabric_filtered))
+            raw_resp = await dev_ctrl.ReadAttribute(self._node_id,
+                                                    [(self._endpoint, self._request_object)],
+                                                    fabricFiltered=self._fabric_filtered)
         except chip.interaction_model.InteractionModelError as error:
             return _ActionResult(status=_ActionStatus.ERROR, response=error)
         except ChipStackError as error:
@@ -317,12 +316,12 @@ def __init__(self, test_step, cluster: str, context: _ExecutionContext):
             raise UnexpectedActionCreationError(
                 f'ReadEvent should not contain arguments. {self.label}')
 
-    def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
+    async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
         try:
             urgent = 0
             request = [(self._endpoint, self._request_object, urgent)]
-            resp = asyncio.run(dev_ctrl.ReadEvent(self._node_id, events=request, eventNumberFilter=self._event_number_filter,
-                                                  fabricFiltered=self._fabric_filtered))
+            resp = await dev_ctrl.ReadEvent(self._node_id, events=request, eventNumberFilter=self._event_number_filter,
+                                            fabricFiltered=self._fabric_filtered)
         except chip.interaction_model.InteractionModelError as error:
             return _ActionResult(status=_ActionStatus.ERROR, response=error)
 
@@ -358,7 +357,7 @@ def __init__(self, test_step):
             # Timeout is provided in seconds we need to conver to milliseconds.
             self._timeout_ms = request_data_as_dict['timeout'] * 1000
 
-    def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
+    async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
         try:
             if self._expire_existing_session:
                 dev_ctrl.ExpireSessions(self._node_id)
@@ -437,12 +436,11 @@ def __init__(self, test_step, cluster: str, context: _ExecutionContext):
                 f'SubscribeAttribute action does not have max_interval {self.label}')
         self._max_interval = test_step.max_interval
 
-    def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
+    async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
         try:
-            subscription = asyncio.run(
-                dev_ctrl.ReadAttribute(self._node_id, [(self._endpoint, self._request_object)],
-                                       reportInterval=(self._min_interval, self._max_interval),
-                                       keepSubscriptions=False))
+            subscription = await dev_ctrl.ReadAttribute(self._node_id, [(self._endpoint, self._request_object)],
+                                                        reportInterval=(self._min_interval, self._max_interval),
+                                                        keepSubscriptions=False)
         except chip.interaction_model.InteractionModelError as error:
             return _ActionResult(status=_ActionStatus.ERROR, response=error)
 
@@ -490,14 +488,13 @@ def __init__(self, test_step, cluster: str, context: _ExecutionContext):
                 f'SubscribeEvent action does not have max_interval {self.label}')
         self._max_interval = test_step.max_interval
 
-    def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
+    async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
         try:
             urgent = 0
             request = [(self._endpoint, self._request_object, urgent)]
-            subscription = asyncio.run(
-                dev_ctrl.ReadEvent(self._node_id, events=request, eventNumberFilter=self._event_number_filter,
-                                   reportInterval=(self._min_interval, self._max_interval),
-                                   keepSubscriptions=False))
+            subscription = await dev_ctrl.ReadEvent(self._node_id, events=request, eventNumberFilter=self._event_number_filter,
+                                                    reportInterval=(self._min_interval, self._max_interval),
+                                                    keepSubscriptions=False)
         except chip.interaction_model.InteractionModelError as error:
             return _ActionResult(status=_ActionStatus.ERROR, response=error)
 
@@ -571,16 +568,15 @@ def __init__(self, test_step, cluster: str, context: _ExecutionContext):
         # Create a cluster object for the request from the provided YAML data.
         self._request_object = attribute(request_data)
 
-    def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
+    async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
         try:
             if self._group_id:
                 resp = dev_ctrl.WriteGroupAttribute(self._group_id, [(self._request_object,)],
                                                     busyWaitMs=self._busy_wait_ms)
             else:
-                resp = asyncio.run(
-                    dev_ctrl.WriteAttribute(self._node_id, [(self._endpoint, self._request_object)],
-                                            timedRequestTimeoutMs=self._interation_timeout_ms,
-                                            busyWaitMs=self._busy_wait_ms))
+                resp = await dev_ctrl.WriteAttribute(self._node_id, [(self._endpoint, self._request_object)],
+                                                     timedRequestTimeoutMs=self._interation_timeout_ms,
+                                                     busyWaitMs=self._busy_wait_ms)
         except chip.interaction_model.InteractionModelError as error:
             return _ActionResult(status=_ActionStatus.ERROR, response=error)
 
@@ -624,7 +620,7 @@ def __init__(self, test_step, context: _ExecutionContext):
         if self._output_queue is None:
             raise UnexpectedActionCreationError(f'Could not find output queue')
 
-    def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
+    async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
         try:
             # While there should be a timeout here provided by the test, the current codegen version
             # of YAML tests doesn't have a per test step timeout, only a global timeout for the
@@ -662,7 +658,7 @@ def __init__(self, test_step):
         else:
             raise UnexpectedActionCreationError(f'Unexpected CommisionerCommand {test_step.command}')
 
-    def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
+    async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
         if self._command == 'GetCommissionerNodeId':
             return _ActionResult(status=_ActionStatus.SUCCESS, response=_GetCommissionerNodeIdResult(dev_ctrl.nodeId))
 
@@ -713,7 +709,7 @@ def __init__(self, test_step):
         super().__init__(test_step)
         self.filterType, self.filter = DiscoveryCommandAction._filter_for_step(test_step)
 
-    def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
+    async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
         devices = dev_ctrl.DiscoverCommissionableNodes(
             filterType=self.filterType, filter=self.filter, stopOnFirst=True, timeoutSecond=5)
 
@@ -737,7 +733,7 @@ def __init__(self, test_step, cluster, command):
         self.cluster = cluster
         self.command = command
 
-    def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
+    async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
         raise Exception(f"NOT YET IMPLEMENTED: {self.cluster}::{self.command}")
 
 
@@ -753,7 +749,8 @@ def __init__(self, test_spec_definition, certificate_authority_manager, alpha_de
         self._certificate_authority_manager = certificate_authority_manager
         self._dev_ctrls = {}
 
-        alpha_dev_ctrl.InitGroupTestingData()
+        if alpha_dev_ctrl is not None:
+            alpha_dev_ctrl.InitGroupTestingData()
         self._dev_ctrls['alpha'] = alpha_dev_ctrl
 
     def _invoke_action_factory(self, test_step, cluster: str):
@@ -1012,9 +1009,9 @@ def _get_dev_ctrl(self, action: BaseAction):
 
         return dev_ctrl
 
-    def execute(self, action: BaseAction):
+    async def execute(self, action: BaseAction):
         dev_ctrl = self._get_dev_ctrl(action)
-        return action.run_action(dev_ctrl)
+        return await action.run_action(dev_ctrl)
 
     def shutdown(self):
         for subscription in self._context.subscriptions:

src/controller/python/py_matter_yamltest_repl_adapter/matter_yamltest_repl_adapter/adapter.py

@@ -0,0 +1,33 @@
+# Copyright (c) 2023 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from chip.yaml.runner import ReplTestRunner
+from matter_yamltests.adapter import TestAdapter
+
+
+class Adapter(TestAdapter):
+    def __init__(self, specifications):
+        self._adapter = ReplTestRunner(specifications, None, None)
+
+    def encode(self, request):
+        return self._adapter.encode(request)
+
+    def decode(self, response):
+        # TODO We should provide more meaningful logs here, but to adhere to
+        # abstract function definition we do need to return list here.
+        logs = []
+        decoded_response = self._adapter.decode(response)
+        if len(decoded_response) == 0:
+            decoded_response = [{}]
+        return decoded_response, logs