python testing: Per node endpoint fixes (#34819)

* Change name of decorator * Add a force option for per endpoint tests * fix test * Add warning on force endpoint * Disallow --endpoint with automatic endpoint selection * Restyled by isort * merge conflict fix * Don't automatically override endpoint in MockRunner * Restyled by isort * Fix endpoint default on command line * Going the other way on this With the way the ATLs expect the test files currently, this is causing issues with submission. The ATLs aren't familiar with how to read these files and it's making cert challenging. Instead, we're going to gate or skip tests PER ENDPOINT such that the --endpoint flag can be set. This also (in theory) lets the tests run alongside other tests, where the --endpoint is specified. The auto-runner was too much, too soon. It's great for CI, but we don't have TH features to select these tests, nor ways to allow the file submission. Let's fall back to once per endpoint. It's still better, we're not relying on PICS. * Update comments. * Update all tests * more things * fix singleton test * Restyled by isort * fix var * Update src/python_testing/matter_testing_support.py Co-authored-by: Boris Zbarsky <[email protected]> * Restyled by isort * add endpoint flags --------- Co-authored-by: Restyled.io <[email protected]> Co-authored-by: Boris Zbarsky <[email protected]>
project-chip · Sep 11, 2024 · 1129266 · 1129266
1 parent dcc5ede
commit 1129266
Show file tree

Hide file tree

Showing 11 changed files with 269 additions and 205 deletions.
diff --git a/src/python_testing/TC_CCTRL_2_1.py b/src/python_testing/TC_CCTRL_2_1.py
@@ -16,7 +16,7 @@
 #
 
 import chip.clusters as Clusters
-from matter_testing_support import MatterBaseTest, TestStep, default_matter_test_main, has_cluster, per_endpoint_test
+from matter_testing_support import MatterBaseTest, TestStep, default_matter_test_main, has_cluster, run_if_endpoint_matches
 from mobly import asserts
 
 
@@ -27,7 +27,7 @@ def steps_TC_CCTRL_2_1(self) -> list[TestStep]:
                  TestStep(2, "Validate SupportedDeviceCategories is set accordingly based on MCORE.FS")]
         return steps
 
-    @per_endpoint_test(has_cluster(Clusters.CommissionerControl))
+    @run_if_endpoint_matches(has_cluster(Clusters.CommissionerControl))
     async def test_TC_CCTRL_2_1(self):
         self.step(1)
         is_fabric_sync_pics_enabled = self.check_pics("MCORE.FS")

diff --git a/src/python_testing/TC_CCTRL_2_2.py b/src/python_testing/TC_CCTRL_2_2.py
@@ -34,7 +34,7 @@
 from chip import ChipDeviceCtrl
 from chip.interaction_model import InteractionModelError, Status
 from matter_testing_support import (MatterBaseTest, TestStep, async_test_body, default_matter_test_main, has_cluster,
-                                    per_endpoint_test)
+                                    run_if_endpoint_matches)
 from mobly import asserts
 
 
@@ -119,7 +119,7 @@ def steps_TC_CCTRL_2_2(self) -> list[TestStep]:
 
         return steps
 
-    @per_endpoint_test(has_cluster(Clusters.CommissionerControl))
+    @run_if_endpoint_matches(has_cluster(Clusters.CommissionerControl))
     async def test_TC_CCTRL_2_2(self):
         self.is_ci = self.check_pics('PICS_SDK_CI_ONLY')
 

diff --git a/src/python_testing/TC_CC_2_2.py b/src/python_testing/TC_CC_2_2.py
@@ -24,7 +24,7 @@
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
-# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --endpoint 1 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
 # === END CI TEST ARGUMENTS ===
 
 import logging
@@ -33,7 +33,7 @@
 import chip.clusters as Clusters
 from chip.clusters import ClusterObjects as ClusterObjects
 from matter_testing_support import (ClusterAttributeChangeAccumulator, MatterBaseTest, TestStep, default_matter_test_main,
-                                    has_cluster, per_endpoint_test)
+                                    has_cluster, run_if_endpoint_matches)
 from mobly import asserts
 from test_plan_support import commission_if_required, if_feature_supported, read_attribute, verify_success
 
@@ -107,7 +107,7 @@ def entry_count_verification() -> str:
                          "The third entry in _reportedRemainingTimeValuesList_ is equal to 0")
                 ]
 
-    @per_endpoint_test(has_cluster(Clusters.ColorControl))
+    @run_if_endpoint_matches(has_cluster(Clusters.ColorControl))
     async def test_TC_CC_2_2(self):
         gather_time = 20
 

diff --git a/src/python_testing/TC_LVL_2_3.py b/src/python_testing/TC_LVL_2_3.py
@@ -24,7 +24,7 @@
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
-# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --endpoint 1 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
 # === END CI TEST ARGUMENTS ===
 
 import logging
@@ -33,7 +33,7 @@
 import chip.clusters as Clusters
 import test_plan_support
 from matter_testing_support import (ClusterAttributeChangeAccumulator, MatterBaseTest, TestStep, default_matter_test_main,
-                                    has_cluster, per_endpoint_test)
+                                    has_cluster, run_if_endpoint_matches)
 from mobly import asserts
 
 
@@ -84,7 +84,7 @@ def steps_TC_LVL_2_3(self) -> list[TestStep]:
                          "The third entry in reportedRemainingTimeValuesList is equal to 0")
                 ]
 
-    @per_endpoint_test(has_cluster(Clusters.LevelControl))
+    @run_if_endpoint_matches(has_cluster(Clusters.LevelControl))
     async def test_TC_LVL_2_3(self):
         # Commissioning - already done
         self.step(1)

diff --git a/src/python_testing/TC_SWTCH.py b/src/python_testing/TC_SWTCH.py
@@ -18,13 +18,35 @@
 # for details about the block below.
 #
 # === BEGIN CI TEST ARGUMENTS ===
-# test-runner-runs: run1
+# test-runner-runs: run1 run2 run3 run4
+#
 # test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
-# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto --PICS src/app/tests/suites/certification/ci-pics-values
+# test-runner-run/run1/script-args: --endpoint 1 --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto --PICS src/app/tests/suites/certification/ci-pics-values
+#
+# test-runner-run/run2/app: ${ALL_CLUSTERS_APP}
+# test-runner-run/run2/factoryreset: True
+# test-runner-run/run2/quiet: True
+# test-runner-run/run2/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
+# test-runner-run/run2/script-args: --endpoint 2 --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto --PICS src/app/tests/suites/certification/ci-pics-values
+#
+# test-runner-run/run3/app: ${ALL_CLUSTERS_APP}
+# test-runner-run/run3/factoryreset: True
+# test-runner-run/run3/quiet: True
+# test-runner-run/run3/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
+# test-runner-run/run3/script-args: --endpoint 3 --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto --PICS src/app/tests/suites/certification/ci-pics-values
+#
+# test-runner-run/run4/app: ${ALL_CLUSTERS_APP}
+# test-runner-run/run4/factoryreset: True
+# test-runner-run/run4/quiet: True
+# test-runner-run/run4/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
+# test-runner-run/run4/script-args: --endpoint 4 --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto --PICS src/app/tests/suites/certification/ci-pics-values
+#
 # === END CI TEST ARGUMENTS ===
+#
+# These tests run on every endpoint regardless of whether a switch is present because they are set up to auto-select.
 
 import json
 import logging
@@ -39,7 +61,8 @@
 from chip.clusters.Attribute import EventReadResult
 from chip.tlv import uint
 from matter_testing_support import (AttributeValue, ClusterAttributeChangeAccumulator, EventChangeCallback, MatterBaseTest,
-                                    TestStep, await_sequence_of_reports, default_matter_test_main, has_feature, per_endpoint_test)
+                                    TestStep, await_sequence_of_reports, default_matter_test_main, has_feature,
+                                    run_if_endpoint_matches)
 from mobly import asserts
 
 logger = logging.getLogger(__name__)
@@ -277,7 +300,7 @@ def steps_TC_SWTCH_2_2(self):
                          "Verify that the value is 0, and that a subscription report was received for that change."),
                 ]
 
-    @per_endpoint_test(has_feature(Clusters.Switch, Clusters.Switch.Bitmaps.Feature.kLatchingSwitch))
+    @run_if_endpoint_matches(has_feature(Clusters.Switch, Clusters.Switch.Bitmaps.Feature.kLatchingSwitch))
     async def test_TC_SWTCH_2_2(self):
         post_prompt_settle_delay_seconds = 10.0
         cluster = Clusters.Switch
@@ -387,7 +410,7 @@ def steps_TC_SWTCH_2_3(self):
                 TestStep(9, "TH reads the CurrentPosition attribute from the DUT", "Verify that the value is 0"),
                 ]
 
-    @per_endpoint_test(has_feature(Clusters.Switch, Clusters.Switch.Bitmaps.Feature.kMomentarySwitch))
+    @run_if_endpoint_matches(has_feature(Clusters.Switch, Clusters.Switch.Bitmaps.Feature.kMomentarySwitch))
     async def test_TC_SWTCH_2_3(self):
         # Commissioning - already done
         self.step(1)
@@ -465,7 +488,7 @@ def steps_TC_SWTCH_2_4(self):
                 """)
                 ]
 
-    @per_endpoint_test(has_feature(Clusters.Switch, Clusters.Switch.Bitmaps.Feature.kMomentarySwitch))
+    @run_if_endpoint_matches(has_feature(Clusters.Switch, Clusters.Switch.Bitmaps.Feature.kMomentarySwitch))
     async def test_TC_SWTCH_2_4(self):
         switch_pressed_position = self._default_pressed_position
         post_prompt_settle_delay_seconds = 10.0
@@ -639,7 +662,7 @@ def should_run_SWTCH_2_5(wildcard, endpoint):
         asf = has_feature(Clusters.Switch, Clusters.Switch.Bitmaps.Feature.kActionSwitch)
         return msm(wildcard, endpoint) and not asf(wildcard, endpoint)
 
-    @per_endpoint_test(should_run_SWTCH_2_5)
+    @run_if_endpoint_matches(should_run_SWTCH_2_5)
     async def test_TC_SWTCH_2_5(self):
         # Commissioning - already done
         self.step(1)
@@ -818,7 +841,7 @@ def should_run_SWTCH_2_6(wildcard, endpoint):
         asf = has_feature(Clusters.Switch, 0x20)
         return msm(wildcard, endpoint) and asf(wildcard, endpoint)
 
-    @per_endpoint_test(should_run_SWTCH_2_6)
+    @run_if_endpoint_matches(should_run_SWTCH_2_6)
     async def test_TC_SWTCH_2_6(self):
         # Commissioning - already done
         self.step(1)

diff --git a/src/python_testing/TC_TIMESYNC_2_1.py b/src/python_testing/TC_TIMESYNC_2_1.py
@@ -24,15 +24,15 @@
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
-# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# test-runner-run/run1/script-args: --endpoint 0 --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
 # === END CI TEST ARGUMENTS ===
 
 import ipaddress
 from datetime import timedelta
 
 import chip.clusters as Clusters
 from chip.clusters.Types import NullValue
-from matter_testing_support import (MatterBaseTest, default_matter_test_main, has_attribute, has_cluster, per_endpoint_test,
+from matter_testing_support import (MatterBaseTest, default_matter_test_main, has_attribute, has_cluster, run_if_endpoint_matches,
                                     utc_time_in_matter_epoch)
 from mobly import asserts
 
@@ -42,7 +42,7 @@ async def read_ts_attribute_expect_success(self, attribute):
         cluster = Clusters.Objects.TimeSynchronization
         return await self.read_single_attribute_check_success(endpoint=None, cluster=cluster, attribute=attribute)
 
-    @per_endpoint_test(has_cluster(Clusters.TimeSynchronization) and has_attribute(Clusters.TimeSynchronization.Attributes.TimeSource))
+    @run_if_endpoint_matches(has_cluster(Clusters.TimeSynchronization) and has_attribute(Clusters.TimeSynchronization.Attributes.TimeSource))
     async def test_TC_TIMESYNC_2_1(self):
         attributes = Clusters.TimeSynchronization.Attributes
         features = await self.read_ts_attribute_expect_success(attribute=attributes.FeatureMap)