From f1704b8837a3f3450059ae717ed2c2bb1e47b2ff Mon Sep 17 00:00:00 2001 From: Cecille Freeman Date: Tue, 7 Nov 2023 23:33:17 -0500 Subject: [PATCH] Fail tests on error, add more in progress --- src/python_testing/TC_DeviceConformance.py | 33 ++++++++++++++++------ 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/src/python_testing/TC_DeviceConformance.py b/src/python_testing/TC_DeviceConformance.py index 3a05f816b1dc3e..abcab99b608f55 100644 --- a/src/python_testing/TC_DeviceConformance.py +++ b/src/python_testing/TC_DeviceConformance.py @@ -56,10 +56,16 @@ def conformance_str(conformance: Callable, feature_map: uint, feature_dict: dict is_ci = self.check_pics('PICS_SDK_CI_ONLY') ignore_attributes: dict[int, list[int]] = {} + ignore_features: dict[int, list[int]] = {} if ignore_in_progress: # This is a manually curated list of attributes that are in-progress in the SDK, but have landed in the spec - in_progress_attributes = {Clusters.BasicInformation.id: [0x15, 0x016]} + in_progress_attributes = {Clusters.BasicInformation.id: [0x15, 0x016], + Clusters.PowerSource.id: [0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A]} ignore_attributes.update(in_progress_attributes) + # The spec currently has an error on the power source features + # This should be removed once https://github.com/CHIP-Specifications/connectedhomeip-spec/pull/7823 lands + in_progress_features = {Clusters.PowerSource.id: [(1 << 2), (1 << 3), (1 << 4), (1 << 5)]} + ignore_features.update(in_progress_features) if is_ci: # The network commissioning clusters on the CI select the features on the fly and end up non-conformant @@ -96,6 +102,8 @@ def conformance_str(conformance: Callable, feature_map: uint, feature_dict: dict self.record_error(self.get_test_name(), location=location, problem=f'Unknown feature with mask 0x{f:02x}') success = False continue + if cluster_id in ignore_features and feature_mask in ignore_features[cluster_id]: + continue xml_feature = self.xml_clusters[cluster_id].features[f] conformance_decision = xml_feature.conformance(feature_map, attribute_list, all_command_list) if not conformance_allowed(conformance_decision, allow_provisional): @@ -103,6 +111,8 @@ def conformance_str(conformance: Callable, feature_map: uint, feature_dict: dict problem=f'Disallowed feature with mask 0x{f:02x}') success = False for feature_mask, xml_feature in self.xml_clusters[cluster_id].features.items(): + if cluster_id in ignore_features and feature_mask in ignore_features[cluster_id]: + continue conformance_decision = xml_feature.conformance(feature_map, attribute_list, all_command_list) if conformance_decision == ConformanceDecision.MANDATORY and feature_mask not in feature_masks: self.record_error(self.get_test_name(), location=location, @@ -179,13 +189,19 @@ def check_spec_conformance_for_commands(command_type: CommandType) -> bool: # TODO: Add choice checkers if not success: - # TODO: Right now, we have failures in all-cluster, so we can't fail this test and keep it in CI. For now, just log. - # Issue tracking: #29812 - # self.fail_current_test("Problems with conformance") - logging.error("Problems found with conformance, this should turn into a test failure once #29812 is resolved") + self.fail_current_test("Problems with conformance") def test_IDM_10_3(self): success = True + + ignore_in_progress = self.user_params.get("ignore_in_progress", False) + + ignore_revisions: list[int] = [] + if ignore_in_progress: + # This is a manually curated list of cluster revisions that are in-progress in the SDK, but have landed in the spec + in_progress_revisions = [Clusters.BasicInformation.id, Clusters.PowerSource.id, Clusters.NetworkCommissioning.id] + ignore_revisions.extend(in_progress_revisions) + for endpoint_id, endpoint in self.endpoints_tlv.items(): for cluster_id, cluster in endpoint.items(): if cluster_id not in self.xml_clusters.keys(): @@ -197,6 +213,8 @@ def test_IDM_10_3(self): self.record_warning(self.get_test_name(), location=location, problem='Standard cluster found on device, but is not present in spec data') continue + if cluster_id in ignore_revisions: + continue if int(self.xml_clusters[cluster_id].revision) != cluster[GlobalAttributeIds.CLUSTER_REVISION_ID]: location = AttributePathLocation(endpoint_id=endpoint_id, cluster_id=cluster_id, attribute_id=GlobalAttributeIds.CLUSTER_REVISION_ID) @@ -204,10 +222,7 @@ def test_IDM_10_3(self): ), location=location, problem=f'Revision found on cluster ({cluster[GlobalAttributeIds.CLUSTER_REVISION_ID]}) does not match revision listed in the spec ({self.xml_clusters[cluster_id].revision})') success = False if not success: - # TODO: Right now, we have failures in all-cluster, so we can't fail this test and keep it in CI. For now, just log. - # Issue tracking: #30210 - # self.fail_current_test("Problems with cluster revision on at least one cluster") - logging.error('Problems with cluster revision on at least one cluster') + self.fail_current_test("Problems with cluster revision on at least one cluster") if __name__ == "__main__":