Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Conformance: separate files, turn on errors #30296

Merged
merged 5 commits into from
Nov 16, 2023
Merged

Conformance: separate files, turn on errors #30296

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f5e74a3
Move conformance tests to new file
cecille Nov 7, 2023
f1704b8
Fail tests on error, add more in progress
cecille Nov 8, 2023
e2e3009
Fix linter
cecille Nov 8, 2023
cc8580c
bad lint
cecille Nov 8, 2023
97ad862
Restyled by isort
restyled-commits Nov 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
174 changes: 0 additions & 174 deletions src/python_testing/TC_DeviceBasicComposition.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,10 @@
import chip.tlv
from basic_composition_support import BasicCompositionTests
from chip.clusters.Attribute import ValueDecodeFailure
from chip.tlv import uint
from conformance_support import ConformanceDecision, conformance_allowed
from global_attribute_ids import GlobalAttributeIds
from matter_testing_support import (AttributePathLocation, ClusterPathLocation, CommandPathLocation, MatterBaseTest,
async_test_body, default_matter_test_main)
from mobly import asserts
from spec_parsing_support import CommandType, build_xml_clusters
from taglist_and_topology_test_support import (create_device_type_list_for_root, create_device_type_lists, find_tag_list_problems,
find_tree_roots, get_all_children, get_direct_children_of_root, parts_list_cycles,
separate_endpoint_types)
Expand Down Expand Up @@ -708,177 +705,6 @@ def test_DESC_2_2(self):
if problems or root_problems:
self.fail_current_test("Problems with tags lists")

def test_spec_conformance(self):
def conformance_str(conformance: Callable, feature_map: uint, feature_dict: dict[str, uint]) -> str:
codes = []
for mask, details in feature_dict.items():
if mask & feature_map:
codes.append(details.code)

return f'Conformance: {str(conformance)}, implemented features: {",".join(codes)}'

ignore_in_progress = self.user_params.get("ignore_in_progress", False)
is_ci = self.check_pics('PICS_SDK_CI_ONLY')

ignore_attributes: dict[int, list[int]] = {}
if ignore_in_progress:
# This is a manually curated list of attributes that are in-progress in the SDK, but have landed in the spec
in_progress_attributes = {Clusters.BasicInformation.id: [0x15, 0x016]}
ignore_attributes.update(in_progress_attributes)

if is_ci:
# The network commissioning clusters on the CI select the features on the fly and end up non-conformant
# on these attributes. Production devices should not.
ci_ignore_attributes = {Clusters.NetworkCommissioning.id: [
Clusters.NetworkCommissioning.Attributes.ScanMaxTimeSeconds.attribute_id, Clusters.NetworkCommissioning.Attributes.ConnectMaxTimeSeconds.attribute_id]}
ignore_attributes.update(ci_ignore_attributes)

success = True
allow_provisional = self.user_params.get("allow_provisional", False)
clusters, problems = build_xml_clusters()
self.problems = self.problems + problems
for endpoint_id, endpoint in self.endpoints_tlv.items():
for cluster_id, cluster in endpoint.items():
if cluster_id not in clusters.keys():
if (cluster_id & 0xFFFF_0000) != 0:
# manufacturer cluster
continue
location = ClusterPathLocation(endpoint_id=endpoint_id, cluster_id=cluster_id)
# TODO: update this from a warning once we have all the data
self.record_warning(self.get_test_name(), location=location,
problem='Standard cluster found on device, but is not present in spec data')
continue

feature_map = cluster[GlobalAttributeIds.FEATURE_MAP_ID]
attribute_list = cluster[GlobalAttributeIds.ATTRIBUTE_LIST_ID]
all_command_list = cluster[GlobalAttributeIds.ACCEPTED_COMMAND_LIST_ID] + \
cluster[GlobalAttributeIds.GENERATED_COMMAND_LIST_ID]

# Feature conformance checking
feature_masks = [1 << i for i in range(32) if feature_map & (1 << i)]
for f in feature_masks:
location = AttributePathLocation(endpoint_id=endpoint_id, cluster_id=cluster_id,
attribute_id=GlobalAttributeIds.FEATURE_MAP_ID)
if f not in clusters[cluster_id].features.keys():
self.record_error(self.get_test_name(), location=location, problem=f'Unknown feature with mask 0x{f:02x}')
success = False
continue
xml_feature = clusters[cluster_id].features[f]
conformance_decision = xml_feature.conformance(feature_map, attribute_list, all_command_list)
if not conformance_allowed(conformance_decision, allow_provisional):
self.record_error(self.get_test_name(), location=location,
problem=f'Disallowed feature with mask 0x{f:02x}')
success = False
for feature_mask, xml_feature in clusters[cluster_id].features.items():
conformance_decision = xml_feature.conformance(feature_map, attribute_list, all_command_list)
if conformance_decision == ConformanceDecision.MANDATORY and feature_mask not in feature_masks:
self.record_error(self.get_test_name(), location=location,
problem=f'Required feature with mask 0x{f:02x} is not present in feature map. {conformance_str(xml_feature.conformance, feature_map, clusters[cluster_id].features)}')
success = False

# Attribute conformance checking
for attribute_id, attribute in cluster.items():
if cluster_id in ignore_attributes and attribute_id in ignore_attributes[cluster_id]:
continue
location = AttributePathLocation(endpoint_id=endpoint_id, cluster_id=cluster_id, attribute_id=attribute_id)
if attribute_id not in clusters[cluster_id].attributes.keys():
# TODO: Consolidate the range checks with IDM-10.1 once that lands
if attribute_id <= 0x4FFF:
# manufacturer attribute
self.record_error(self.get_test_name(), location=location,
problem='Standard attribute found on device, but not in spec')
success = False
continue
xml_attribute = clusters[cluster_id].attributes[attribute_id]
conformance_decision = xml_attribute.conformance(feature_map, attribute_list, all_command_list)
if not conformance_allowed(conformance_decision, allow_provisional):
location = AttributePathLocation(endpoint_id=endpoint_id, cluster_id=cluster_id, attribute_id=attribute_id)
self.record_error(self.get_test_name(), location=location,
problem=f'Attribute 0x{attribute_id:02x} is included, but is disallowed by conformance. {conformance_str(xml_attribute.conformance, feature_map, clusters[cluster_id].features)}')
success = False
for attribute_id, xml_attribute in clusters[cluster_id].attributes.items():
if cluster_id in ignore_attributes and attribute_id in ignore_attributes[cluster_id]:
continue
conformance_decision = xml_attribute.conformance(feature_map, attribute_list, all_command_list)
if conformance_decision == ConformanceDecision.MANDATORY and attribute_id not in cluster.keys():
location = AttributePathLocation(endpoint_id=endpoint_id, cluster_id=cluster_id, attribute_id=attribute_id)
self.record_error(self.get_test_name(), location=location,
problem=f'Attribute 0x{attribute_id:02x} is required, but is not present on the DUT. {conformance_str(xml_attribute.conformance, feature_map, clusters[cluster_id].features)}')
success = False

def check_spec_conformance_for_commands(command_type: CommandType) -> bool:
success = True
global_attribute_id = GlobalAttributeIds.ACCEPTED_COMMAND_LIST_ID if command_type == CommandType.ACCEPTED else GlobalAttributeIds.GENERATED_COMMAND_LIST_ID
xml_commands_dict = clusters[cluster_id].accepted_commands if command_type == CommandType.ACCEPTED else clusters[cluster_id].generated_commands
command_list = cluster[global_attribute_id]
for command_id in command_list:
location = CommandPathLocation(endpoint_id=endpoint_id, cluster_id=cluster_id, command_id=command_id)
if command_id not in xml_commands_dict:
# TODO: Consolidate range checks with IDM-10.1 once that lands
if command_id <= 0xFF:
# manufacturer command
continue
self.record_error(self.get_test_name(), location=location,
problem='Standard command found on device, but not in spec')
success = False
continue
xml_command = xml_commands_dict[command_id]
conformance_decision = xml_command.conformance(feature_map, attribute_list, all_command_list)
if not conformance_allowed(conformance_decision, allow_provisional):
self.record_error(self.get_test_name(), location=location,
problem=f'Command 0x{command_id:02x} is included, but disallowed by conformance. {conformance_str(xml_command.conformance, feature_map, clusters[cluster_id].features)}')
success = False
for command_id, xml_command in xml_commands_dict.items():
conformance_decision = xml_command.conformance(feature_map, attribute_list, all_command_list)
if conformance_decision == ConformanceDecision.MANDATORY and command_id not in command_list:
location = CommandPathLocation(endpoint_id=endpoint_id, cluster_id=cluster_id, command_id=command_id)
self.record_error(self.get_test_name(), location=location,
problem=f'Command 0x{command_id:02x} is required, but is not present on the DUT. {conformance_str(xml_command.conformance, feature_map, clusters[cluster_id].features)}')
success = False
return success

# Command conformance checking
cmd_success = check_spec_conformance_for_commands(CommandType.ACCEPTED)
success = False if not cmd_success else success
cmd_success = check_spec_conformance_for_commands(CommandType.GENERATED)
success = False if not cmd_success else success

# TODO: Add choice checkers

if not success:
# TODO: Right now, we have failures in all-cluster, so we can't fail this test and keep it in CI. For now, just log.
# Issue tracking: #29812
# self.fail_current_test("Problems with conformance")
logging.error("Problems found with conformance, this should turn into a test failure once #29812 is resolved")

def test_IDM_10_3(self):
# TODO: move to class setup
success = True
clusters, problems = build_xml_clusters()
self.problems = self.problems + problems
for endpoint_id, endpoint in self.endpoints_tlv.items():
for cluster_id, cluster in endpoint.items():
if cluster_id not in clusters.keys():
if (cluster_id & 0xFFFF_0000) != 0:
# manufacturer cluster
continue
location = ClusterPathLocation(endpoint_id=endpoint_id, cluster_id=cluster_id)
# TODO: update this from a warning once we have all the data
self.record_warning(self.get_test_name(), location=location,
problem='Standard cluster found on device, but is not present in spec data')
continue
if int(clusters[cluster_id].revision) != cluster[GlobalAttributeIds.CLUSTER_REVISION_ID]:
location = AttributePathLocation(endpoint_id=endpoint_id, cluster_id=cluster_id,
attribute_id=GlobalAttributeIds.CLUSTER_REVISION_ID)
self.record_error(self.get_test_name(
), location=location, problem=f'Revision found on cluster ({cluster[GlobalAttributeIds.CLUSTER_REVISION_ID]}) does not match revision listed in the spec ({clusters[cluster_id].revision})')
success = False
if not success:
# TODO: Right now, we have failures in all-cluster, so we can't fail this test and keep it in CI. For now, just log.
# Issue tracking: #30210
# self.fail_current_test("Problems with cluster revision on at least one cluster")
logging.error('Problems with cluster revision on at least one cluster')


if __name__ == "__main__":
default_matter_test_main()
Loading
Loading