diff --git a/HISTORY.rst b/HISTORY.rst index fee1c9ee3..289c87a00 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,9 @@ Release History **IoT Central updates** * az iot central device|device-template|api-token|diagnostic help strings updated with improved language. +* update parsing template logic to support DTDLV2 models. +* remove deprecated commands 1) iot central app device-twin 2) iot central app monitor-events + **IoT Hub updates** diff --git a/azext_iot/central/_help.py b/azext_iot/central/_help.py index 98cfc38de..05ecf8159 100644 --- a/azext_iot/central/_help.py +++ b/azext_iot/central/_help.py @@ -39,8 +39,6 @@ def load_central_help(): _load_central_monitors_help() _load_central_command_help() _load_central_compute_device_key() - # TODO: Delete this by end of Dec 2020 - _load_central_deprecated_commands() def _load_central_devices_help(): @@ -528,23 +526,6 @@ def _load_central_monitors_help(): az iot central diagnostics registration-summary --app-id {appid} """ - -# TODO: Delete this by end of Dec 2020 -def _load_central_deprecated_commands(): - helps[ - "iot central app device-twin" - ] = """ - type: group - short-summary: Manage IoT Central device twins. - """ - - helps[ - "iot central app device-twin show" - ] = """ - type: command - short-summary: Get the device twin from IoT Hub. - """ - helps[ "iot central device twin" ] = """ @@ -556,48 +537,5 @@ def _load_central_deprecated_commands(): "iot central device twin show" ] = """ type: command - short-summary: Get the device twin from IoT Central application. - long-summary: Returns back the desired and reported device properties from the IoT Central application. - """ - - helps[ - "iot central app monitor-events" - ] = """ - type: command - short-summary: Monitor device telemetry & messages sent to the IoT Hub for an IoT Central app. - long-summary: | - EXPERIMENTAL requires Python 3.5+ - This command relies on and may install dependent Cython package (uamqp) upon first execution. - https://github.com/Azure/azure-uamqp-python - examples: - - name: Basic usage - text: > - az iot central app monitor-events --app-id {app_id} - - name: Basic usage when filtering on target device - text: > - az iot central app monitor-events --app-id {app_id} -d {device_id} - - name: Basic usage when filtering targeted devices with a wildcard in the ID - text: > - az iot central app monitor-events --app-id {app_id} -d Device*d - - name: Basic usage when filtering on module. - text: > - az iot central app monitor-events --app-id {app_id} -m {module_id} - - name: Basic usage when filtering targeted modules with a wildcard in the ID - text: > - az iot central app monitor-events --app-id {app_id} -m Module* - - name: Filter device and specify an Event Hub consumer group to bind to. - text: > - az iot central app monitor-events --app-id {app_id} -d {device_id} --cg {consumer_group_name} - - name: Receive message annotations (message headers) - text: > - az iot central app monitor-events --app-id {app_id} -d {device_id} --properties anno - - name: Receive message annotations + system properties. Never time out. - text: > - az iot central app monitor-events --app-id {app_id} -d {device_id} --properties anno sys --timeout 0 - - name: Receive all message attributes from all device messages - text: > - az iot central app monitor-events --app-id {app_id} --props all - - name: Receive all messages and parse message payload as JSON - text: > - az iot central app monitor-events --app-id {app_id} --output json + short-summary: Get the device twin from IoT Hub. """ diff --git a/azext_iot/central/command_map.py b/azext_iot/central/command_map.py index 9c5579616..42c092bd1 100644 --- a/azext_iot/central/command_map.py +++ b/azext_iot/central/command_map.py @@ -109,28 +109,3 @@ def load_central_commands(self, _): cmd_group.show_command( "show", "device_twin_show", ) - - # TODO: Delete this by end of Dec 2020 - _load_central_deprecated_commands(self, _) - - -def _load_central_deprecated_commands(self, _): - with self.command_group( - "iot central app device-twin", - command_type=central_device_twin_ops, - deprecate_info=self.deprecate(redirect="iot central device twin"), - ) as cmd_group: - cmd_group.show_command( - "show", "device_twin_show", - ) - - with self.command_group( - "iot central app", command_type=central_monitor_ops, - ) as cmd_group: - cmd_group.command( - "monitor-events", - "monitor_events", - deprecate_info=self.deprecate( - redirect="iot central diagnostics monitor-events" - ), - ) diff --git a/azext_iot/central/models/template.py b/azext_iot/central/models/template.py index fe631ccd7..cfb767a84 100644 --- a/azext_iot/central/models/template.py +++ b/azext_iot/central/models/template.py @@ -15,30 +15,79 @@ def __init__(self, template: dict): self.name = template.get("displayName") self.interfaces = self._extract_interfaces(template) self.schema_names = self._extract_schema_names(self.interfaces) + self.components = self._extract_components(template) + if self.components: + self.component_schema_names = self._extract_schema_names( + self.components + ) + except: raise CLIError("Could not parse iot central device template.") - def get_schema(self, name, interface_name=""): - # interface_name has been specified, do a pointed lookup - if interface_name: - interface = self.interfaces.get(interface_name, {}) - return interface.get(name) + def get_schema(self, name, is_component=False, identifier="") -> dict: + entities = self.components if is_component else self.interfaces + if identifier: + # identifier specified, do a pointed lookup + entry = entities.get(identifier, {}) + return entry.get(name) - # find first matching name in any interface - for interface in self.interfaces.values(): - schema = interface.get(name) + # find first matching name in any component + for entry in entities.values(): + schema = entry.get(name) if schema: return schema # not found return None + def _extract_components(self, template: dict) -> dict: + try: + dcm = template.get("capabilityModel", {}) + if dcm.get("contents"): + rootContents = dcm.get("contents", {}) + components = [ + entity + for entity in rootContents + if entity.get("@type") == "Component" + ] + + if components: + return { + component["name"]: self._extract_schemas(component) + for component in components + } + return {} + return {} + except Exception: + details = "Unable to extract schema for component from template '{}'.".format( + self.id + ) + raise CLIError(details) + + def _extract_root_interface_contents(self, dcm: dict): + rootContents = dcm.get("contents", {}) + contents = [ + entity for entity in rootContents if entity.get("@type") != "Component" + ] + + return {"@id": dcm.get("@id", {}), "schema": {"contents": contents}} + def _extract_interfaces(self, template: dict) -> dict: try: + + interfaces = [] dcm = template.get("capabilityModel", {}) - interfaces = dcm.get("implements", {}) + + if dcm.get("contents"): + interfaces.append(self._extract_root_interface_contents(dcm)) + + if dcm.get("@type") == "CapabilityModel": + interfaces.extend(dcm.get("implements")) + else: + interfaces.extend(dcm.get("extends")) + return { - interface["name"]: self._extract_schemas(interface) + interface["@id"]: self._extract_schemas(interface) for interface in interfaces } except Exception: @@ -47,13 +96,13 @@ def _extract_interfaces(self, template: dict) -> dict: ) raise CLIError(details) - def _extract_schemas(self, interface: dict) -> dict: - return {schema["name"]: schema for schema in interface["schema"]["contents"]} + def _extract_schemas(self, entity: dict) -> dict: + return {schema["name"]: schema for schema in entity["schema"]["contents"]} - def _extract_schema_names(self, interfaces: dict) -> dict: + def _extract_schema_names(self, entity: dict) -> dict: return { - interface_name: list(interface_schemas.keys()) - for interface_name, interface_schemas in interfaces.items() + entity_name: list(entity_schemas.keys()) + for entity_name, entity_schemas in entity.items() } def _get_interface_list_property(self, property_name): diff --git a/azext_iot/central/params.py b/azext_iot/central/params.py index 98361d705..b800ccfa1 100644 --- a/azext_iot/central/params.py +++ b/azext_iot/central/params.py @@ -196,15 +196,3 @@ def load_central_arguments(self, _): context.argument( "module_id", options_list=["--module-id", "-m"], help="Provide IoT Edge Module ID if the device type is IoT Edge.", ) - # TODO: Delete this by end of Dec 2020 - load_deprecated_params(self, _) - - -def load_deprecated_params(self, _): - with self.argument_context("iot central app monitor-events") as context: - context.argument("timeout", arg_type=event_timeout_type) - context.argument("properties", arg_type=event_msg_prop_type) - context.argument( - "module_id", options_list=["--module-id", "-m"], help="Iot Edge Module ID", - ) - context.argument("minimum_severity", arg_type=severity_type) diff --git a/azext_iot/constants.py b/azext_iot/constants.py index 5d80ead0e..4e30e223b 100644 --- a/azext_iot/constants.py +++ b/azext_iot/constants.py @@ -44,7 +44,7 @@ DEVICETWIN_MONITOR_TIME_SEC = 15 # (Lib name, minimum version (including), maximum version (excluding)) EVENT_LIB = ("uamqp", "1.2", "1.3") -CENTRAL_PNP_INTERFACE_PREFIX = "$iotin:" +PNP_DTDLV2_COMPONENT_MARKER = "__t" # Config Key's CONFIG_KEY_UAMQP_EXT_VERSION = "uamqp_ext_version" diff --git a/azext_iot/monitor/parsers/central_parser.py b/azext_iot/monitor/parsers/central_parser.py index 5e8001564..f5a8c4686 100644 --- a/azext_iot/monitor/parsers/central_parser.py +++ b/azext_iot/monitor/parsers/central_parser.py @@ -92,17 +92,29 @@ def _perform_dynamic_validations(self, payload: dict): if not isinstance(template, Template): return - # pnp device is sending data to an unrecognized interface - if self.interface_name and (self.interface_name not in template.interfaces): - details = strings.invalid_interface_name( - self.interface_name, list(template.interfaces.keys()) + # if component name is not defined then data should be mapped to root/inherited interfaces + if not self.component_name: + self._validate_payload( + payload=payload, template=template, is_component=False ) + return + + if not template.components: + # template does not have any valid components + details = strings.invalid_component_name(self.component_name, list()) self._add_central_issue(severity=Severity.warning, details=details) return - self._validate_payload_against_interfaces( - payload=payload, template=template, - ) + # if component name is defined check to see if its a valid name + if self.component_name not in template.components: + details = strings.invalid_component_name( + self.component_name, list(template.components.keys()) + ) + self._add_central_issue(severity=Severity.warning, details=details) + return + + # if component name is valid check to see if payload is valid + self._validate_payload(payload=payload, template=template, is_component=True) def _get_template(self): try: @@ -119,13 +131,13 @@ def _get_template(self): # currently validates: # 1) primitive types match (e.g. boolean is indeed bool etc) # 2) names match (i.e. Humidity vs humidity etc) - def _validate_payload_against_interfaces( - self, payload: dict, template: Template, - ): + def _validate_payload(self, payload: dict, template: Template, is_component: bool): name_miss = [] for telemetry_name, telemetry in payload.items(): schema = template.get_schema( - name=telemetry_name, interface_name=self.interface_name + name=telemetry_name, + identifier=self.component_name, + is_component=is_component, ) if not schema: name_miss.append(telemetry_name) @@ -133,9 +145,14 @@ def _validate_payload_against_interfaces( self._process_telemetry(telemetry_name, schema, telemetry) if name_miss: - details = strings.invalid_field_name_mismatch_template( - name_miss, template.schema_names - ) + if is_component: + details = strings.invalid_field_name_component_mismatch_template( + name_miss, template.component_schema_names + ) + else: + details = strings.invalid_field_name_mismatch_template( + name_miss, template.schema_names, + ) self._add_central_issue(severity=Severity.warning, details=details) def _process_telemetry(self, telemetry_name: str, schema, telemetry): diff --git a/azext_iot/monitor/parsers/strings.py b/azext_iot/monitor/parsers/strings.py index 9c3450a1c..48123456b 100644 --- a/azext_iot/monitor/parsers/strings.py +++ b/azext_iot/monitor/parsers/strings.py @@ -56,10 +56,10 @@ def invalid_custom_headers(): # warning -def invalid_interface_name(interface_name: str, allowed_interfaces: list): +def invalid_component_name(component_name: str, allowed_components: list): return ( - "Device is specifying an interface that is unknown. Device specified interface: '{}'. Allowed interfaces: '{}'." - ).format(interface_name, allowed_interfaces) + "Device is specifying a component that is unknown. Device specified component: '{}'. Allowed components: '{}'." + ).format(component_name, allowed_components) # warning @@ -73,6 +73,17 @@ def invalid_field_name_mismatch_template( ).format(unmodeled_capabilities, modeled_capabilities) +# warning +def invalid_field_name_component_mismatch_template( + unmodeled_capabilities: list, modeled_capabilities: list +): + return ( + "Device is sending data that has not been defined in the device template. " + "Following capabilities have NOT been defined in the device template '{}'. " + "Following capabilities have been defined in the device template (grouped by components) '{}'. " + ).format(unmodeled_capabilities, modeled_capabilities) + + # warning def duplicate_property_name(duplicate_prop_name, interfaces: list): return ( diff --git a/azext_iot/monitor/property.py b/azext_iot/monitor/property.py index 3c78e3d88..b91a7e889 100644 --- a/azext_iot/monitor/property.py +++ b/azext_iot/monitor/property.py @@ -13,7 +13,7 @@ CENTRAL_ENDPOINT, DEVICETWIN_POLLING_INTERVAL_SEC, DEVICETWIN_MONITOR_TIME_SEC, - CENTRAL_PNP_INTERFACE_PREFIX, + PNP_DTDLV2_COMPONENT_MARKER, ) from azext_iot.central.models.devicetwin import DeviceTwin, Property @@ -77,10 +77,12 @@ def _is_relevant(self, key, val): return last_updated.timestamp() >= updated_within.timestamp() def _changed_props(self, prop, metadata, property_name): + # not an interface - whole thing is change log - if not self._is_interface(property_name): + if not self._is_component(prop): return prop - # iterate over property in the interface + + # iterate over properties in the component # if the property is not an exact match for what is present in the previous set of properties # track it as a change diff = { @@ -90,34 +92,29 @@ def _changed_props(self, prop, metadata, property_name): } return diff + def _is_component(self, prop): + return type(prop) == dict and prop.get(PNP_DTDLV2_COMPONENT_MARKER) == "c" + def _validate_payload(self, changes, minimum_severity): for value in changes: - issues = self._validate_payload_against_interfaces( + issues = self._validate_payload_against_entities( changes[value], value, minimum_severity ) for issue in issues: issue.log() - def _validate_payload_against_interfaces( - self, payload: dict, name, minimum_severity - ): + def _validate_payload_against_entities(self, payload: dict, name, minimum_severity): name_miss = [] issues_handler = IssueHandler() - interface_name = name.replace(CENTRAL_PNP_INTERFACE_PREFIX, "") - if self._is_interface(interface_name): - # if the payload is an interface then iterate thru the properties under the interface - for property_name in payload: - schema = self._template.get_schema( - name=property_name, interface_name=interface_name - ) - if not schema: - name_miss.append(property_name) - else: - # if the payload is a property then process the payload as a single unit. - schema = self._template.get_schema(name=name) + if not self._is_component(payload): + # update is not part of a component check under interfaces + schema = self._template.get_schema(name=name) if not schema: name_miss.append(name) + details = strings.invalid_field_name_mismatch_template( + name_miss, self._template.schema_names + ) interfaces_with_specified_property = self._template._get_interface_list_property( name @@ -134,11 +131,24 @@ def _validate_payload_against_interfaces( device_id=self._device_id, template_id=self._template.id, ) + else: + # Property update is part of a component perform additional validations under component list. + component_property_updates = [ + property_name + for property_name in payload + if property_name != PNP_DTDLV2_COMPONENT_MARKER + ] + for property_name in component_property_updates: + schema = self._template.get_schema( + name=property_name, identifier=name, is_component=True + ) + if not schema: + name_miss.append(property_name) + details = strings.invalid_field_name_component_mismatch_template( + name_miss, self._template.component_schema_names + ) if name_miss: - details = strings.invalid_field_name_mismatch_template( - name_miss, self._template.schema_names - ) issues_handler.add_central_issue( severity=Severity.warning, details=details, @@ -149,13 +159,6 @@ def _validate_payload_against_interfaces( return issues_handler.get_issues_with_minimum_severity(minimum_severity) - def _is_interface(self, interface_name): - # Remove PNP interface prefix to get the actual interface name - interface_name_modified = interface_name.replace( - CENTRAL_PNP_INTERFACE_PREFIX, "" - ) - return interface_name_modified in self._template.interfaces - def _get_device_template(self): device = self._central_device_provider.get_device(self._device_id) template = self._central_template_provider.get_device_template( diff --git a/azext_iot/tests/central/json/deeply_nested_template.json b/azext_iot/tests/central/json/deeply_nested_template.json index de66b86a4..7231d657d 100644 Binary files a/azext_iot/tests/central/json/deeply_nested_template.json and b/azext_iot/tests/central/json/deeply_nested_template.json differ diff --git a/azext_iot/tests/central/json/device_template.json b/azext_iot/tests/central/json/device_template.json index 5178f5b18..41cedaf66 100644 --- a/azext_iot/tests/central/json/device_template.json +++ b/azext_iot/tests/central/json/device_template.json @@ -7,9 +7,7 @@ "displayName": "duplicate-field-name", "capabilityModel": { "@id": "urn:sampleApp:modelOne_bz:2", - "@type": [ - "CapabilityModel" - ], + "@type": "CapabilityModel", "implements": [ { "@id": "urn:sampleApp:modelOne_bz:_rpgcmdpo:1", diff --git a/azext_iot/tests/central/json/device_template_int_test.json b/azext_iot/tests/central/json/device_template_int_test.json index bb57afa16..689656bf8 100644 --- a/azext_iot/tests/central/json/device_template_int_test.json +++ b/azext_iot/tests/central/json/device_template_int_test.json @@ -5,8 +5,15 @@ "displayName": "int-test-device-template", "capabilityModel": { "@id": "urn:sampleApp:modelOne_bz:2", - "@type": [ - "CapabilityModel" + "@type": "CapabilityModel", + "contents": [ + { + "@id": "urn:testazuresphere:AzureSphereSampleDevice_614:testDefaultCapability:2", + "@type": "Telemetry", + "displayName": "testDefaultCapability", + "name": "testDefaultCapability", + "schema": "double" + } ], "implements": [ { diff --git a/azext_iot/tests/central/json/property_validation_template.json b/azext_iot/tests/central/json/property_validation_template.json index 4c3530f21..ce0bee64b 100644 --- a/azext_iot/tests/central/json/property_validation_template.json +++ b/azext_iot/tests/central/json/property_validation_template.json @@ -7,8 +7,129 @@ "displayName": "property-validation", "capabilityModel": { "@id": "urn:sampleApp:groupOne_bz:2", - "@type": [ - "CapabilityModel" + "@type": "CapabilityModel", + "contents": [ + { + "@id": "urn:sampleApp:root_bz:_rpgcmdpo:1", + "@type": "Property", + "displayName": "addRootProperty", + "name": "addRootProperty", + "schema": "boolean", + "writable": true + }, + { + "@id": "urn:sampleApp:componentOne_bz:_rpgcmdpo:1", + "@type": "Component", + "displayName": "Component", + "name": "_rpgcmdpo", + "schema": { + "@id": "dtmi:sampleApp:_rpgcmdpo;4", + "@type": "Interface", + "contents": [ + { + "@id": "dtmi:sampleApp:_rpgcmdpo:component1Prop;1", + "@type": "Property", + "displayName": "component1Prop", + "name": "component1Prop", + "schema": "boolean", + "writable": true + }, + { + "@id": "dtmi:sampleApp:_rpgcmdpo:testComponent;1", + "@type": "Property", + "displayName": "testComponent", + "name": "testComponent", + "schema": "boolean", + "writable": true + }, + { + "@id": "dtmi:sampleApp:_rpgcmdpo:component1PropReadonly;1", + "@type": "Property", + "displayName": "component1PropReadonly", + "name": "component1PropReadonly", + "schema": "boolean", + "writable": false + }, + { + "@id": "dtmi:sampleApp:_rpgcmdpo:component1Prop2;1", + "@type": "Property", + "displayName": "component1Prop2", + "name": "component1Prop2", + "schema": "boolean", + "writable": true + } + ], + "displayName": "Component" + } + }, + { + "@id": "urn:rigado:RS40_Occupancy_Sensor:RS40OccupancySensorV36fy:3", + "@type": "Component", + "displayName": "Component", + "name": "RS40OccupancySensorV36fy", + "schema": { + "@id": "dtmi:cliIntegrationtestApp:RS40OccupancySensorV36fy;4", + "@type": "Interface", + "contents": [ + { + "@id": "dtmi:cliIntegrationtestApp:RS40OccupancySensorV36fy:component2prop;1", + "@type": "Property", + "displayName": "component2prop", + "name": "component2prop", + "schema": "boolean", + "writable": true + }, + { + "@id": "dtmi:cliIntegrationtestApp:RS40OccupancySensorV36fy:testComponent;2", + "@type": "Property", + "displayName": "testComponent", + "name": "testComponent", + "schema": "boolean", + "writable": true + }, + { + "@id": "dtmi:cliIntegrationtestApp:RS40OccupancySensorV36fy:component2PropReadonly;3", + "@type": "Property", + "displayName": "component2PropReadonly", + "name": "component2PropReadonly", + "schema": "boolean", + "writable": false + }, + { + "@id": "dtmi:cliIntegrationtestApp:RS40OccupancySensorV36fy:component2Prop2;4", + "@type": "Property", + "displayName": "component2Prop2", + "name": "component2Prop2", + "schema": "boolean", + "writable": true + }, + { + "@id": "dtmi:cliIntegrationtestApp:RS40OccupancySensorV36fy:component1Telemetry;5", + "@type": "Telemetry", + "displayName": "component1Telemetry", + "name": "component1Telemetry", + "schema": "double" + } + ], + "displayName": "Component" + } + }, + { + "@id": "urn:rigado:RS40_Occupancy_Sensor:addRootPropertyReadOnly:5", + "@type": "Property", + "displayName": "addRootPropertyReadOnly", + "name": "addRootPropertyReadOnly", + "schema": "boolean", + "writable": false + }, + { + "@id": "urn:rigado:RS40_Occupancy_Sensor:addRootProperty2:7", + "@type": "Property", + "displayName": "addRootProperty2", + "name": "addRootProperty2", + "schema": "boolean", + "writable": true + } ], "implements": [ { diff --git a/azext_iot/tests/test_iot_central_int.py b/azext_iot/tests/test_iot_central_int.py index 335af0eee..4d80e9f0a 100644 --- a/azext_iot/tests/test_iot_central_int.py +++ b/azext_iot/tests/test_iot_central_int.py @@ -40,14 +40,14 @@ def test_central_device_twin_show_fail(self): # Verify incorrect app-id throws error self.cmd( - "iot central app device-twin show --app-id incorrect-app --device-id {}".format( + "iot central device twin show --app-id incorrect-app --device-id {}".format( device_id ), expect_failure=True, ) # Verify incorrect device-id throws error self.cmd( - "iot central app device-twin show --app-id {} --device-id incorrect-device".format( + "iot central device twin show --app-id {} --device-id incorrect-device".format( APP_ID ), expect_failure=True, @@ -79,7 +79,7 @@ def test_central_device_twin_show_success(self): time.sleep(60) self.cmd( - "iot central app device-twin show --app-id {} --device-id {}".format( + "iot central device twin show --app-id {} --device-id {}".format( APP_ID, device_id ), checks=[self.check("deviceId", device_id)], @@ -95,38 +95,6 @@ def test_central_device_twin_show_success(self): self._delete_device(device_id) self._delete_device_template(template_id) - # TODO: Delete this by end of Dec 2020 - def test_central_monitor_events_deprecated(self): - (template_id, _) = self._create_device_template() - (device_id, _) = self._create_device(instance_of=template_id) - credentials = self._get_credentials(device_id) - - device_client = helpers.dps_connect_device(device_id, credentials) - - enqueued_time = utility.calculate_millisec_since_unix_epoch_utc() - 10000 - - payload = {"Bool": True} - msg = Message( - data=json.dumps(payload), - content_encoding="utf-8", - content_type="application/json", - ) - device_client.send_message(msg) - - # Test with invalid app-id - self.cmd( - "iot central app monitor-events --app-id {} -y".format(APP_ID + "zzz"), - expect_failure=True, - ) - - # Ensure no failure - output = self._get_monitor_events_output_deprecated(device_id, enqueued_time) - - self._delete_device(device_id) - self._delete_device_template(template_id) - assert '"Bool": true' in output - assert device_id in output - def test_central_monitor_events(self): (template_id, _) = self._create_device_template() (device_id, _) = self._create_device(instance_of=template_id) @@ -679,22 +647,3 @@ def _get_monitor_events_output(self, device_id, enqueued_time, asserts=None): output = "" return output - - # TODO: Delete this by end of Dec 2020 - def _get_monitor_events_output_deprecated( - self, device_id, enqueued_time, asserts=None - ): - if not asserts: - asserts = [] - - output = self.command_execute_assert( - "iot central app monitor-events -n {} -d {} --et {} --to 1 -y".format( - APP_ID, device_id, enqueued_time - ), - asserts, - ) - - if not output: - output = "" - - return output diff --git a/azext_iot/tests/test_iot_central_unit.py b/azext_iot/tests/test_iot_central_unit.py index 5e51e9758..50b1d1933 100644 --- a/azext_iot/tests/test_iot_central_unit.py +++ b/azext_iot/tests/test_iot_central_unit.py @@ -24,7 +24,7 @@ from azext_iot.monitor.models.enum import Severity from .helpers import load_json from .test_constants import FileNames - +from azext_iot.constants import PNP_DTDLV2_COMPONENT_MARKER device_id = "mydevice" app_id = "myapp" @@ -125,7 +125,9 @@ class Cmd: class TestDeviceTwinShow: @pytest.fixture - def service_client(self, mocked_response, fixture_cmd, fixture_get_iot_central_tokens): + def service_client( + self, mocked_response, fixture_cmd, fixture_get_iot_central_tokens + ): mocked_response.add( method=responses.GET, url="https://{}/twins/{}".format(resource, device_id), @@ -137,9 +139,7 @@ def service_client(self, mocked_response, fixture_cmd, fixture_get_iot_central_t yield mocked_response - def test_device_twin_show_calls_get_twin( - self, service_client - ): + def test_device_twin_show_calls_get_twin(self, service_client): result = commands_device_twin.device_twin_show(fixture_cmd, device_id, app_id) assert result == device_twin_result @@ -294,20 +294,21 @@ def test_validate_properties_declared_multiple_interfaces( model = {"Model": "test_model"} - issues = monitor._validate_payload_against_interfaces( + issues = monitor._validate_payload_against_entities( model, list(model.keys())[0], Severity.warning, ) assert ( issues[0].details == "Duplicate property: 'Model' found under following " - "interfaces ['groupOne_g4', 'groupTwo_ed', 'groupThree_ed'] " + "interfaces ['urn:sampleApp:groupOne_bz:_rpgcmdpo:1', 'urn:sampleApp:groupTwo_bz:myxqftpsr:2', " + "'urn:sampleApp:groupThree_bz:myxqftpsr:2'] " "in the device model. Either provide the interface name as part " "of the device payload or make the propery name unique in the device model" ) version = {"OsName": "test_osName"} - issues = monitor._validate_payload_against_interfaces( + issues = monitor._validate_payload_against_entities( version, list(version.keys())[0], Severity.warning, ) @@ -315,7 +316,7 @@ def test_validate_properties_declared_multiple_interfaces( @mock.patch("azext_iot.central.services.device_template") @mock.patch("azext_iot.central.services.device") - def test_validate_properties_name_miss( + def test_validate_properties_name_miss_under_interface( self, mock_device_svc, mock_device_template_svc ): @@ -335,39 +336,19 @@ def test_validate_properties_name_miss( # invalid interface / property definition = {"definition": "test_definition"} - issues = monitor._validate_payload_against_interfaces( + issues = monitor._validate_payload_against_entities( definition, list(definition.keys())[0], Severity.warning, ) assert ( issues[0].details - == "Device is sending data that has not been defined in the device " - "template. Following capabilities have NOT been defined in the device template " - "'['definition']'. Following capabilities have been defined in the device template " - "(grouped by interface) '{'groupOne_g4': ['Model', 'Version', 'TotalStorage'], " - "'groupTwo_ed': ['Model', 'Manufacturer'], 'groupThree_ed': ['Manufacturer', " - "'Version', 'Model', 'OsName']}'. " - ) - - # invalid and valid property with valid interface - property_collection = { - "Model": "test_model", - "Manufacturer": "test_manufacturer", - "OsName": "test_osName", - } - - issues = monitor._validate_payload_against_interfaces( - property_collection, "groupOne_g4", Severity.warning, - ) - - assert ( - issues[0].details - == "Device is sending data that has not been defined in the device " - "template. Following capabilities have NOT been defined in the device template " - "'['Manufacturer', 'OsName']'. Following capabilities have been defined in the device template " - "(grouped by interface) '{'groupOne_g4': ['Model', 'Version', 'TotalStorage'], " - "'groupTwo_ed': ['Model', 'Manufacturer'], 'groupThree_ed': ['Manufacturer', " - "'Version', 'Model', 'OsName']}'. " + == "Device is sending data that has not been defined in the device template." + " Following capabilities have NOT been defined in the device template '['definition']'." + " Following capabilities have been defined in the device template (grouped by interface)" + " '{'urn:sampleApp:groupOne_bz:2': ['addRootProperty', 'addRootPropertyReadOnly', 'addRootProperty2']," + " 'urn:sampleApp:groupOne_bz:_rpgcmdpo:1': ['Model', 'Version', 'TotalStorage']," + " 'urn:sampleApp:groupTwo_bz:myxqftpsr:2': ['Model', 'Manufacturer']," + " 'urn:sampleApp:groupThree_bz:myxqftpsr:2': ['Manufacturer', 'Version', 'Model', 'OsName']}'. " ) @mock.patch("azext_iot.central.services.device_template") @@ -392,23 +373,64 @@ def test_validate_properties_severity_level( # severity level info definition = {"definition": "test_definition"} - issues = monitor._validate_payload_against_interfaces( + issues = monitor._validate_payload_against_entities( definition, list(definition.keys())[0], Severity.info, ) assert ( issues[0].details - == "Device is sending data that has not been defined in the device " - "template. Following capabilities have NOT been defined in the device template " + == "Device is sending data that has not been defined in the device template. " + "Following capabilities have NOT been defined in the device template " "'['definition']'. Following capabilities have been defined in the device template " - "(grouped by interface) '{'groupOne_g4': ['Model', 'Version', 'TotalStorage'], " - "'groupTwo_ed': ['Model', 'Manufacturer'], 'groupThree_ed': ['Manufacturer', " - "'Version', 'Model', 'OsName']}'. " + "(grouped by interface) '{'urn:sampleApp:groupOne_bz:2': " + "['addRootProperty', 'addRootPropertyReadOnly', 'addRootProperty2'], " + "'urn:sampleApp:groupOne_bz:_rpgcmdpo:1': ['Model', 'Version', 'TotalStorage'], " + "'urn:sampleApp:groupTwo_bz:myxqftpsr:2': ['Model', 'Manufacturer'], " + "'urn:sampleApp:groupThree_bz:myxqftpsr:2': ['Manufacturer', 'Version', 'Model', 'OsName']}'. " ) # severity level error - issues = monitor._validate_payload_against_interfaces( + issues = monitor._validate_payload_against_entities( definition, list(definition.keys())[0], Severity.error, ) assert len(issues) == 0 + + @mock.patch("azext_iot.central.services.device_template") + @mock.patch("azext_iot.central.services.device") + def test_validate_properties_name_miss_under_component( + self, mock_device_svc, mock_device_template_svc + ): + + # setup + mock_device_template_svc.get_device_template.return_value = Template( + self._duplicate_property_template + ) + + monitor = PropertyMonitor( + cmd=None, + app_id=app_id, + device_id=device_id, + token=None, + central_dns_suffix=None, + ) + + # invalid component property + definition = { + PNP_DTDLV2_COMPONENT_MARKER: "c", + "data": {"definition": "test_definition"}, + } + + issues = monitor._validate_payload_against_entities( + definition, list(definition.keys())[0], Severity.warning, + ) + + assert ( + issues[0].details + == "Device is sending data that has not been defined in the device template. " + "Following capabilities have NOT been defined in the device template '['data']'. " + "Following capabilities have been defined in the device template (grouped by components) " + "'{'_rpgcmdpo': ['component1Prop', 'testComponent', 'component1PropReadonly', 'component1Prop2'], " + "'RS40OccupancySensorV36fy': ['component2prop', 'testComponent', 'component2PropReadonly', " + "'component2Prop2', 'component1Telemetry']}'. " + ) diff --git a/azext_iot/tests/test_iot_central_validator_unit.py b/azext_iot/tests/test_iot_central_validator_unit.py index c401e0a40..844b5c6ac 100644 --- a/azext_iot/tests/test_iot_central_validator_unit.py +++ b/azext_iot/tests/test_iot_central_validator_unit.py @@ -5,6 +5,7 @@ # -------------------------------------------------------------------------------------------- import pytest +import collections from azext_iot.central.models.template import Template from azext_iot.monitor.central_validator import validate, extract_schema_type @@ -13,7 +14,69 @@ from .test_constants import FileNames +class TestTemplateValidations: + def test_template_interface_list(self): + expected_interface_list = [ + "urn:sampleApp:groupOne_bz:_rpgcmdpo:1", + "urn:sampleApp:groupTwo_bz:myxqftpsr:2", + "urn:sampleApp:groupThree_bz:myxqftpsr:2", + "urn:sampleApp:groupOne_bz:2", + ] + template = Template( + load_json(FileNames.central_property_validation_template_file) + ) + + assert collections.Counter(template.interfaces.keys()) == collections.Counter( + expected_interface_list + ) + + def test_template_component_list(self): + expected_component_list = [ + "_rpgcmdpo", + "RS40OccupancySensorV36fy", + ] + template = Template( + load_json(FileNames.central_property_validation_template_file) + ) + + assert collections.Counter(template.components.keys()) == collections.Counter( + expected_component_list + ) + + class TestExtractSchemaType: + def test_extract_schema_type_component(self): + expected_mapping = { + "component1Prop": "boolean", + "testComponent": "boolean", + "component1PropReadonly": "boolean", + "component1Prop2": "boolean", + } + template = Template( + load_json(FileNames.central_property_validation_template_file) + ) + for key, val in expected_mapping.items(): + schema = template.get_schema(key, is_component=True) + schema_type = extract_schema_type(schema) + assert schema_type == val + + def test_extract_schema_type_component_identifier(self): + expected_mapping = { + "component2prop": "boolean", + "component2Prop2": "boolean", + "testComponent": "boolean", + "component2PropReadonly": "boolean", + } + template = Template( + load_json(FileNames.central_property_validation_template_file) + ) + for key, val in expected_mapping.items(): + schema = template.get_schema( + key, is_component=True, identifier="RS40OccupancySensorV36fy" + ) + schema_type = extract_schema_type(schema) + assert schema_type == val + def test_extract_schema_type(self): expected_mapping = { "Bool": "boolean", diff --git a/azext_iot/tests/test_monitor_parsers_unit.py b/azext_iot/tests/test_monitor_parsers_unit.py index 36cb9a6e6..d8c4f9a8f 100644 --- a/azext_iot/tests/test_monitor_parsers_unit.py +++ b/azext_iot/tests/test_monitor_parsers_unit.py @@ -310,6 +310,7 @@ class TestCentralParser: encoding = "UTF-8" content_type = "application/json" app_properties = {"appPropsKey": "appPropsValue"} + component_name = "some-component-name" bad_encoding = "ascii" bad_payload = "bad-payload" @@ -394,6 +395,145 @@ def test_validate_against_template_should_fail(self): _validate_issues(parser, Severity.warning, 1, 1, [expected_details]) + def test_validate_against_no_component_template_should_fail(self): + # setup + device_template = self._get_template() + + properties = MessageProperties( + content_encoding=self.encoding, content_type=self.content_type + ) + message = Message( + body=json.dumps(self.bad_dcm_payload).encode(), + properties=properties, + annotations={ + common_parser.DEVICE_ID_IDENTIFIER: self.device_id.encode(), + common_parser.COMPONENT_NAME_IDENTIFIER: self.component_name.encode(), + }, + application_properties=_encode_app_props(self.app_properties), + ) + args = CommonParserArguments(properties=["all"]) + parser = self._create_parser( + device_template=device_template, message=message, args=args + ) + + # act + parsed_msg = parser.parse_message() + + # verify + assert parsed_msg["event"]["payload"] == self.bad_dcm_payload + assert parsed_msg["event"]["origin"] == self.device_id + device_identifier = str(common_parser.DEVICE_ID_IDENTIFIER, "utf8") + assert parsed_msg["event"]["annotations"][device_identifier] == self.device_id + component_identifier = str(common_parser.COMPONENT_NAME_IDENTIFIER, "utf8") + assert ( + parsed_msg["event"]["annotations"][component_identifier] + == self.component_name + ) + properties = parsed_msg["event"]["properties"] + assert properties["system"]["content_encoding"] == self.encoding + assert properties["system"]["content_type"] == self.content_type + assert properties["application"] == self.app_properties + + expected_details = strings.invalid_component_name(self.component_name, list()) + + _validate_issues(parser, Severity.warning, 1, 1, [expected_details]) + + def test_validate_against_invalid_component_template_should_fail(self): + # setup + device_template = Template( + load_json(FileNames.central_property_validation_template_file) + ) + + properties = MessageProperties( + content_encoding=self.encoding, content_type=self.content_type + ) + message = Message( + body=json.dumps(self.bad_dcm_payload).encode(), + properties=properties, + annotations={ + common_parser.DEVICE_ID_IDENTIFIER: self.device_id.encode(), + common_parser.COMPONENT_NAME_IDENTIFIER: self.component_name.encode(), + }, + application_properties=_encode_app_props(self.app_properties), + ) + args = CommonParserArguments(properties=["all"]) + parser = self._create_parser( + device_template=device_template, message=message, args=args + ) + + # act + parsed_msg = parser.parse_message() + + # verify + assert parsed_msg["event"]["payload"] == self.bad_dcm_payload + assert parsed_msg["event"]["origin"] == self.device_id + device_identifier = str(common_parser.DEVICE_ID_IDENTIFIER, "utf8") + assert parsed_msg["event"]["annotations"][device_identifier] == self.device_id + component_identifier = str(common_parser.COMPONENT_NAME_IDENTIFIER, "utf8") + assert ( + parsed_msg["event"]["annotations"][component_identifier] + == self.component_name + ) + properties = parsed_msg["event"]["properties"] + assert properties["system"]["content_encoding"] == self.encoding + assert properties["system"]["content_type"] == self.content_type + assert properties["application"] == self.app_properties + + expected_details = strings.invalid_component_name( + self.component_name, list(device_template.components.keys()) + ) + + _validate_issues(parser, Severity.warning, 1, 1, [expected_details]) + + def test_validate_invalid_telmetry_component_template_should_fail(self): + # setup + device_template = Template( + load_json(FileNames.central_property_validation_template_file) + ) + + properties = MessageProperties( + content_encoding=self.encoding, content_type=self.content_type + ) + message = Message( + body=json.dumps(self.bad_dcm_payload).encode(), + properties=properties, + annotations={ + common_parser.DEVICE_ID_IDENTIFIER: self.device_id.encode(), + common_parser.COMPONENT_NAME_IDENTIFIER: list( + device_template.components.keys() + )[1].encode(), + }, + application_properties=_encode_app_props(self.app_properties), + ) + args = CommonParserArguments(properties=["all"]) + parser = self._create_parser( + device_template=device_template, message=message, args=args + ) + + # act + parsed_msg = parser.parse_message() + + # verify + assert parsed_msg["event"]["payload"] == self.bad_dcm_payload + assert parsed_msg["event"]["origin"] == self.device_id + device_identifier = str(common_parser.DEVICE_ID_IDENTIFIER, "utf8") + assert parsed_msg["event"]["annotations"][device_identifier] == self.device_id + component_identifier = str(common_parser.COMPONENT_NAME_IDENTIFIER, "utf8") + assert ( + parsed_msg["event"]["annotations"][component_identifier] + == list(device_template.components.keys())[1] + ) + properties = parsed_msg["event"]["properties"] + assert properties["system"]["content_encoding"] == self.encoding + assert properties["system"]["content_type"] == self.content_type + assert properties["application"] == self.app_properties + + expected_details = strings.invalid_field_name_component_mismatch_template( + list(self.bad_dcm_payload.keys()), device_template.component_schema_names, + ) + + _validate_issues(parser, Severity.warning, 1, 1, [expected_details]) + def test_validate_against_bad_template_should_not_throw(self): # setup device_template = "an_unparseable_template"