From 45be15408b80dbf337d6634b43041e34d6f247fd Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Tue, 20 Sep 2022 17:09:27 -0400 Subject: [PATCH 1/5] copy ADO branch changes into GH branch --- sdk/ml/azure-ai-ml/CHANGELOG.md | 7 ++++++- .../azure/ai/ml/entities/_compute/_identity.py | 3 ++- .../ai/ml/entities/_compute/compute_instance.py | 12 +++++++++++- .../ml/entities/_deployment/code_configuration.py | 3 ++- .../azure/ai/ml/entities/_job/identity.py | 2 +- .../azure/ai/ml/entities/_job/pipeline/_io.py | 3 +++ sdk/ml/azure-ai-ml/azure/ai/ml/entities/_mixins.py | 14 ++++++++++---- 7 files changed, 35 insertions(+), 9 deletions(-) diff --git a/sdk/ml/azure-ai-ml/CHANGELOG.md b/sdk/ml/azure-ai-ml/CHANGELOG.md index 7b3e82e6ccdd..48f2044ae33e 100644 --- a/sdk/ml/azure-ai-ml/CHANGELOG.md +++ b/sdk/ml/azure-ai-ml/CHANGELOG.md @@ -3,12 +3,15 @@ ## 0.1.0 (Unreleased) ### Features Added + - Added a `show_progress` parameter to MLClient for enable/disable progress bars of long running operations. ### Breaking Changes ### Bugs Fixed ### Other Changes + - Removed declaration on Python 3.6 support + - Added support for custom setup scripts on compute instances. - Removed declaration on Python 3.6 support. - Updated dependencies upper bounds to be major versions. @@ -20,6 +23,7 @@ - Entity load and dump now also accept a file pointer as input. - Load and dump input names changed from path to 'source' and 'dest', respectively. - Load and dump 'path' input still works, but is deprecated and emits a warning. + - Most configuration classes from the entity package now implement the standard mapping protocol. - Managed Identity Support for Compute Instance (experimental). - Enable using @dsl.pipeline without brackets when no additional parameters. - Expose Azure subscription Id and resource group name from MLClient objects. @@ -30,14 +34,15 @@ - Remove invalid option from create_or_update typehints. - Change error returned by (begin_)create_or_update invalid input to TypeError. - Rename set_image_model APIs for all vision tasks to set_training_parameters + - JobOperations.download no longer provides a default value for download_path - JobOperations.download defaults to "." instead of Path.cwd() + - Workspace.list_keys renamed to Workspace.get_keys. ### Bugs Fixed ### Other Changes - Show 'properties' on data assets - ## 0.1.0b6 ### Features Added diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/_identity.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/_identity.py index 4aaabba29862..dae8d411e634 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/_identity.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/_identity.py @@ -7,11 +7,12 @@ from azure.ai.ml._restclient.v2022_01_01_preview.models import Identity as RestIdentity from azure.ai.ml._utils.utils import camel_to_snake, snake_to_pascal from azure.ai.ml.entities._mixins import RestTranslatableMixin +from azure.ai.ml.entities._mixins import DictMixin from ._user_assigned_identity import UserAssignedIdentity -class IdentityConfiguration(RestTranslatableMixin): +class IdentityConfiguration(RestTranslatableMixin, DictMixin): """Managed identity specification.""" def __init__(self, *, type: str, user_assigned_identities: List[UserAssignedIdentity] = None): diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/compute_instance.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/compute_instance.py index 4c9c8a1db858..b74974f86e4a 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/compute_instance.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/compute_instance.py @@ -20,11 +20,13 @@ from azure.ai.ml.constants._common import BASE_PATH_CONTEXT_KEY, TYPE from azure.ai.ml.constants._compute import ComputeDefaults, ComputeType from azure.ai.ml.entities._compute.compute import Compute, NetworkSettings +from azure.ai.ml.entities._mixins import DictMixin from azure.ai.ml.entities._util import load_from_dict from azure.ai.ml.exceptions import ErrorCategory, ErrorTarget, ValidationException from ._identity import IdentityConfiguration from ._schedule import ComputeSchedules +from ._setup_scripts import SetupScripts class ComputeInstanceSshSettings: @@ -69,7 +71,7 @@ def ssh_port(self) -> str: return self._ssh_port -class AssignedUserConfiguration: +class AssignedUserConfiguration(DictMixin): """Settings to create a compute on behalf of another user.""" def __init__(self, *, user_tenant_id: str, user_object_id: str): @@ -123,6 +125,8 @@ class ComputeInstance(Compute): :param idle_time_before_shutdown: Stops compute instance after user defined period of inactivity. Time is defined in ISO8601 format. Minimum is 15 min, maximum is 3 days. :type idle_time_before_shutdown: Optional[str], optional + :param setup_scripts: Details of customized scripts to execute for setting up the cluster. + :type setup_scripts: Optional[SetupScripts], optional """ def __init__( @@ -138,6 +142,7 @@ def __init__( schedules: Optional[ComputeSchedules] = None, identity: IdentityConfiguration = None, idle_time_before_shutdown: Optional[str] = None, + setup_scripts: Optional[SetupScripts] = None, **kwargs, ): kwargs[TYPE] = ComputeType.COMPUTEINSTANCE @@ -159,6 +164,7 @@ def __init__( self.schedules = schedules self.identity = identity self.idle_time_before_shutdown = idle_time_before_shutdown + self.setup_scripts = setup_scripts self.subnet = None @property @@ -227,6 +233,7 @@ def _to_rest_object(self) -> ComputeResource: idle_time_before_shutdown=self.idle_time_before_shutdown, ) compute_instance_prop.schedules = self.schedules._to_rest_object() if self.schedules else None + compute_instance_prop.setup_scripts = self.setup_scripts._to_rest_object() if self.setup_scripts else None compute_instance = CIRest( description=self.description, compute_type=self.type, @@ -318,6 +325,9 @@ def _load_from_rest(cls, rest_obj: ComputeResource) -> "ComputeInstance": if prop.properties and prop.properties.schedules and prop.properties.schedules.compute_start_stop else None, identity=IdentityConfiguration._from_rest_object(rest_obj.identity) if rest_obj.identity else None, + setup_scripts=SetupScripts._from_rest_object(prop.properties.setup_scripts) + if prop.properties and prop.properties.setup_scripts + else None, ) return response diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_deployment/code_configuration.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_deployment/code_configuration.py index b09dacb52283..3fc3da9f272b 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_deployment/code_configuration.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_deployment/code_configuration.py @@ -6,12 +6,13 @@ from azure.ai.ml._restclient.v2021_10_01.models import CodeConfiguration as RestCodeConfiguration from azure.ai.ml.entities._assets import Code +from azure.ai.ml.entities._mixins import DictMixin from azure.ai.ml.exceptions import ErrorCategory, ErrorTarget, ValidationException module_logger = logging.getLogger(__name__) -class CodeConfiguration: +class CodeConfiguration(DictMixin): """CodeConfiguration. :param code: Code entity, defaults to None diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/identity.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/identity.py index ef6b26663eb2..98ac17479d31 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/identity.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/identity.py @@ -61,7 +61,7 @@ def _from_rest_object(cls, obj: RestAmlToken) -> "AmlToken": return cls() -class ManagedIdentity(Identity, DictMixin): +class ManagedIdentity(Identity): """Managed identity configuration. :param client_id: Specifies a user-assigned identity by client ID. For system-assigned, do not diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/pipeline/_io.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/pipeline/_io.py index 6c9bab2d6e31..656704155595 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/pipeline/_io.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/pipeline/_io.py @@ -168,6 +168,9 @@ def _data_binding(self) -> str: """Return data binding string representation for this input/output.""" raise NotImplementedError() + # Why did we have this function? It prevents the DictMixin from being applied. + # Unclear if we explicitly do NOT want the mapping protocol to be applied to this, or it this was just + # confirmation that it didn't at the time. def keys(self): # This property is introduced to raise catchable Exception in marshmallow mapping validation trial. raise TypeError(f"'{type(self).__name__}' object is not a mapping") diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_mixins.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_mixins.py index c0effd84a0d6..053cfaa45d2c 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_mixins.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_mixins.py @@ -17,6 +17,12 @@ def _from_rest_object(cls, obj: Any) -> Any: class DictMixin(object): + def __contains__(self, item): + return self.__dict__.__contains__(item) + + def __iter__(self): + return self.__dict__.__iter__() + def __setitem__(self, key, item): # type: (Any, Any) -> None self.__dict__[key] = item @@ -29,10 +35,6 @@ def __repr__(self): # type: () -> str return str(self) - def __len__(self): - # type: () -> int - return len(self.keys()) - def __delitem__(self, key): # type: (Any) -> None self.__dict__[key] = None @@ -79,6 +81,10 @@ def get(self, key, default=None): return self.__dict__[key] return default + def __len__(self): + # type: () -> int + return len(self.keys()) + class TelemetryMixin: def _get_telemetry_values(self, *args, **kwargs): # pylint: disable=unused-argument From 9d0e72939c55c52cdea0001e7acf0e88dfd51dc3 Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Wed, 21 Sep 2022 12:05:41 -0400 Subject: [PATCH 2/5] sync with closed source comments --- sdk/ml/azure-ai-ml/azure/ai/ml/entities/_mixins.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_mixins.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_mixins.py index 053cfaa45d2c..2497849a7140 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_mixins.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_mixins.py @@ -35,6 +35,10 @@ def __repr__(self): # type: () -> str return str(self) + def __len__(self): + # type: () -> int + return len(self.keys()) + def __delitem__(self, key): # type: (Any) -> None self.__dict__[key] = None @@ -81,10 +85,6 @@ def get(self, key, default=None): return self.__dict__[key] return default - def __len__(self): - # type: () -> int - return len(self.keys()) - class TelemetryMixin: def _get_telemetry_values(self, *args, **kwargs): # pylint: disable=unused-argument From 7e4c4942bf52a00a9f4210ef397870beea6900c3 Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Thu, 6 Oct 2022 18:32:02 -0400 Subject: [PATCH 3/5] fix merge issues --- sdk/ml/azure-ai-ml/CHANGELOG.md | 1 - .../ai/ml/entities/_compute/_identity.py | 56 --------- .../azure/ai/ml/entities/_job/identity.py | 118 ------------------ 3 files changed, 175 deletions(-) delete mode 100644 sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/_identity.py delete mode 100644 sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/identity.py diff --git a/sdk/ml/azure-ai-ml/CHANGELOG.md b/sdk/ml/azure-ai-ml/CHANGELOG.md index d789586e89b2..2b7187597eba 100644 --- a/sdk/ml/azure-ai-ml/CHANGELOG.md +++ b/sdk/ml/azure-ai-ml/CHANGELOG.md @@ -75,7 +75,6 @@ - Remove invalid option from create_or_update typehints. - Change error returned by (begin_)create_or_update invalid input to TypeError. - Rename set_image_model APIs for all vision tasks to set_training_parameters - - JobOperations.download no longer provides a default value for download_path - JobOperations.download defaults to "." instead of Path.cwd() - Workspace.list_keys renamed to Workspace.get_keys. diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/_identity.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/_identity.py deleted file mode 100644 index dae8d411e634..000000000000 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/_identity.py +++ /dev/null @@ -1,56 +0,0 @@ -# --------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# --------------------------------------------------------- - -from typing import List - -from azure.ai.ml._restclient.v2022_01_01_preview.models import Identity as RestIdentity -from azure.ai.ml._utils.utils import camel_to_snake, snake_to_pascal -from azure.ai.ml.entities._mixins import RestTranslatableMixin -from azure.ai.ml.entities._mixins import DictMixin - -from ._user_assigned_identity import UserAssignedIdentity - - -class IdentityConfiguration(RestTranslatableMixin, DictMixin): - """Managed identity specification.""" - - def __init__(self, *, type: str, user_assigned_identities: List[UserAssignedIdentity] = None): - """Managed identity specification. - - :param type: Managed identity type, defaults to None - :type type: str, optional - :param user_assigned_identities: List of UserAssignedIdentity objects. - :type user_assigned_identities: list, optional - """ - - self.type = type - self.user_assigned_identities = user_assigned_identities - self.principal_id = None - self.tenant_id = None - - def _to_rest_object(self) -> RestIdentity: - rest_user_assigned_identities = ( - {uai.resource_id: uai._to_rest_object() for uai in self.user_assigned_identities} - if self.user_assigned_identities - else None - ) - return RestIdentity(type=snake_to_pascal(self.type), user_assigned_identities=rest_user_assigned_identities) - - @classmethod - def _from_rest_object(cls, rest_obj: RestIdentity) -> "IdentityConfiguration": - from_rest_user_assigned_identities = ( - [ - UserAssignedIdentity._from_rest_object(uai, resource_id=resource_id) - for (resource_id, uai) in rest_obj.user_assigned_identities.items() - ] - if rest_obj.user_assigned_identities - else None - ) - result = cls( - type=camel_to_snake(rest_obj.type), - user_assigned_identities=from_rest_user_assigned_identities, - ) - result.principal_id = rest_obj.principal_id - result.tenant_id = rest_obj.tenant_id - return result diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/identity.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/identity.py deleted file mode 100644 index 98ac17479d31..000000000000 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/identity.py +++ /dev/null @@ -1,118 +0,0 @@ -# --------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# --------------------------------------------------------- - -import logging -from abc import ABC - -from azure.ai.ml._restclient.v2022_06_01_preview.models import AmlToken as RestAmlToken -from azure.ai.ml._restclient.v2022_06_01_preview.models import IdentityConfiguration, IdentityConfigurationType -from azure.ai.ml._restclient.v2022_06_01_preview.models import ManagedIdentity as RestManagedIdentity -from azure.ai.ml._restclient.v2022_06_01_preview.models import UserIdentity as RestUserIdentity -from azure.ai.ml._utils.utils import camel_to_snake -from azure.ai.ml.entities._mixins import DictMixin, RestTranslatableMixin -from azure.ai.ml.exceptions import ErrorCategory, ErrorTarget, JobException - -module_logger = logging.getLogger(__name__) - - -class Identity(ABC, RestTranslatableMixin): - def __init__(self): - self.type = None - - @classmethod - def _from_rest_object(cls, obj: IdentityConfiguration) -> "Identity": - mapping = { - IdentityConfigurationType.AML_TOKEN: AmlToken, - IdentityConfigurationType.MANAGED: ManagedIdentity, - IdentityConfigurationType.USER_IDENTITY: UserIdentity, - } - - identity_class = mapping.get(obj.identity_type, None) - if identity_class: - return identity_class._from_rest_object(obj) - else: - msg = f"Unknown identity type: {obj.identity_type}" - raise JobException( - message=msg, - no_personal_data_message=msg, - target=ErrorTarget.IDENTITY, - error_category=ErrorCategory.SYSTEM_ERROR, - ) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Identity): - return NotImplemented - return self._to_rest_object() == other._to_rest_object() - - -class AmlToken(Identity): - """AML Token identity configuration.""" - - def __init__(self): - self.type = camel_to_snake(IdentityConfigurationType.AML_TOKEN) - - def _to_rest_object(self) -> RestAmlToken: - return RestAmlToken() - - @classmethod - # pylint: disable=unused-argument - def _from_rest_object(cls, obj: RestAmlToken) -> "AmlToken": - return cls() - - -class ManagedIdentity(Identity): - """Managed identity configuration. - - :param client_id: Specifies a user-assigned identity by client ID. For system-assigned, do not - set this field. - :type client_id: str - :param object_id: Specifies a user-assigned identity by object ID. For system-assigned, do not - set this field. - :type object_id: str - :param msi_resource_id: Specifies a user-assigned identity by ARM resource ID. For system-assigned, - do not set this field. - :type msi_resource_id: str - """ - - def __init__( - self, - *, - client_id: str = None, - object_id: str = None, - msi_resource_id: str = None, - ): - self.type = camel_to_snake(IdentityConfigurationType.MANAGED) - self.client_id = client_id - self.object_id = object_id - self.msi_resource_id = msi_resource_id - - def _to_rest_object(self) -> RestManagedIdentity: - return RestManagedIdentity( - client_id=self.client_id, - object_id=self.object_id, - resource_id=self.msi_resource_id, - ) - - @classmethod - def _from_rest_object(cls, obj: RestManagedIdentity) -> "ManagedIdentity": - return cls( - client_id=obj.client_id, - object_id=obj.client_id, - msi_resource_id=obj.resource_id, - ) - - -class UserIdentity(Identity): - """User identity configuration.""" - - def __init__(self): - self.type = camel_to_snake(IdentityConfigurationType.USER_IDENTITY) - - def _to_rest_object(self) -> RestUserIdentity: - return RestUserIdentity() - - @classmethod - # pylint: disable=unused-argument - def _from_rest_object(cls, obj: RestUserIdentity) -> "UserIdentity": - return cls() From ee19e235b93bf7a80efe09c5d0cbd0909b625bd0 Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Thu, 6 Oct 2022 18:34:48 -0400 Subject: [PATCH 4/5] continue to fix changelog file merge issues --- sdk/ml/azure-ai-ml/CHANGELOG.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sdk/ml/azure-ai-ml/CHANGELOG.md b/sdk/ml/azure-ai-ml/CHANGELOG.md index 26fc59a42136..f18a6e8131ed 100644 --- a/sdk/ml/azure-ai-ml/CHANGELOG.md +++ b/sdk/ml/azure-ai-ml/CHANGELOG.md @@ -3,7 +3,6 @@ ## 0.2.0 (Unreleased) ### Features Added - - Added a `show_progress` parameter to MLClient for enable/disable progress bars of long running operations. ### Breaking Changes @@ -17,6 +16,7 @@ ### Features Added - Most configuration classes from the entity package now implement the standard mapping protocol. + ### Breaking Changes - OnlineDeploymentOperations.delete has been renamed to begin_delete. - Datastore credentials are switched to use unified credential configuration classes. @@ -53,7 +53,6 @@ ### Other Changes - Removed declaration on Python 3.6 support - Added support for custom setup scripts on compute instances. - - Removed declaration on Python 3.6 support. - Updated dependencies upper bounds to be major versions. ## 0.1.0b7 (2022-09-22) @@ -77,7 +76,6 @@ - Change error returned by (begin_)create_or_update invalid input to TypeError. - Rename set_image_model APIs for all vision tasks to set_training_parameters - JobOperations.download defaults to "." instead of Path.cwd() - - Workspace.list_keys renamed to Workspace.get_keys. ### Bugs Fixed From 5e186b9599284f86a37cefc5cb5d6474f8cf5fd1 Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Fri, 7 Oct 2022 20:19:33 -0400 Subject: [PATCH 5/5] move CL entry to correct section --- sdk/ml/azure-ai-ml/CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk/ml/azure-ai-ml/CHANGELOG.md b/sdk/ml/azure-ai-ml/CHANGELOG.md index f18a6e8131ed..385b51093346 100644 --- a/sdk/ml/azure-ai-ml/CHANGELOG.md +++ b/sdk/ml/azure-ai-ml/CHANGELOG.md @@ -3,7 +3,7 @@ ## 0.2.0 (Unreleased) ### Features Added - + - Most configuration classes from the entity package now implement the standard mapping protocol. ### Breaking Changes ### Bugs Fixed @@ -15,7 +15,6 @@ - Dropped support for Python 3.6. The Python versions supported for this release are 3.7-3.10. ### Features Added - - Most configuration classes from the entity package now implement the standard mapping protocol. ### Breaking Changes - OnlineDeploymentOperations.delete has been renamed to begin_delete.