From e5b5a6f07a2f0b043d775db8f6336de5acf79341 Mon Sep 17 00:00:00 2001 From: TonyJ1 <41344586+TonyJ1@users.noreply.github.com> Date: Fri, 14 Oct 2022 23:04:54 -0700 Subject: [PATCH 1/4] [ML] Validate the values of JobService.job_service_type passed to Command() and use snake_case --- .../azure/ai/ml/constants/_job/job.py | 23 +++++++++++++++ .../azure/ai/ml/entities/_job/job_service.py | 29 +++++++++++++++++-- .../dsl/unittests/test_command_builder.py | 12 ++++---- .../tests/dsl/unittests/test_dsl_pipeline.py | 8 ++--- .../unittests/test_pipeline_job_schema.py | 16 +++++----- ...t_compute_instance_stop_start_restart.json | 8 ++--- ...oworld_pipeline_job_with_node_services.yml | 10 +++---- ...line_job_with_node_services_inline_job.yml | 10 +++---- 8 files changed, 81 insertions(+), 35 deletions(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/constants/_job/job.py b/sdk/ml/azure-ai-ml/azure/ai/ml/constants/_job/job.py index 0fa85a6035ff..3a0224fa44af 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/constants/_job/job.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/constants/_job/job.py @@ -72,3 +72,26 @@ class RestSparkConfKey: DYNAMIC_ALLOCATION_MIN_EXECUTORS = "spark.dynamicAllocation.minExecutors" DYNAMIC_ALLOCATION_MAX_EXECUTORS = "spark.dynamicAllocation.maxExecutors" DYNAMIC_ALLOCATION_ENABLED = "spark.dynamicAllocation.enabled" + + +class JobServiceTypeNames: + class EntityNames: + JUPYTER_LAB = "jupyter_lab" + SSH = "ssh" + TENSOR_BOARD = "tensor_board" + VS_CODE = "vs_code" + + class RestNames: + JUPYTER_LAB = "JupyterLab" + SSH = "SSH" + TENSOR_BOARD = "TensorBoard" + VS_CODE = "VSCode" + + ENTITY_TO_REST = { + EntityNames.JUPYTER_LAB: RestNames.JUPYTER_LAB, + EntityNames.SSH: RestNames.SSH, + EntityNames.TENSOR_BOARD: RestNames.TENSOR_BOARD, + EntityNames.VS_CODE: RestNames.VS_CODE, + } + + REST_TO_ENTITY = {v: k for k, v in ENTITY_TO_REST.items()} diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/job_service.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/job_service.py index 85a73300bdf2..f84386af9f05 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/job_service.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/job_service.py @@ -8,6 +8,7 @@ from azure.ai.ml._restclient.v2022_10_01_preview.models import AllNodes from azure.ai.ml._restclient.v2022_10_01_preview.models import JobService as RestJobService +from azure.ai.ml.constants._job.job import JobServiceTypeNames from azure.ai.ml.entities._mixins import RestTranslatableMixin from azure.ai.ml.exceptions import ErrorCategory, ErrorTarget, ValidationErrorType, ValidationException @@ -37,7 +38,9 @@ def __init__( self, *, endpoint: Optional[str] = None, - job_service_type: Optional[Literal["JupyterLab", "SSH", "TensorBoard", "VSCode"]] = None, + # TODO: Is VS Code allowed here? + # TODO: As this is a breaking change, should we still allow the old format (any-case) and deprecate in the next release? + job_service_type: Optional[Literal["jupyter_lab", "ssh", "tensor_board", "vs-code"]] = None, nodes: Optional[Literal["all"]] = None, status: Optional[str] = None, port: Optional[int] = None, @@ -51,11 +54,14 @@ def __init__( self.port = port self.properties = properties self._validate_nodes() + self._validate_job_service_type_name() def _to_rest_object(self) -> RestJobService: return RestJobService( endpoint=self.endpoint, - job_service_type=self.job_service_type, + job_service_type=JobServiceTypeNames.ENTITY_TO_REST.get(self.job_service_type, None) + if self.job_service_type + else None, nodes=AllNodes() if self.nodes else None, status=self.status, port=self.port, @@ -73,6 +79,20 @@ def _validate_nodes(self): error_type=ValidationErrorType.INVALID_VALUE, ) + def _validate_job_service_type_name(self): + if self.job_service_type and not self.job_service_type in JobServiceTypeNames.ENTITY_TO_REST.keys(): + msg = ( + f"job_service_type should be one of " + f"{list(JobServiceTypeNames.ENTITY_TO_REST.keys())}, but received '{self.job_service_type}'." + ) + raise ValidationException( + message=msg, + no_personal_data_message=msg, + target=ErrorTarget.JOB, + error_category=ErrorCategory.USER_ERROR, + error_type=ValidationErrorType.INVALID_VALUE, + ) + @classmethod def _to_rest_job_services(cls, services: Dict[str, "JobService"]) -> Dict[str, RestJobService]: if services is None: @@ -85,7 +105,10 @@ def _to_rest_job_services(cls, services: Dict[str, "JobService"]) -> Dict[str, R def _from_rest_object(cls, obj: RestJobService) -> "JobService": return cls( endpoint=obj.endpoint, - job_service_type=obj.job_service_type, + job_service_type=JobServiceTypeNames.REST_TO_ENTITY.get(obj.job_service_type, None) + if obj.job_service_type + else None, + # obj.job_service_type, # nodes="all" if isinstance(obj.nodes, AllNodes) else None, nodes="all" if obj.nodes else None, status=obj.status, diff --git a/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_command_builder.py b/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_command_builder.py index 26f23004cda2..a083bd75dc3d 100644 --- a/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_command_builder.py +++ b/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_command_builder.py @@ -839,9 +839,9 @@ def test_spark_job_with_additional_conf(self): def test_command_services_nodes(self) -> None: services = { - "my_jupyterlab": {"job_service_type": "JupyterLab", "nodes": "all"}, + "my_jupyterlab": {"job_service_type": "jupyter_lab", "nodes": "all"}, "my_tensorboard": { - "job_service_type": "TensorBoard", + "job_service_type": "tensor_board", "properties": { "logDir": "~/tblog", }, @@ -874,14 +874,14 @@ def test_command_services_nodes(self) -> None: def test_command_services(self) -> None: services = { - "my_jupyter": {"job_service_type": "Jupyter"}, + "my_ssh": {"job_service_type": "ssh"}, "my_tensorboard": { - "job_service_type": "TensorBoard", + "job_service_type": "tensor_board", "properties": { "logDir": "~/tblog", }, }, - "my_jupyterlab": {"job_service_type": "JupyterLab"}, + "my_jupyterlab": {"job_service_type": "jupyter_lab"}, } node = command( name="interactive-command-job", @@ -903,7 +903,7 @@ def test_command_services(self) -> None: assert node_rest_obj["services"] == services # test invalid services - invalid_services_0 = "jupyter" + invalid_services_0 = "ssh" with pytest.raises(ValidationException, match="Services must be a dict"): node = command( name="interactive-command-job", diff --git a/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_dsl_pipeline.py b/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_dsl_pipeline.py index 15666d0d1f9f..77af6cfcafaa 100644 --- a/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_dsl_pipeline.py +++ b/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_dsl_pipeline.py @@ -3700,14 +3700,14 @@ def root_pipeline(component_in_number: int, component_in_path: str): def test_pipeline_with_command_services(self): services = { - "my_jupyter": {"job_service_type": "Jupyter"}, + "my_ssh": {"job_service_type": "ssh"}, "my_tensorboard": { - "job_service_type": "TensorBoard", + "job_service_type": "tensor_board", "properties": { "logDir": "~/tblog", }, }, - "my_jupyterlab": {"job_service_type": "JupyterLab"}, + "my_jupyterlab": {"job_service_type": "jupyter_lab"}, } command_func = command( @@ -3749,7 +3749,7 @@ def sample_pipeline(): assert isinstance(service, JobService) # test set services in pipeline - new_services = {"my_jupyter": {"job_service_type": "Jupyter"}} + new_services = {"my_ssh": {"job_service_type": "ssh"}} @dsl.pipeline() def sample_pipeline_with_new_services(): diff --git a/sdk/ml/azure-ai-ml/tests/pipeline_job/unittests/test_pipeline_job_schema.py b/sdk/ml/azure-ai-ml/tests/pipeline_job/unittests/test_pipeline_job_schema.py index 1f70ada8114c..25a7ed8c30a8 100644 --- a/sdk/ml/azure-ai-ml/tests/pipeline_job/unittests/test_pipeline_job_schema.py +++ b/sdk/ml/azure-ai-ml/tests/pipeline_job/unittests/test_pipeline_job_schema.py @@ -1686,15 +1686,15 @@ def test_command_job_node_services_in_pipeline(self): rest_services = job_rest_obj.properties.jobs["hello_world_component_inline"]["services"] # rest object of node in pipeline should be pure dict assert rest_services == { - "my_jupyter": { - "job_service_type": "Jupyter", + "my_ssh": { + "job_service_type": "ssh", }, "my_tensorboard": { - "job_service_type": "TensorBoard", + "job_service_type": "tensor_board", "properties": {"logDir": "~/tblog"}, }, "my_jupyterlab": { - "job_service_type": "JupyterLab", + "job_service_type": "jupyter_lab", }, } @@ -1710,15 +1710,15 @@ def test_command_job_node_services_in_pipeline_with_no_component(self): # rest object of node in pipeline should be pure dict assert job_rest_obj.properties.jobs["hello_world_component_inline"]["services"] == { - "my_jupyter": { - "job_service_type": "Jupyter", + "my_ssh": { + "job_service_type": "ssh", }, "my_tensorboard": { - "job_service_type": "TensorBoard", + "job_service_type": "tensor_board", "properties": {"logDir": "~/tblog"}, }, "my_jupyterlab": { - "job_service_type": "JupyterLab", + "job_service_type": "jupyter_lab", }, } diff --git a/sdk/ml/azure-ai-ml/tests/recordings/compute/e2etests/test_compute.pyTestComputetest_compute_instance_stop_start_restart.json b/sdk/ml/azure-ai-ml/tests/recordings/compute/e2etests/test_compute.pyTestComputetest_compute_instance_stop_start_restart.json index 4f28fde65638..97e1de25a0b6 100644 --- a/sdk/ml/azure-ai-ml/tests/recordings/compute/e2etests/test_compute.pyTestComputetest_compute_instance_stop_start_restart.json +++ b/sdk/ml/azure-ai-ml/tests/recordings/compute/e2etests/test_compute.pyTestComputetest_compute_instance_stop_start_restart.json @@ -696,7 +696,7 @@ }, "applications": [ { - "displayName": "Jupyter", + "displayName": "ssh", "endpointUri": "https://test322425765248.eastus2euap.instances.azureml.ms/tree/" }, { @@ -949,7 +949,7 @@ }, "applications": [ { - "displayName": "Jupyter", + "displayName": "ssh", "endpointUri": "https://test322425765248.eastus2euap.instances.azureml.ms/tree/" }, { @@ -1237,7 +1237,7 @@ }, "applications": [ { - "displayName": "Jupyter", + "displayName": "ssh", "endpointUri": "https://test322425765248.eastus2euap.instances.azureml.ms/tree/" }, { @@ -1315,4 +1315,4 @@ "Variables": { "compute_name": "test322425765248" } -} +} \ No newline at end of file diff --git a/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services.yml b/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services.yml index bf6cbd170bb4..bc661ff82680 100644 --- a/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services.yml +++ b/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services.yml @@ -18,11 +18,11 @@ jobs: environment: azureml:AzureML-sklearn-0.24-ubuntu18.04-py37-cpu:1 code: "./" services: - "my_jupyter": - job_service_type: "Jupyter" # Jupyter Notebook + "my_ssh": + job_service_type: "Jupyter" "my_tensorboard": - job_service_type: "TensorBoard" + job_service_type: "tensor_board" properties: - logDir: "~/tblog" # where you want to store the TensorBoard output + logDir: "~/tblog" # where you want to store the tensor_board output "my_jupyterlab": - job_service_type: "JupyterLab" + job_service_type: "jupyter_lab" diff --git a/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services_inline_job.yml b/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services_inline_job.yml index 726f8cbd49cd..d51d6a4791d4 100644 --- a/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services_inline_job.yml +++ b/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services_inline_job.yml @@ -13,11 +13,11 @@ jobs: command: echo Hello World & sleep 1h environment: azureml:AzureML-sklearn-0.24-ubuntu18.04-py37-cpu:1 services: - "my_jupyter": - job_service_type: "Jupyter" # Jupyter Notebook + "my_ssh": + job_service_type: "Jupyter" "my_tensorboard": - job_service_type: "TensorBoard" + job_service_type: "tensor_board" properties: - logDir: "~/tblog" # where you want to store the TensorBoard output + logDir: "~/tblog" # where you want to store the tensor_board output "my_jupyterlab": - job_service_type: "JupyterLab" + job_service_type: "jupyter_lab" From 2e9e79ff9f7d25bf8515e2caa13476c0d2bd1cf5 Mon Sep 17 00:00:00 2001 From: TonyJ1 <41344586+TonyJ1@users.noreply.github.com> Date: Thu, 20 Oct 2022 14:20:09 -0700 Subject: [PATCH 2/4] Fix types list --- sdk/ml/azure-ai-ml/azure/ai/ml/constants/_job/job.py | 11 +++++++++++ .../azure/ai/ml/entities/_job/job_service.py | 6 ++---- ...mputetest_compute_instance_stop_start_restart.json | 8 -------- .../helloworld_pipeline_job_with_node_services.yml | 2 -- ...rld_pipeline_job_with_node_services_inline_job.yml | 2 -- 5 files changed, 13 insertions(+), 16 deletions(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/constants/_job/job.py b/sdk/ml/azure-ai-ml/azure/ai/ml/constants/_job/job.py index 3a0224fa44af..0116dfa611f8 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/constants/_job/job.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/constants/_job/job.py @@ -76,18 +76,27 @@ class RestSparkConfKey: class JobServiceTypeNames: class EntityNames: + CUSTOM = "custom" + TRACKING = "tracking" + STUDIO = "studio" JUPYTER_LAB = "jupyter_lab" SSH = "ssh" TENSOR_BOARD = "tensor_board" VS_CODE = "vs_code" class RestNames: + CUSTOM = "Custom" + TRACKING = "Tracking" + STUDIO = "Studio" JUPYTER_LAB = "JupyterLab" SSH = "SSH" TENSOR_BOARD = "TensorBoard" VS_CODE = "VSCode" ENTITY_TO_REST = { + EntityNames.CUSTOM: RestNames.CUSTOM, + EntityNames.TRACKING: RestNames.TRACKING, + EntityNames.STUDIO: RestNames.STUDIO, EntityNames.JUPYTER_LAB: RestNames.JUPYTER_LAB, EntityNames.SSH: RestNames.SSH, EntityNames.TENSOR_BOARD: RestNames.TENSOR_BOARD, @@ -95,3 +104,5 @@ class RestNames: } REST_TO_ENTITY = {v: k for k, v in ENTITY_TO_REST.items()} + + NAMES_ALLOWED_FOR_PUBLIC = [EntityNames.JUPYTER_LAB, EntityNames.SSH, EntityNames.TENSOR_BOARD, EntityNames.VS_CODE] diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/job_service.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/job_service.py index f84386af9f05..0adcda51a61c 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/job_service.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/job_service.py @@ -38,9 +38,7 @@ def __init__( self, *, endpoint: Optional[str] = None, - # TODO: Is VS Code allowed here? - # TODO: As this is a breaking change, should we still allow the old format (any-case) and deprecate in the next release? - job_service_type: Optional[Literal["jupyter_lab", "ssh", "tensor_board", "vs-code"]] = None, + job_service_type: Optional[Literal["jupyter_lab", "ssh", "tensor_board", "vs_code"]] = None, nodes: Optional[Literal["all"]] = None, status: Optional[str] = None, port: Optional[int] = None, @@ -83,7 +81,7 @@ def _validate_job_service_type_name(self): if self.job_service_type and not self.job_service_type in JobServiceTypeNames.ENTITY_TO_REST.keys(): msg = ( f"job_service_type should be one of " - f"{list(JobServiceTypeNames.ENTITY_TO_REST.keys())}, but received '{self.job_service_type}'." + f"{JobServiceTypeNames.NAMES_ALLOWED_FOR_PUBLIC}, but received '{self.job_service_type}'." ) raise ValidationException( message=msg, diff --git a/sdk/ml/azure-ai-ml/tests/recordings/compute/e2etests/test_compute.pyTestComputetest_compute_instance_stop_start_restart.json b/sdk/ml/azure-ai-ml/tests/recordings/compute/e2etests/test_compute.pyTestComputetest_compute_instance_stop_start_restart.json index 97e1de25a0b6..dbdbda3cdb24 100644 --- a/sdk/ml/azure-ai-ml/tests/recordings/compute/e2etests/test_compute.pyTestComputetest_compute_instance_stop_start_restart.json +++ b/sdk/ml/azure-ai-ml/tests/recordings/compute/e2etests/test_compute.pyTestComputetest_compute_instance_stop_start_restart.json @@ -695,10 +695,6 @@ "privateIpAddress": "10.0.0.4" }, "applications": [ - { - "displayName": "ssh", - "endpointUri": "https://test322425765248.eastus2euap.instances.azureml.ms/tree/" - }, { "displayName": "Jupyter Lab", "endpointUri": "https://test322425765248.eastus2euap.instances.azureml.ms/lab" @@ -948,10 +944,6 @@ "privateIpAddress": "10.0.0.4" }, "applications": [ - { - "displayName": "ssh", - "endpointUri": "https://test322425765248.eastus2euap.instances.azureml.ms/tree/" - }, { "displayName": "Jupyter Lab", "endpointUri": "https://test322425765248.eastus2euap.instances.azureml.ms/lab" diff --git a/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services.yml b/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services.yml index bc661ff82680..4c6c52463955 100644 --- a/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services.yml +++ b/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services.yml @@ -18,8 +18,6 @@ jobs: environment: azureml:AzureML-sklearn-0.24-ubuntu18.04-py37-cpu:1 code: "./" services: - "my_ssh": - job_service_type: "Jupyter" "my_tensorboard": job_service_type: "tensor_board" properties: diff --git a/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services_inline_job.yml b/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services_inline_job.yml index d51d6a4791d4..4fd554b04bb7 100644 --- a/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services_inline_job.yml +++ b/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services_inline_job.yml @@ -13,8 +13,6 @@ jobs: command: echo Hello World & sleep 1h environment: azureml:AzureML-sklearn-0.24-ubuntu18.04-py37-cpu:1 services: - "my_ssh": - job_service_type: "Jupyter" "my_tensorboard": job_service_type: "tensor_board" properties: From 9154715fbf344761ba69e57ba5962ac4c2efe031 Mon Sep 17 00:00:00 2001 From: TonyJ1 <41344586+TonyJ1@users.noreply.github.com> Date: Thu, 20 Oct 2022 18:49:22 -0700 Subject: [PATCH 3/4] Fix tests --- .../dsl/unittests/test_command_builder.py | 12 ++++++++++- .../test_dsl_pipeline_with_specific_nodes.py | 20 +++++++++++++++---- .../unittests/test_pipeline_job_schema.py | 12 +++++------ ...stPipelineJobtest_pipeline_job_create.json | 10 +++++----- .../command_job/command_job_inputs_rest.yml | 4 ++-- .../command_job/command_job_rest.yml | 4 ++-- .../command_job_rest_amltoken_identity.yml | 4 ++-- .../command_job_rest_msi_identity.yml | 4 ++-- ...oworld_pipeline_job_with_node_services.yml | 2 ++ ...line_job_with_node_services_inline_job.yml | 2 ++ 10 files changed, 50 insertions(+), 24 deletions(-) diff --git a/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_command_builder.py b/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_command_builder.py index c4b584d04b05..082fa9e92514 100644 --- a/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_command_builder.py +++ b/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_command_builder.py @@ -884,6 +884,16 @@ def test_command_services(self) -> None: }, "my_jupyterlab": {"job_service_type": "jupyter_lab"}, } + rest_services = { + "my_ssh": {"job_service_type": "SSH"}, + "my_tensorboard": { + "job_service_type": "TensorBoard", + "properties": { + "logDir": "~/tblog", + }, + }, + "my_jupyterlab": {"job_service_type": "JupyterLab"}, + } node = command( name="interactive-command-job", description="description", @@ -901,7 +911,7 @@ def test_command_services(self) -> None: assert isinstance(service, JobService) node_rest_obj = node._to_rest_object() - assert node_rest_obj["services"] == services + assert node_rest_obj["services"] == rest_services # test invalid services invalid_services_0 = "ssh" diff --git a/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_dsl_pipeline_with_specific_nodes.py b/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_dsl_pipeline_with_specific_nodes.py index b36708b03210..69bac56e7f3c 100644 --- a/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_dsl_pipeline_with_specific_nodes.py +++ b/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_dsl_pipeline_with_specific_nodes.py @@ -1405,6 +1405,7 @@ def pipeline(number, path): "tags": {}, } } + def test_multi_parallel_components_with_file_input_pipeline_output(self) -> None: components_dir = tests_root_dir / "test_configs/dsl_pipeline/parallel_component_with_file_input" batch_inference1 = load_component(source=str(components_dir / "score.yml")) @@ -1835,7 +1836,17 @@ def train_with_automl_in_pipeline(training_data, target_column_name_input: str): def test_pipeline_with_command_services(self): services = { - "my_jupyter": {"job_service_type": "Jupyter"}, + "my_ssh": {"job_service_type": "ssh"}, + "my_tensorboard": { + "job_service_type": "tensor_board", + "properties": { + "logDir": "~/tblog", + }, + }, + "my_jupyterlab": {"job_service_type": "jupyter_lab"}, + } + rest_services = { + "my_ssh": {"job_service_type": "SSH"}, "my_tensorboard": { "job_service_type": "TensorBoard", "properties": { @@ -1874,7 +1885,7 @@ def sample_pipeline(): assert isinstance(service, JobService) job_rest_obj = pipeline._to_rest_object() - assert job_rest_obj.properties.jobs["node"]["services"] == services + assert job_rest_obj.properties.jobs["node"]["services"] == rest_services recovered_obj = PipelineJob._from_rest_object(job_rest_obj) node_services = recovered_obj.jobs["node"].services @@ -1884,7 +1895,8 @@ def sample_pipeline(): assert isinstance(service, JobService) # test set services in pipeline - new_services = {"my_jupyter": {"job_service_type": "Jupyter"}} + new_services = {"my_jupyter": {"job_service_type": "jupyter_lab"}} + rest_new_services = {"my_jupyter": {"job_service_type": "JupyterLab"}} @dsl.pipeline() def sample_pipeline_with_new_services(): @@ -1899,7 +1911,7 @@ def sample_pipeline_with_new_services(): assert isinstance(service, JobService) job_rest_obj = pipeline._to_rest_object() - assert job_rest_obj.properties.jobs["node"]["services"] == new_services + assert job_rest_obj.properties.jobs["node"]["services"] == rest_new_services def test_pipeline_with_pipeline_component_entity(self): path = "./tests/test_configs/components/helloworld_component.yml" diff --git a/sdk/ml/azure-ai-ml/tests/pipeline_job/unittests/test_pipeline_job_schema.py b/sdk/ml/azure-ai-ml/tests/pipeline_job/unittests/test_pipeline_job_schema.py index 68777819e104..6a66e8bffb7e 100644 --- a/sdk/ml/azure-ai-ml/tests/pipeline_job/unittests/test_pipeline_job_schema.py +++ b/sdk/ml/azure-ai-ml/tests/pipeline_job/unittests/test_pipeline_job_schema.py @@ -1687,14 +1687,14 @@ def test_command_job_node_services_in_pipeline(self): # rest object of node in pipeline should be pure dict assert rest_services == { "my_ssh": { - "job_service_type": "ssh", + "job_service_type": "SSH", }, "my_tensorboard": { - "job_service_type": "tensor_board", + "job_service_type": "TensorBoard", "properties": {"logDir": "~/tblog"}, }, "my_jupyterlab": { - "job_service_type": "jupyter_lab", + "job_service_type": "JupyterLab", }, } @@ -1711,14 +1711,14 @@ def test_command_job_node_services_in_pipeline_with_no_component(self): # rest object of node in pipeline should be pure dict assert job_rest_obj.properties.jobs["hello_world_component_inline"]["services"] == { "my_ssh": { - "job_service_type": "ssh", + "job_service_type": "SSH", }, "my_tensorboard": { - "job_service_type": "tensor_board", + "job_service_type": "TensorBoard", "properties": {"logDir": "~/tblog"}, }, "my_jupyterlab": { - "job_service_type": "jupyter_lab", + "job_service_type": "JupyterLab", }, } diff --git a/sdk/ml/azure-ai-ml/tests/recordings/pipeline_job/e2etests/test_pipeline_job.pyTestPipelineJobtest_pipeline_job_create.json b/sdk/ml/azure-ai-ml/tests/recordings/pipeline_job/e2etests/test_pipeline_job.pyTestPipelineJobtest_pipeline_job_create.json index faeadeed82a3..512a12738162 100644 --- a/sdk/ml/azure-ai-ml/tests/recordings/pipeline_job/e2etests/test_pipeline_job.pyTestPipelineJobtest_pipeline_job_create.json +++ b/sdk/ml/azure-ai-ml/tests/recordings/pipeline_job/e2etests/test_pipeline_job.pyTestPipelineJobtest_pipeline_job_create.json @@ -647,7 +647,7 @@ "experimentName": "my_first_experiment", "services": { "Tracking": { - "jobServiceType": "Tracking", + "jobServiceType": "tracking", "port": null, "endpoint": "azureml://eastus2euap.api.azureml.ms/mlflow/v1.0/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000?", "status": null, @@ -655,7 +655,7 @@ "properties": null }, "Studio": { - "jobServiceType": "Studio", + "jobServiceType": "studio", "port": null, "endpoint": "https://ml.azure.com/runs/test_917642482668?wsid=/subscriptions/00000000-0000-0000-0000-000000000/resourcegroups/00000/workspaces/00000", "status": null, @@ -1174,7 +1174,7 @@ "experimentName": "my_first_experiment", "services": { "Tracking": { - "jobServiceType": "Tracking", + "jobServiceType": "tracking", "port": null, "endpoint": "azureml://eastus2euap.api.azureml.ms/mlflow/v1.0/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000?", "status": null, @@ -1182,7 +1182,7 @@ "properties": null }, "Studio": { - "jobServiceType": "Studio", + "jobServiceType": "studio", "port": null, "endpoint": "https://ml.azure.com/runs/test_917642482668?wsid=/subscriptions/00000000-0000-0000-0000-000000000/resourcegroups/00000/workspaces/00000", "status": null, @@ -1294,4 +1294,4 @@ "new_tag_name": "test_907076112763", "new_tag_value": "test_34294107164" } -} +} \ No newline at end of file diff --git a/sdk/ml/azure-ai-ml/tests/test_configs/command_job/command_job_inputs_rest.yml b/sdk/ml/azure-ai-ml/tests/test_configs/command_job/command_job_inputs_rest.yml index be9a545bef83..ec4b84057125 100644 --- a/sdk/ml/azure-ai-ml/tests/test_configs/command_job/command_job_inputs_rest.yml +++ b/sdk/ml/azure-ai-ml/tests/test_configs/command_job/command_job_inputs_rest.yml @@ -7,11 +7,11 @@ "provisioning_state":"Succeeded", "services":{ "Tracking":{ - "type":"Tracking", + "type":"tracking", "endpoint":"azureml://master.api.azureml-test.ms/mlflow/v1.0/subscriptions/d511f82f-71ba-49a4-8233-d7be8a3650f4/resourceGroups/RLTesting/providers/Microsoft.MachineLearningServices/workspaces/AnkitWS?" }, "Studio":{ - "type":"Studio", + "type":"studio", "endpoint":"https://ml.azure.com/experiments/mfe-test1/runs/test_773798882091?wsid=/subscriptions/d511f82f-71ba-49a4-8233-d7be8a3650f4/resourcegroups/RLTesting/workspaces/AnkitWS" } }, diff --git a/sdk/ml/azure-ai-ml/tests/test_configs/command_job/command_job_rest.yml b/sdk/ml/azure-ai-ml/tests/test_configs/command_job/command_job_rest.yml index 19a057273596..5c864f43f1a5 100644 --- a/sdk/ml/azure-ai-ml/tests/test_configs/command_job/command_job_rest.yml +++ b/sdk/ml/azure-ai-ml/tests/test_configs/command_job/command_job_rest.yml @@ -7,11 +7,11 @@ "provisioning_state":"Succeeded", "services":{ "Tracking":{ - "type":"Tracking", + "type":"tracking", "endpoint":"azureml://master.api.azureml-test.ms/mlflow/v1.0/subscriptions/d511f82f-71ba-49a4-8233-d7be8a3650f4/resourceGroups/RLTesting/providers/Microsoft.MachineLearningServices/workspaces/AnkitWS?" }, "Studio":{ - "type":"Studio", + "type":"studio", "endpoint":"https://ml.azure.com/experiments/mfe-test1/runs/test_617704734544?wsid=/subscriptions/d511f82f-71ba-49a4-8233-d7be8a3650f4/resourcegroups/RLTesting/workspaces/AnkitWS" } }, diff --git a/sdk/ml/azure-ai-ml/tests/test_configs/command_job/command_job_rest_amltoken_identity.yml b/sdk/ml/azure-ai-ml/tests/test_configs/command_job/command_job_rest_amltoken_identity.yml index b316853e4a35..22d5cb7cb4b1 100644 --- a/sdk/ml/azure-ai-ml/tests/test_configs/command_job/command_job_rest_amltoken_identity.yml +++ b/sdk/ml/azure-ai-ml/tests/test_configs/command_job/command_job_rest_amltoken_identity.yml @@ -7,11 +7,11 @@ "provisioning_state":"Succeeded", "services":{ "Tracking":{ - "type":"Tracking", + "type":"tracking", "endpoint":"azureml://master.api.azureml-test.ms/mlflow/v1.0/subscriptions/d511f82f-71ba-49a4-8233-d7be8a3650f4/resourceGroups/RLTesting/providers/Microsoft.MachineLearningServices/workspaces/AnkitWS?" }, "Studio":{ - "type":"Studio", + "type":"studio", "endpoint":"https://ml.azure.com/experiments/mfe-test1/runs/test_617704734544?wsid=/subscriptions/d511f82f-71ba-49a4-8233-d7be8a3650f4/resourcegroups/RLTesting/workspaces/AnkitWS" } }, diff --git a/sdk/ml/azure-ai-ml/tests/test_configs/command_job/command_job_rest_msi_identity.yml b/sdk/ml/azure-ai-ml/tests/test_configs/command_job/command_job_rest_msi_identity.yml index b4cab3a9a142..eb6eb0d6509f 100644 --- a/sdk/ml/azure-ai-ml/tests/test_configs/command_job/command_job_rest_msi_identity.yml +++ b/sdk/ml/azure-ai-ml/tests/test_configs/command_job/command_job_rest_msi_identity.yml @@ -7,11 +7,11 @@ "provisioning_state":"Succeeded", "services":{ "Tracking":{ - "type":"Tracking", + "type":"tracking", "endpoint":"azureml://master.api.azureml-test.ms/mlflow/v1.0/subscriptions/d511f82f-71ba-49a4-8233-d7be8a3650f4/resourceGroups/RLTesting/providers/Microsoft.MachineLearningServices/workspaces/AnkitWS?" }, "Studio":{ - "type":"Studio", + "type":"studio", "endpoint":"https://ml.azure.com/experiments/mfe-test1/runs/test_297155932351?wsid=/subscriptions/d511f82f-71ba-49a4-8233-d7be8a3650f4/resourcegroups/RLTesting/workspaces/AnkitWS" } }, diff --git a/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services.yml b/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services.yml index 4c6c52463955..c6fc0a0d3913 100644 --- a/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services.yml +++ b/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services.yml @@ -18,6 +18,8 @@ jobs: environment: azureml:AzureML-sklearn-0.24-ubuntu18.04-py37-cpu:1 code: "./" services: + "my_ssh": + job_service_type: "ssh" "my_tensorboard": job_service_type: "tensor_board" properties: diff --git a/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services_inline_job.yml b/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services_inline_job.yml index 4fd554b04bb7..55c9c56b37ea 100644 --- a/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services_inline_job.yml +++ b/sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_node_services_inline_job.yml @@ -13,6 +13,8 @@ jobs: command: echo Hello World & sleep 1h environment: azureml:AzureML-sklearn-0.24-ubuntu18.04-py37-cpu:1 services: + "my_ssh": + job_service_type: "ssh" "my_tensorboard": job_service_type: "tensor_board" properties: From 7b354b89c8ea81c8dc54fe7ed106e7b285ca727b Mon Sep 17 00:00:00 2001 From: TonyJ1 <41344586+TonyJ1@users.noreply.github.com> Date: Fri, 21 Oct 2022 00:55:07 -0700 Subject: [PATCH 4/4] Update changelog.md --- sdk/ml/azure-ai-ml/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/ml/azure-ai-ml/CHANGELOG.md b/sdk/ml/azure-ai-ml/CHANGELOG.md index edd8ea1f8005..ebde8402cda1 100644 --- a/sdk/ml/azure-ai-ml/CHANGELOG.md +++ b/sdk/ml/azure-ai-ml/CHANGELOG.md @@ -6,6 +6,7 @@ - Registry list operation now accepts scope value to allow subscription-only based requests. - Most configuration classes from the entity package now implement the standard mapping protocol. - Add registry delete operation. +- The values of JobService.job_service_type are now using the snake case. e.g jupyter_lab, ssh, tensor_board, vs_code. ### Breaking Changes