From 2bb1e4a984d76465c0b26da29c489cd9e08943ef Mon Sep 17 00:00:00 2001 From: awaelchli Date: Thu, 29 Sep 2022 00:04:04 +0200 Subject: [PATCH 1/8] message --- src/lightning_app/runners/cloud.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/lightning_app/runners/cloud.py b/src/lightning_app/runners/cloud.py index 38083d95076a4..0731126f957f1 100644 --- a/src/lightning_app/runners/cloud.py +++ b/src/lightning_app/runners/cloud.py @@ -39,6 +39,7 @@ ) from lightning_cloud.openapi.rest import ApiException +from lightning_app import LightningWork from lightning_app.core.app import LightningApp from lightning_app.core.constants import CLOUD_UPLOAD_WARNING, DISABLE_DEPENDENCY_CACHE from lightning_app.runners.backends.cloud import CloudBackend @@ -112,6 +113,7 @@ def dispatch( for flow in self.app.flows: for work in flow.works(recurse=False): work_requirements = "\n".join(work.cloud_build_config.requirements) + _validate_build_spec_and_compute(work) build_spec = V1BuildSpec( commands=work.cloud_build_config.build_commands(), python_dependencies=V1PythonDependencyInfo( @@ -361,3 +363,12 @@ def _project_has_sufficient_credits(self, project: V1Membership, app: Optional[L balance = 0 # value is missing in some tests return balance >= 1 + + +def _validate_build_spec_and_compute(work: LightningWork) -> None: + if work.cloud_build_config.image is not None and work.cloud_compute.name == "default": + raise ValueError( + f"You requested a custom base image for the Work with name '{work.name}', but custom images are currently" + " not supported on the default cloud compute instance. Please choose a different configuration, for example" + " `CloudCompute('cpu-medium')`." + ) From b9d8204cbfff46dfce21ef3496d09529a436f74e Mon Sep 17 00:00:00 2001 From: awaelchli Date: Thu, 29 Sep 2022 00:12:34 +0200 Subject: [PATCH 2/8] test --- tests/tests_app/runners/test_cloud.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/tests_app/runners/test_cloud.py b/tests/tests_app/runners/test_cloud.py index fcf53d92b7f00..e7f796acb288e 100644 --- a/tests/tests_app/runners/test_cloud.py +++ b/tests/tests_app/runners/test_cloud.py @@ -30,8 +30,9 @@ V1Work, ) -from lightning_app import LightningApp, LightningWork +from lightning_app import LightningApp, LightningWork, CloudCompute, BuildConfig, LightningFlow from lightning_app.runners import backends, cloud +from lightning_app.runners.cloud import _validate_build_spec_and_compute from lightning_app.storage import Drive from lightning_app.utilities.cloud import _get_project from lightning_app.utilities.dependency_caching import get_hash @@ -702,3 +703,20 @@ def test_project_has_sufficient_credits(): for balance, result in credits_and_test_value: project = V1Membership(name="test-project1", project_id="test-project-id1", balance=balance) assert cloud_runtime._project_has_sufficient_credits(project) is result + + +def test_incompatible_cloud_compute_and_build_config(): + """Test that an exception is raised when a build config has a custom image defined, but the cloud compute is the + default. This combination is not supported by the platform.""" + + class Work(LightningWork): + def __init__(self): + super().__init__() + self.cloud_compute = CloudCompute(name="default") + self.cloud_build_config = BuildConfig(image="custom") + + def run(self): + pass + + with pytest.raises(ValueError, match="You requested a custom base image for the Work with name"): + _validate_build_spec_and_compute(Work()) From ebfee4c050640fea812b40219468422f55c1d510 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Sep 2022 22:17:36 +0000 Subject: [PATCH 3/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/tests_app/runners/test_cloud.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/tests_app/runners/test_cloud.py b/tests/tests_app/runners/test_cloud.py index e7f796acb288e..c2d82c945a10b 100644 --- a/tests/tests_app/runners/test_cloud.py +++ b/tests/tests_app/runners/test_cloud.py @@ -30,7 +30,7 @@ V1Work, ) -from lightning_app import LightningApp, LightningWork, CloudCompute, BuildConfig, LightningFlow +from lightning_app import BuildConfig, CloudCompute, LightningApp, LightningFlow, LightningWork from lightning_app.runners import backends, cloud from lightning_app.runners.cloud import _validate_build_spec_and_compute from lightning_app.storage import Drive @@ -706,8 +706,11 @@ def test_project_has_sufficient_credits(): def test_incompatible_cloud_compute_and_build_config(): - """Test that an exception is raised when a build config has a custom image defined, but the cloud compute is the - default. This combination is not supported by the platform.""" + """Test that an exception is raised when a build config has a custom image defined, but the cloud compute is + the default. + + This combination is not supported by the platform. + """ class Work(LightningWork): def __init__(self): From 32c87ecd9efaabf4c8dd9a1f10f3729bbfd70229 Mon Sep 17 00:00:00 2001 From: awaelchli Date: Thu, 29 Sep 2022 00:21:36 +0200 Subject: [PATCH 4/8] add changelog --- src/lightning_app/CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lightning_app/CHANGELOG.md b/src/lightning_app/CHANGELOG.md index d7c11179e48fa..18fc262a3dee3 100644 --- a/src/lightning_app/CHANGELOG.md +++ b/src/lightning_app/CHANGELOG.md @@ -15,6 +15,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Add `--secret` option to CLI to allow binding Secrets to app environment variables when running in the cloud ([#14612](https://github.com/Lightning-AI/lightning/pull/14612)) +- Added a friendly error message when attempting to run the default cloud compute with a custom base image configured ([#14929](https://github.com/Lightning-AI/lightning/pull/14929)) + + + ### Changed - From d9a60f0784da4705620714df94a8c8b39d6acbff Mon Sep 17 00:00:00 2001 From: awaelchli Date: Thu, 6 Oct 2022 02:10:07 +0200 Subject: [PATCH 5/8] unused import --- tests/tests_app/runners/test_cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_app/runners/test_cloud.py b/tests/tests_app/runners/test_cloud.py index c2d82c945a10b..d607ba2e7dce0 100644 --- a/tests/tests_app/runners/test_cloud.py +++ b/tests/tests_app/runners/test_cloud.py @@ -30,7 +30,7 @@ V1Work, ) -from lightning_app import BuildConfig, CloudCompute, LightningApp, LightningFlow, LightningWork +from lightning_app import BuildConfig, CloudCompute, LightningApp, LightningWork from lightning_app.runners import backends, cloud from lightning_app.runners.cloud import _validate_build_spec_and_compute from lightning_app.storage import Drive From e4342ebf6b27c5e523df623f6a11a096396cd45e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 8 Nov 2022 18:10:04 +0000 Subject: [PATCH 6/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/tests_app/runners/test_cloud.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/tests_app/runners/test_cloud.py b/tests/tests_app/runners/test_cloud.py index 6f0a83b25e85c..6493787f4885d 100644 --- a/tests/tests_app/runners/test_cloud.py +++ b/tests/tests_app/runners/test_cloud.py @@ -37,14 +37,11 @@ V1Work, ) -from lightning_app import _PROJECT_ROOT -from lightning_app.runners import CloudRuntime -from lightning_app.storage import Mount -from lightning_app.testing.helpers import EmptyFlow -from lightning_app import BuildConfig, LightningApp, LightningWork -from lightning_app.runners import backends, cloud +from lightning_app import _PROJECT_ROOT, BuildConfig, LightningApp, LightningWork +from lightning_app.runners import backends, cloud, CloudRuntime from lightning_app.runners.cloud import _validate_build_spec_and_compute -from lightning_app.storage import Drive +from lightning_app.storage import Drive, Mount +from lightning_app.testing.helpers import EmptyFlow from lightning_app.utilities.cloud import _get_project from lightning_app.utilities.dependency_caching import get_hash from lightning_app.utilities.packaging.cloud_compute import CloudCompute From b10daa0a7f79756a0ece37725e41d12ea4f9201f Mon Sep 17 00:00:00 2001 From: awaelchli Date: Sun, 13 Nov 2022 00:44:38 +0100 Subject: [PATCH 7/8] fix mere error --- src/lightning_app/runners/cloud.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lightning_app/runners/cloud.py b/src/lightning_app/runners/cloud.py index 8c1e886937d4b..ebe54cfb83881 100644 --- a/src/lightning_app/runners/cloud.py +++ b/src/lightning_app/runners/cloud.py @@ -144,9 +144,10 @@ def dispatch( works: List[V1Work] = [] for work in self.app.works: + _validate_build_spec_and_compute(work) + if not work._start_with_flow: continue - _validate_build_spec_and_compute(work) work_requirements = "\n".join(work.cloud_build_config.requirements) build_spec = V1BuildSpec( From 160bb20bf032b7bfcf2f86d314bdff86ba970f4e Mon Sep 17 00:00:00 2001 From: awaelchli Date: Sun, 13 Nov 2022 01:02:58 +0100 Subject: [PATCH 8/8] update tests --- tests/tests_app/runners/test_cloud.py | 74 +++++++++++++-------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/tests/tests_app/runners/test_cloud.py b/tests/tests_app/runners/test_cloud.py index 67508e689d358..e00892e22dce0 100644 --- a/tests/tests_app/runners/test_cloud.py +++ b/tests/tests_app/runners/test_cloud.py @@ -55,8 +55,8 @@ def run(self): class WorkWithSingleDrive(LightningWork): - def __init__(self): - super().__init__() + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.drive = None def run(self): @@ -64,8 +64,8 @@ def run(self): class WorkWithTwoDrives(LightningWork): - def __init__(self): - super().__init__() + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.lit_drive_1 = None self.lit_drive_2 = None @@ -404,13 +404,13 @@ def test_call_with_work_app(self, lightningapps, start_with_flow, monkeypatch, t monkeypatch.setattr(cloud, "_prepare_lightning_wheels_and_requirements", mock.MagicMock()) app = mock.MagicMock() - work = MyWork(start_with_flow=start_with_flow) - monkeypatch.setattr(work, "_name", "test-work") - monkeypatch.setattr(work._cloud_build_config, "build_commands", lambda: ["echo 'start'"]) - monkeypatch.setattr(work._cloud_build_config, "requirements", ["torch==1.0.0", "numpy==1.0.0"]) - monkeypatch.setattr(work._cloud_build_config, "image", "random_base_public_image") - monkeypatch.setattr(work._cloud_compute, "disk_size", 0) - monkeypatch.setattr(work, "_port", 8080) + work = MyWork(start_with_flow=start_with_flow, cloud_compute=CloudCompute("custom")) + work._name = "test-work" + work._cloud_build_config.build_commands = lambda: ["echo 'start'"] + work._cloud_build_config.requirements = ["torch==1.0.0", "numpy==1.0.0"] + work._cloud_build_config.image = "random_base_public_image" + work._cloud_compute.disk_size = 0 + work._port = 8080 app.works = [work] cloud_runtime = cloud.CloudRuntime(app=app, entrypoint_file=(source_code_root_dir / "entrypoint.py")) @@ -451,7 +451,7 @@ def test_call_with_work_app(self, lightningapps, start_with_flow, monkeypatch, t ), drives=[], user_requested_compute_config=V1UserRequestedComputeConfig( - name="default", + name="custom", count=1, disk_size=0, shm_size=0, @@ -586,7 +586,7 @@ def test_call_with_work_app_and_attached_drives(self, lightningapps, monkeypatch # should be the results of the deepcopy operation (an instance of the original class) mocked_drive.__deepcopy__.return_value = copy(mocked_drive) - work = WorkWithSingleDrive() + work = WorkWithSingleDrive(cloud_compute=CloudCompute("custom")) monkeypatch.setattr(work, "drive", mocked_drive) monkeypatch.setattr(work, "_state", {"_port", "drive"}) monkeypatch.setattr(work, "_name", "test-work") @@ -647,7 +647,7 @@ def test_call_with_work_app_and_attached_drives(self, lightningapps, monkeypatch ), ], user_requested_compute_config=V1UserRequestedComputeConfig( - name="default", + name="custom", count=1, disk_size=0, shm_size=0, @@ -710,14 +710,14 @@ def test_call_with_work_app_and_app_comment_command_execution_set(self, lightnin monkeypatch.setattr(cloud, "_prepare_lightning_wheels_and_requirements", mock.MagicMock()) app = mock.MagicMock() - work = MyWork() - monkeypatch.setattr(work, "_state", {"_port"}) - monkeypatch.setattr(work, "_name", "test-work") - monkeypatch.setattr(work._cloud_build_config, "build_commands", lambda: ["echo 'start'"]) - monkeypatch.setattr(work._cloud_build_config, "requirements", ["torch==1.0.0", "numpy==1.0.0"]) - monkeypatch.setattr(work._cloud_build_config, "image", "random_base_public_image") - monkeypatch.setattr(work._cloud_compute, "disk_size", 0) - monkeypatch.setattr(work, "_port", 8080) + work = MyWork(cloud_compute=CloudCompute("custom")) + work._state = {"_port"} + work._name = "test-work" + work._cloud_build_config.build_commands = lambda: ["echo 'start'"] + work._cloud_build_config.requirements = ["torch==1.0.0", "numpy==1.0.0"] + work._cloud_build_config.image = "random_base_public_image" + work._cloud_compute.disk_size = 0 + work._port = 8080 app.works = [work] cloud_runtime = cloud.CloudRuntime(app=app, entrypoint_file=(source_code_root_dir / "entrypoint.py")) @@ -756,7 +756,7 @@ def test_call_with_work_app_and_app_comment_command_execution_set(self, lightnin ), drives=[], user_requested_compute_config=V1UserRequestedComputeConfig( - name="default", count=1, disk_size=0, shm_size=0, preemptible=mock.ANY + name="custom", count=1, disk_size=0, shm_size=0, preemptible=mock.ANY ), network_config=[V1NetworkConfig(name=mock.ANY, host=None, port=8080)], cluster_id=mock.ANY, @@ -836,16 +836,16 @@ def test_call_with_work_app_and_multiple_attached_drives(self, lightningapps, mo # should be the results of the deepcopy operation (an instance of the original class) mocked_lit_drive.__deepcopy__.return_value = copy(mocked_lit_drive) - work = WorkWithTwoDrives() - monkeypatch.setattr(work, "lit_drive_1", mocked_lit_drive) - monkeypatch.setattr(work, "lit_drive_2", mocked_lit_drive) - monkeypatch.setattr(work, "_state", {"_port", "_name", "lit_drive_1", "lit_drive_2"}) - monkeypatch.setattr(work, "_name", "test-work") - monkeypatch.setattr(work._cloud_build_config, "build_commands", lambda: ["echo 'start'"]) - monkeypatch.setattr(work._cloud_build_config, "requirements", ["torch==1.0.0", "numpy==1.0.0"]) - monkeypatch.setattr(work._cloud_build_config, "image", "random_base_public_image") - monkeypatch.setattr(work._cloud_compute, "disk_size", 0) - monkeypatch.setattr(work, "_port", 8080) + work = WorkWithTwoDrives(cloud_compute=CloudCompute("custom")) + work.lit_drive_1 = mocked_lit_drive + work.lit_drive_2 = mocked_lit_drive + work._state = {"_port", "_name", "lit_drive_1", "lit_drive_2"} + work._name = "test-work" + work._cloud_build_config.build_commands = lambda: ["echo 'start'"] + work._cloud_build_config.requirements = ["torch==1.0.0", "numpy==1.0.0"] + work._cloud_build_config.image = "random_base_public_image" + work._cloud_compute.disk_size = 0 + work._port = 8080 app.works = [work] cloud_runtime = cloud.CloudRuntime(app=app, entrypoint_file=(source_code_root_dir / "entrypoint.py")) @@ -915,7 +915,7 @@ def test_call_with_work_app_and_multiple_attached_drives(self, lightningapps, mo ), drives=[lit_drive_2_spec, lit_drive_1_spec], user_requested_compute_config=V1UserRequestedComputeConfig( - name="default", + name="custom", count=1, disk_size=0, shm_size=0, @@ -954,7 +954,7 @@ def test_call_with_work_app_and_multiple_attached_drives(self, lightningapps, mo ), drives=[lit_drive_1_spec, lit_drive_2_spec], user_requested_compute_config=V1UserRequestedComputeConfig( - name="default", + name="custom", count=1, disk_size=0, shm_size=0, @@ -1044,7 +1044,7 @@ def test_call_with_work_app_and_attached_mount_and_drive(self, lightningapps, mo setattr(mocked_mount, "mount_path", "/content/foo") setattr(mocked_mount, "protocol", "s3://") - work = WorkWithSingleDrive() + work = WorkWithSingleDrive(cloud_compute=CloudCompute("custom")) monkeypatch.setattr(work, "drive", mocked_drive) monkeypatch.setattr(work, "_state", {"_port", "drive"}) monkeypatch.setattr(work, "_name", "test-work") @@ -1120,7 +1120,7 @@ def test_call_with_work_app_and_attached_mount_and_drive(self, lightningapps, mo ), ], user_requested_compute_config=V1UserRequestedComputeConfig( - name="default", + name="custom", count=1, disk_size=0, shm_size=0,