From 46ebbbf8a456f93778ade8aaf9ad2437f41bcf55 Mon Sep 17 00:00:00 2001 From: ilongin Date: Fri, 5 Sep 2025 11:50:12 +0200 Subject: [PATCH 1/7] added remove_project public method --- src/datachain/__init__.py | 2 + src/datachain/data_storage/metastore.py | 39 +++++++++++++++++-- src/datachain/error.py | 4 ++ src/datachain/lib/projects.py | 45 +++++++++++++++++++++- tests/unit/lib/test_project.py | 51 +++++++++++++++++++++++++ 5 files changed, 136 insertions(+), 5 deletions(-) diff --git a/src/datachain/__init__.py b/src/datachain/__init__.py index 0ea735829..c6fa7bc4d 100644 --- a/src/datachain/__init__.py +++ b/src/datachain/__init__.py @@ -38,6 +38,7 @@ ) from datachain.lib.model_store import ModelStore from datachain.lib.projects import create as create_project +from datachain.lib.projects import delete as delete_project from datachain.lib.udf import Aggregator, Generator, Mapper from datachain.lib.utils import AbstractUDF, DataChainError from datachain.query import metrics, param @@ -74,6 +75,7 @@ "create_project", "datasets", "delete_dataset", + "delete_project", "is_chain_type", "is_studio", "listings", diff --git a/src/datachain/data_storage/metastore.py b/src/datachain/data_storage/metastore.py index c3548f6d8..48cd3597b 100644 --- a/src/datachain/data_storage/metastore.py +++ b/src/datachain/data_storage/metastore.py @@ -22,6 +22,7 @@ UniqueConstraint, select, ) +from sqlalchemy.sql import func as f from datachain.data_storage import JobQueryType, JobStatus from datachain.data_storage.serializer import Serializable @@ -190,10 +191,22 @@ def get_project( It also creates project if not found and create flag is set to True. """ + @abstractmethod + def is_default_project(self, project_name: str, namespace_name: str) -> bool: + """Checks if some project in namespace is default""" + + @abstractmethod + def is_listing_project(self, project_name: str, namespace_name: str) -> bool: + """Checks if some project in namespace is listing""" + @abstractmethod def get_project_by_id(self, project_id: int, conn=None) -> Project: """Gets a single project by id""" + @abstractmethod + def remove_project(self, project_id: int, conn=None) -> None: + """Removes a single project by id""" + @abstractmethod def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]: """Gets list of projects in some namespace or in general (in all namespaces)""" @@ -270,6 +283,10 @@ def list_datasets( ) -> Iterator[DatasetListRecord]: """Lists all datasets in some project or in all projects.""" + @abstractmethod + def count_datasets(self, project_id: Optional[int] = None) -> int: + """Counts datasets in some project or in all projects.""" + @abstractmethod def list_datasets_by_prefix( self, prefix: str, project_id: Optional[int] = None @@ -796,13 +813,13 @@ def create_project( return self.get_project(name, namespace.name) - def _is_listing_project(self, project_name: str, namespace_name: str) -> bool: + def is_listing_project(self, project_name: str, namespace_name: str) -> bool: return ( project_name == self.listing_project_name and namespace_name == self.system_namespace_name ) - def _is_default_project(self, project_name: str, namespace_name: str) -> bool: + def is_default_project(self, project_name: str, namespace_name: str) -> bool: return ( project_name == self.default_project_name and namespace_name == self.default_namespace_name @@ -816,7 +833,7 @@ def get_project( p = self._projects validate = True - if self._is_listing_project(name, namespace_name) or self._is_default_project( + if self.is_listing_project(name, namespace_name) or self.is_default_project( name, namespace_name ): # we are always creating default and listing projects if they don't exist @@ -858,6 +875,11 @@ def get_project_by_id(self, project_id: int, conn=None) -> Project: raise ProjectNotFoundError(f"Project with id {project_id} not found.") return self.project_class.parse(*rows[0]) + def remove_project(self, project_id: int, conn=None) -> None: + p = self._projects + with self.db.transaction(): + self.db.execute(self._projects_delete().where(p.c.id == project_id)) + def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]: """ Gets a list of projects inside some namespace, or in all namespaces @@ -1189,7 +1211,6 @@ def _base_list_datasets_query(self) -> "Select": def list_datasets( self, project_id: Optional[int] = None ) -> Iterator["DatasetListRecord"]: - """Lists all datasets.""" d = self._datasets query = self._base_list_datasets_query().order_by( self._datasets.c.name, self._datasets_versions.c.version @@ -1198,6 +1219,16 @@ def list_datasets( query = query.where(d.c.project_id == project_id) yield from self._parse_dataset_list(self.db.execute(query)) + def count_datasets(self, project_id: Optional[int] = None) -> int: + d = self._datasets + query = self._base_list_datasets_query() + if project_id: + query = query.where(d.c.project_id == project_id) + + query = select(f.count(1)).select_from(query.subquery()) + + return next(self.db.execute(query))[0] + def list_datasets_by_prefix( self, prefix: str, project_id: Optional[int] = None, conn=None ) -> Iterator["DatasetListRecord"]: diff --git a/src/datachain/error.py b/src/datachain/error.py index 78b1c028a..33301ebac 100644 --- a/src/datachain/error.py +++ b/src/datachain/error.py @@ -34,6 +34,10 @@ class ProjectCreateNotAllowedError(NotAllowedError): pass +class ProjectDeleteNotAllowedError(NotAllowedError): + pass + + class ProjectNotFoundError(NotFoundError): pass diff --git a/src/datachain/lib/projects.py b/src/datachain/lib/projects.py index da8423bfd..c78960ddb 100644 --- a/src/datachain/lib/projects.py +++ b/src/datachain/lib/projects.py @@ -1,6 +1,6 @@ from typing import Optional -from datachain.error import ProjectCreateNotAllowedError +from datachain.error import ProjectCreateNotAllowedError, ProjectDeleteNotAllowedError from datachain.project import Project from datachain.query import Session @@ -86,3 +86,46 @@ def ls( namespace_id = session.catalog.metastore.get_namespace(namespace).id return session.catalog.metastore.list_projects(namespace_id) + + +def delete(name: str, namespace: str, session: Optional[Session]) -> None: + """ + Removes a project by name within a namespace. + + Raises: + ProjectNotFoundError: If the project does not exist. + ProjectDeleteNotAllowedError: If the project is non-empty, + is the default project, or is a listing project, + as these cannot be removed. + + Parameters: + name : The name of the project. + namespace : The name of the namespace. + session : Session to use for getting project. + + Example: + ```py + import datachain as dc + from datachain.lib.projects import delete as delete_project + delete_project("my-project", "local") + ``` + """ + session = Session.get(session) + catalog = session.catalog + + project = catalog.metastore.get_project(name, namespace) + + if catalog.metastore.is_listing_project(name, namespace): + raise ProjectDeleteNotAllowedError("Project listing cannot be removed") + + if catalog.metastore.is_default_project(name, namespace): + raise ProjectDeleteNotAllowedError("Project default cannot be removed") + + num_datasets = catalog.metastore.count_datasets(project.id) + if num_datasets > 0: + raise ProjectDeleteNotAllowedError( + f"Project cannot be removed. It contains {num_datasets} dataset(s). " + "Please remove the dataset(s) first." + ) + + catalog.metastore.remove_project(project.id) diff --git a/tests/unit/lib/test_project.py b/tests/unit/lib/test_project.py index 2ce35de52..12156dc67 100644 --- a/tests/unit/lib/test_project.py +++ b/tests/unit/lib/test_project.py @@ -4,10 +4,12 @@ from datachain.error import ( InvalidProjectNameError, ProjectCreateNotAllowedError, + ProjectDeleteNotAllowedError, ProjectNotFoundError, ) from datachain.lib.namespaces import create as create_namespace from datachain.lib.namespaces import get as get_namespace +from datachain.lib.projects import delete as delete_project from datachain.lib.projects import get as get_project from datachain.lib.projects import ls as ls_projects from tests.utils import skip_if_not_sqlite @@ -155,3 +157,52 @@ def test_ls_projects_empty_in_namespace(test_session): create_namespace("ns1") projects = ls_projects("ns1", session=test_session) assert [(p.namespace.name, p.name) for p in projects] == [] + + +def test_delete_project(test_session, chatbot_project, dev_namespace): + delete_project(chatbot_project.name, dev_namespace.name, session=test_session) + with pytest.raises(ProjectNotFoundError): + get_project(chatbot_project.name, dev_namespace.name, session=test_session) + + +def test_delete_project_not_found(test_session, chatbot_project, dev_namespace): + with pytest.raises(ProjectNotFoundError): + delete_project("missing", dev_namespace.name, session=test_session) + + +def test_delete_project_listing(test_session): + metastore = test_session.catalog.metastore + with pytest.raises(ProjectDeleteNotAllowedError) as excinfo: + delete_project( + metastore.listing_project_name, + metastore.system_namespace_name, + session=test_session, + ) + assert str(excinfo.value) == "Project listing cannot be removed" + + +def test_delete_project_default(test_session): + metastore = test_session.catalog.metastore + with pytest.raises(ProjectDeleteNotAllowedError) as excinfo: + delete_project( + metastore.default_project_name, + metastore.default_namespace_name, + session=test_session, + ) + assert str(excinfo.value) == "Project default cannot be removed" + + +def test_delete_project_non_empty(test_session, chatbot_project, dev_namespace): + ( + dc.read_values(num=[1, 2, 3]) + .settings(namespace=dev_namespace.name, project=chatbot_project.name) + .save("numbers") + ) + + with pytest.raises(ProjectDeleteNotAllowedError) as excinfo: + delete_project(chatbot_project.name, dev_namespace.name, session=test_session) + + assert str(excinfo.value) == ( + "Project cannot be removed. It contains 1 dataset(s)." + " Please remove the dataset(s) first." + ) From 9e4bab3192f7924d463a89ab954c0db410775262 Mon Sep 17 00:00:00 2001 From: ilongin Date: Fri, 5 Sep 2025 13:57:23 +0200 Subject: [PATCH 2/7] added delete namespace --- src/datachain/data_storage/metastore.py | 9 +++++ src/datachain/error.py | 4 ++ src/datachain/lib/namespaces.py | 51 ++++++++++++++++++++++++- src/datachain/lib/projects.py | 20 ++++++---- tests/unit/lib/test_namespace.py | 45 ++++++++++++++++++++++ tests/unit/lib/test_project.py | 8 +++- 6 files changed, 126 insertions(+), 11 deletions(-) diff --git a/src/datachain/data_storage/metastore.py b/src/datachain/data_storage/metastore.py index 48cd3597b..aa862f32c 100644 --- a/src/datachain/data_storage/metastore.py +++ b/src/datachain/data_storage/metastore.py @@ -142,6 +142,10 @@ def create_namespace( def get_namespace(self, name: str, conn=None) -> Namespace: """Gets a single namespace by name""" + @abstractmethod + def remove_namespace(self, namespace_id: int, conn=None) -> None: + """Removes a single namespace by id""" + @abstractmethod def list_namespaces(self, conn=None) -> list[Namespace]: """Gets a list of all namespaces""" @@ -752,6 +756,11 @@ def create_namespace( return self.get_namespace(name) + def remove_namespace(self, namespace_id: int, conn=None) -> None: + n = self._namespaces + with self.db.transaction(): + self.db.execute(self._namespaces_delete().where(n.c.id == namespace_id)) + def get_namespace(self, name: str, conn=None) -> Namespace: """Gets a single namespace by name""" n = self._namespaces diff --git a/src/datachain/error.py b/src/datachain/error.py index 33301ebac..cebb658c5 100644 --- a/src/datachain/error.py +++ b/src/datachain/error.py @@ -38,6 +38,10 @@ class ProjectDeleteNotAllowedError(NotAllowedError): pass +class NamespaceDeleteNotAllowedError(NotAllowedError): + pass + + class ProjectNotFoundError(NotFoundError): pass diff --git a/src/datachain/lib/namespaces.py b/src/datachain/lib/namespaces.py index 75c88cb94..d9e9e7049 100644 --- a/src/datachain/lib/namespaces.py +++ b/src/datachain/lib/namespaces.py @@ -1,6 +1,9 @@ from typing import Optional -from datachain.error import NamespaceCreateNotAllowedError +from datachain.error import ( + NamespaceCreateNotAllowedError, + NamespaceDeleteNotAllowedError, +) from datachain.namespace import Namespace from datachain.query import Session @@ -71,3 +74,49 @@ def ls(session: Optional[Session] = None) -> list[Namespace]: ``` """ return Session.get(session).catalog.metastore.list_namespaces() + + +def delete(name: str, session: Optional[Session]) -> None: + """ + Removes a namespace by name. + + Raises: + NamespaceNotFoundError: If the namespace does not exist. + NamespaceDeleteNotAllowedError: If the namespace is non-empty, + is the default namespace, or is a system namespace, + as these cannot be removed. + + Parameters: + name : The name of the namespace. + session : Session to use for getting project. + + Example: + ```py + import datachain as dc + from datachain.lib.namespace import delete as delete_namespace + delete_namespace("dev") + ``` + """ + session = Session.get(session) + metastore = session.catalog.metastore + + namespace = metastore.get_namespace(name) + + if name == metastore.system_namespace_name: + raise NamespaceDeleteNotAllowedError( + f"Namespace {metastore.system_namespace_name} cannot be removed" + ) + + if name == metastore.default_namespace_name: + raise NamespaceDeleteNotAllowedError( + f"Namespace {metastore.default_namespace_name} cannot be removed" + ) + + projects = metastore.list_projects(namespace.id) + if len(projects) > 0: + raise NamespaceDeleteNotAllowedError( + f"Namespace cannot be removed. It contains {len(projects)} project(s). " + "Please remove the project(s) first." + ) + + metastore.remove_namespace(namespace.id) diff --git a/src/datachain/lib/projects.py b/src/datachain/lib/projects.py index c78960ddb..971985642 100644 --- a/src/datachain/lib/projects.py +++ b/src/datachain/lib/projects.py @@ -111,21 +111,25 @@ def delete(name: str, namespace: str, session: Optional[Session]) -> None: ``` """ session = Session.get(session) - catalog = session.catalog + metastore = session.catalog.metastore - project = catalog.metastore.get_project(name, namespace) + project = metastore.get_project(name, namespace) - if catalog.metastore.is_listing_project(name, namespace): - raise ProjectDeleteNotAllowedError("Project listing cannot be removed") + if metastore.is_listing_project(name, namespace): + raise ProjectDeleteNotAllowedError( + f"Project {metastore.listing_project_name} cannot be removed" + ) - if catalog.metastore.is_default_project(name, namespace): - raise ProjectDeleteNotAllowedError("Project default cannot be removed") + if metastore.is_default_project(name, namespace): + raise ProjectDeleteNotAllowedError( + f"Project {metastore.default_project_name} cannot be removed" + ) - num_datasets = catalog.metastore.count_datasets(project.id) + num_datasets = metastore.count_datasets(project.id) if num_datasets > 0: raise ProjectDeleteNotAllowedError( f"Project cannot be removed. It contains {num_datasets} dataset(s). " "Please remove the dataset(s) first." ) - catalog.metastore.remove_project(project.id) + metastore.remove_project(project.id) diff --git a/tests/unit/lib/test_namespace.py b/tests/unit/lib/test_namespace.py index 0241e93cc..77d7e8a1d 100644 --- a/tests/unit/lib/test_namespace.py +++ b/tests/unit/lib/test_namespace.py @@ -3,11 +3,14 @@ from datachain.error import ( InvalidNamespaceNameError, NamespaceCreateNotAllowedError, + NamespaceDeleteNotAllowedError, NamespaceNotFoundError, ) from datachain.lib.namespaces import create as create_namespace +from datachain.lib.namespaces import delete as delete_namespace from datachain.lib.namespaces import get as get_namespace from datachain.lib.namespaces import ls as ls_namespaces +from datachain.lib.projects import create as create_project from tests.utils import skip_if_not_sqlite @@ -77,3 +80,45 @@ def test_ls_namespaces_just_local(test_session): system_namespace_name = test_session.catalog.metastore.system_namespace_name namespaces = ls_namespaces(session=test_session) assert [n.name for n in namespaces] == [system_namespace_name] + + +def test_delete_namespace(test_session, dev_namespace): + delete_namespace(dev_namespace.name, session=test_session) + with pytest.raises(NamespaceNotFoundError): + get_namespace(dev_namespace.name, session=test_session) + + +def test_delete_namespace_not_found(test_session, dev_namespace): + with pytest.raises(NamespaceNotFoundError): + delete_namespace("missing", session=test_session) + + +def test_delete_namespace_system(test_session): + metastore = test_session.catalog.metastore + with pytest.raises(NamespaceDeleteNotAllowedError) as excinfo: + delete_namespace(metastore.system_namespace_name, session=test_session) + assert str(excinfo.value) == ( + f"Namespace {metastore.system_namespace_name} cannot be removed" + ) + + +def test_delete_namespace_default(test_session): + metastore = test_session.catalog.metastore + metastore.create_namespace(metastore.default_namespace_name, validate=False) + with pytest.raises(NamespaceDeleteNotAllowedError) as excinfo: + delete_namespace(metastore.default_namespace_name, session=test_session) + assert str(excinfo.value) == ( + f"Namespace {metastore.default_namespace_name} cannot be removed" + ) + + +def test_delete_namespace_non_empty(test_session, dev_namespace): + create_project(dev_namespace.name, "my-project") + + with pytest.raises(NamespaceDeleteNotAllowedError) as excinfo: + delete_namespace(dev_namespace.name, session=test_session) + + assert str(excinfo.value) == ( + "Namespace cannot be removed. It contains 1 project(s)." + " Please remove the project(s) first." + ) diff --git a/tests/unit/lib/test_project.py b/tests/unit/lib/test_project.py index 12156dc67..e8d9fa896 100644 --- a/tests/unit/lib/test_project.py +++ b/tests/unit/lib/test_project.py @@ -178,7 +178,9 @@ def test_delete_project_listing(test_session): metastore.system_namespace_name, session=test_session, ) - assert str(excinfo.value) == "Project listing cannot be removed" + assert str(excinfo.value) == ( + f"Project {metastore.listing_project_name} cannot be removed" + ) def test_delete_project_default(test_session): @@ -189,7 +191,9 @@ def test_delete_project_default(test_session): metastore.default_namespace_name, session=test_session, ) - assert str(excinfo.value) == "Project default cannot be removed" + assert str(excinfo.value) == ( + f"Project {metastore.default_project_name} cannot be removed" + ) def test_delete_project_non_empty(test_session, chatbot_project, dev_namespace): From 9d93604f3907960c4bcd82edc632e95e336d0749 Mon Sep 17 00:00:00 2001 From: ilongin Date: Fri, 5 Sep 2025 14:08:31 +0200 Subject: [PATCH 3/7] making delete namespace public --- src/datachain/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/datachain/__init__.py b/src/datachain/__init__.py index c6fa7bc4d..9f9b7081e 100644 --- a/src/datachain/__init__.py +++ b/src/datachain/__init__.py @@ -37,6 +37,7 @@ VideoFrame, ) from datachain.lib.model_store import ModelStore +from datachain.lib.namespaces import delete as delete_namespace from datachain.lib.projects import create as create_project from datachain.lib.projects import delete as delete_project from datachain.lib.udf import Aggregator, Generator, Mapper @@ -75,6 +76,7 @@ "create_project", "datasets", "delete_dataset", + "delete_namespace", "delete_project", "is_chain_type", "is_studio", From 9ad3e3423fc5777b192461d9e42a12b2afb01e06 Mon Sep 17 00:00:00 2001 From: ilongin Date: Tue, 9 Sep 2025 10:31:54 +0200 Subject: [PATCH 4/7] fixing metastore methods --- src/datachain/data_storage/metastore.py | 24 ++++++++---------------- src/datachain/lib/projects.py | 5 ++--- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/src/datachain/data_storage/metastore.py b/src/datachain/data_storage/metastore.py index aa862f32c..5f0fa0d35 100644 --- a/src/datachain/data_storage/metastore.py +++ b/src/datachain/data_storage/metastore.py @@ -195,13 +195,17 @@ def get_project( It also creates project if not found and create flag is set to True. """ - @abstractmethod def is_default_project(self, project_name: str, namespace_name: str) -> bool: - """Checks if some project in namespace is default""" + return ( + project_name == self.default_project_name + and namespace_name == self.default_namespace_name + ) - @abstractmethod def is_listing_project(self, project_name: str, namespace_name: str) -> bool: - """Checks if some project in namespace is listing""" + return ( + project_name == self.listing_project_name + and namespace_name == self.system_namespace_name + ) @abstractmethod def get_project_by_id(self, project_id: int, conn=None) -> Project: @@ -822,18 +826,6 @@ def create_project( return self.get_project(name, namespace.name) - def is_listing_project(self, project_name: str, namespace_name: str) -> bool: - return ( - project_name == self.listing_project_name - and namespace_name == self.system_namespace_name - ) - - def is_default_project(self, project_name: str, namespace_name: str) -> bool: - return ( - project_name == self.default_project_name - and namespace_name == self.default_namespace_name - ) - def get_project( self, name: str, namespace_name: str, create: bool = False, conn=None ) -> Project: diff --git a/src/datachain/lib/projects.py b/src/datachain/lib/projects.py index 971985642..ed0166275 100644 --- a/src/datachain/lib/projects.py +++ b/src/datachain/lib/projects.py @@ -88,7 +88,7 @@ def ls( return session.catalog.metastore.list_projects(namespace_id) -def delete(name: str, namespace: str, session: Optional[Session]) -> None: +def delete(name: str, namespace: str, session: Optional[Session] = None) -> None: """ Removes a project by name within a namespace. @@ -106,8 +106,7 @@ def delete(name: str, namespace: str, session: Optional[Session]) -> None: Example: ```py import datachain as dc - from datachain.lib.projects import delete as delete_project - delete_project("my-project", "local") + dc.delete_project("my-project", "local") ``` """ session = Session.get(session) From 21e94a01a3f74d6c4dd846f1ab68a5ad7caa91db Mon Sep 17 00:00:00 2001 From: ilongin Date: Wed, 10 Sep 2025 15:32:47 +0200 Subject: [PATCH 5/7] fixing --- src/datachain/data_storage/metastore.py | 20 ++++++++++++++++++-- src/datachain/lib/namespaces.py | 6 +++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/datachain/data_storage/metastore.py b/src/datachain/data_storage/metastore.py index 5f0fa0d35..acaa7924d 100644 --- a/src/datachain/data_storage/metastore.py +++ b/src/datachain/data_storage/metastore.py @@ -211,6 +211,10 @@ def is_listing_project(self, project_name: str, namespace_name: str) -> bool: def get_project_by_id(self, project_id: int, conn=None) -> Project: """Gets a single project by id""" + @abstractmethod + def count_projects(self, namespace_id: Optional[int] = None) -> int: + """Counts projects in some namespace or in general.""" + @abstractmethod def remove_project(self, project_id: int, conn=None) -> None: """Removes a single project by id""" @@ -876,12 +880,24 @@ def get_project_by_id(self, project_id: int, conn=None) -> Project: raise ProjectNotFoundError(f"Project with id {project_id} not found.") return self.project_class.parse(*rows[0]) + def count_projects(self, namespace_id: Optional[int] = None) -> int: + p = self._projects + query = self._projects_select() + if namespace_id: + query = query.where(p.c.namespace_id == namespace_id) + + query = select(f.count(1)).select_from(query.subquery()) + + return next(self.db.execute(query))[0] + def remove_project(self, project_id: int, conn=None) -> None: p = self._projects with self.db.transaction(): self.db.execute(self._projects_delete().where(p.c.id == project_id)) - def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]: + def list_projects( + self, namespace_id: Optional[int] = None, conn=None + ) -> list[Project]: """ Gets a list of projects inside some namespace, or in all namespaces """ @@ -1222,7 +1238,7 @@ def list_datasets( def count_datasets(self, project_id: Optional[int] = None) -> int: d = self._datasets - query = self._base_list_datasets_query() + query = self._datasets_select() if project_id: query = query.where(d.c.project_id == project_id) diff --git a/src/datachain/lib/namespaces.py b/src/datachain/lib/namespaces.py index d9e9e7049..cef72ec88 100644 --- a/src/datachain/lib/namespaces.py +++ b/src/datachain/lib/namespaces.py @@ -112,10 +112,10 @@ def delete(name: str, session: Optional[Session]) -> None: f"Namespace {metastore.default_namespace_name} cannot be removed" ) - projects = metastore.list_projects(namespace.id) - if len(projects) > 0: + num_projects = metastore.count_projects(namespace.id) + if num_projects > 0: raise NamespaceDeleteNotAllowedError( - f"Namespace cannot be removed. It contains {len(projects)} project(s). " + f"Namespace cannot be removed. It contains {num_projects} project(s). " "Please remove the project(s) first." ) From 51d93d243dadab902bd2aedbe08db72ba9b015c8 Mon Sep 17 00:00:00 2001 From: ilongin Date: Wed, 10 Sep 2025 23:14:46 +0200 Subject: [PATCH 6/7] added additional checks for removing namespace and project in metastore --- src/datachain/data_storage/metastore.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/datachain/data_storage/metastore.py b/src/datachain/data_storage/metastore.py index acaa7924d..7d091f0d7 100644 --- a/src/datachain/data_storage/metastore.py +++ b/src/datachain/data_storage/metastore.py @@ -38,7 +38,9 @@ from datachain.error import ( DatasetNotFoundError, DatasetVersionNotFoundError, + NamespaceDeleteNotAllowedError, NamespaceNotFoundError, + ProjectDeleteNotAllowedError, ProjectNotFoundError, TableMissingError, ) @@ -765,6 +767,13 @@ def create_namespace( return self.get_namespace(name) def remove_namespace(self, namespace_id: int, conn=None) -> None: + num_projects = self.count_projects(namespace_id) + if num_projects > 0: + raise NamespaceDeleteNotAllowedError( + f"Namespace cannot be removed. It contains {num_projects} project(s). " + "Please remove the project(s) first." + ) + n = self._namespaces with self.db.transaction(): self.db.execute(self._namespaces_delete().where(n.c.id == namespace_id)) @@ -891,6 +900,13 @@ def count_projects(self, namespace_id: Optional[int] = None) -> int: return next(self.db.execute(query))[0] def remove_project(self, project_id: int, conn=None) -> None: + num_datasets = self.count_datasets(project_id) + if num_datasets > 0: + raise ProjectDeleteNotAllowedError( + f"Project cannot be removed. It contains {num_datasets} dataset(s). " + "Please remove the dataset(s) first." + ) + p = self._projects with self.db.transaction(): self.db.execute(self._projects_delete().where(p.c.id == project_id)) From fab307a6b3ed9dffea57976d3e569231b9d6f2d7 Mon Sep 17 00:00:00 2001 From: ilongin Date: Thu, 11 Sep 2025 11:53:57 +0200 Subject: [PATCH 7/7] added delete project under namespace --- src/datachain/__init__.py | 2 -- src/datachain/lib/namespaces.py | 8 +++++++- src/datachain/namespace.py | 19 +++++++++++++++++++ tests/unit/lib/test_project.py | 25 +++++++++++++++---------- 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/src/datachain/__init__.py b/src/datachain/__init__.py index 9f9b7081e..f40eccb6d 100644 --- a/src/datachain/__init__.py +++ b/src/datachain/__init__.py @@ -39,7 +39,6 @@ from datachain.lib.model_store import ModelStore from datachain.lib.namespaces import delete as delete_namespace from datachain.lib.projects import create as create_project -from datachain.lib.projects import delete as delete_project from datachain.lib.udf import Aggregator, Generator, Mapper from datachain.lib.utils import AbstractUDF, DataChainError from datachain.query import metrics, param @@ -77,7 +76,6 @@ "datasets", "delete_dataset", "delete_namespace", - "delete_project", "is_chain_type", "is_studio", "listings", diff --git a/src/datachain/lib/namespaces.py b/src/datachain/lib/namespaces.py index cef72ec88..36dccb6bb 100644 --- a/src/datachain/lib/namespaces.py +++ b/src/datachain/lib/namespaces.py @@ -4,7 +4,8 @@ NamespaceCreateNotAllowedError, NamespaceDeleteNotAllowedError, ) -from datachain.namespace import Namespace +from datachain.lib.projects import delete as delete_project +from datachain.namespace import Namespace, parse_name from datachain.query import Session @@ -100,6 +101,11 @@ def delete(name: str, session: Optional[Session]) -> None: session = Session.get(session) metastore = session.catalog.metastore + namespace_name, project_name = parse_name(name) + + if project_name: + return delete_project(project_name, namespace_name, session) + namespace = metastore.get_namespace(name) if name == metastore.system_namespace_name: diff --git a/src/datachain/namespace.py b/src/datachain/namespace.py index 86010568b..c7a4bcb98 100644 --- a/src/datachain/namespace.py +++ b/src/datachain/namespace.py @@ -9,6 +9,25 @@ NAMESPACE_NAME_RESERVED_CHARS = [".", "@"] +def parse_name(name: str) -> tuple[str, Optional[str]]: + """ + Parses namespace name into namespace and optional project name. + If both namespace and project are defined in name, they need to be split by dot + e.g dev.my-project + Valid inputs: + - dev.my-project + - dev + """ + parts = name.split(".") + if len(parts) == 1: + return name, None + if len(parts) == 2: + return parts[0], parts[1] + raise InvalidNamespaceNameError( + f"Invalid namespace format: {name}. Expected 'namespace' or 'ns1.ns2'." + ) + + @dataclass(frozen=True) class Namespace: id: int diff --git a/tests/unit/lib/test_project.py b/tests/unit/lib/test_project.py index e8d9fa896..2b557e4a8 100644 --- a/tests/unit/lib/test_project.py +++ b/tests/unit/lib/test_project.py @@ -8,8 +8,8 @@ ProjectNotFoundError, ) from datachain.lib.namespaces import create as create_namespace +from datachain.lib.namespaces import delete as delete_namespace from datachain.lib.namespaces import get as get_namespace -from datachain.lib.projects import delete as delete_project from datachain.lib.projects import get as get_project from datachain.lib.projects import ls as ls_projects from tests.utils import skip_if_not_sqlite @@ -160,22 +160,26 @@ def test_ls_projects_empty_in_namespace(test_session): def test_delete_project(test_session, chatbot_project, dev_namespace): - delete_project(chatbot_project.name, dev_namespace.name, session=test_session) + delete_namespace( + f"{dev_namespace.name}.{chatbot_project.name}", session=test_session + ) with pytest.raises(ProjectNotFoundError): get_project(chatbot_project.name, dev_namespace.name, session=test_session) + # namespace should not be deleted + get_namespace(dev_namespace.name, session=test_session) + def test_delete_project_not_found(test_session, chatbot_project, dev_namespace): with pytest.raises(ProjectNotFoundError): - delete_project("missing", dev_namespace.name, session=test_session) + delete_namespace(f"{dev_namespace.name}.missing", session=test_session) def test_delete_project_listing(test_session): metastore = test_session.catalog.metastore with pytest.raises(ProjectDeleteNotAllowedError) as excinfo: - delete_project( - metastore.listing_project_name, - metastore.system_namespace_name, + delete_namespace( + f"{metastore.system_namespace_name}.{metastore.listing_project_name}", session=test_session, ) assert str(excinfo.value) == ( @@ -186,9 +190,8 @@ def test_delete_project_listing(test_session): def test_delete_project_default(test_session): metastore = test_session.catalog.metastore with pytest.raises(ProjectDeleteNotAllowedError) as excinfo: - delete_project( - metastore.default_project_name, - metastore.default_namespace_name, + delete_namespace( + f"{metastore.default_namespace_name}.{metastore.default_project_name}", session=test_session, ) assert str(excinfo.value) == ( @@ -204,7 +207,9 @@ def test_delete_project_non_empty(test_session, chatbot_project, dev_namespace): ) with pytest.raises(ProjectDeleteNotAllowedError) as excinfo: - delete_project(chatbot_project.name, dev_namespace.name, session=test_session) + delete_namespace( + f"{dev_namespace.name}.{chatbot_project.name}", session=test_session + ) assert str(excinfo.value) == ( "Project cannot be removed. It contains 1 dataset(s)."