diff --git a/src/datachain/__init__.py b/src/datachain/__init__.py index 0ea735829..f40eccb6d 100644 --- a/src/datachain/__init__.py +++ b/src/datachain/__init__.py @@ -37,6 +37,7 @@ VideoFrame, ) from datachain.lib.model_store import ModelStore +from datachain.lib.namespaces import delete as delete_namespace from datachain.lib.projects import create as create_project from datachain.lib.udf import Aggregator, Generator, Mapper from datachain.lib.utils import AbstractUDF, DataChainError @@ -74,6 +75,7 @@ "create_project", "datasets", "delete_dataset", + "delete_namespace", "is_chain_type", "is_studio", "listings", diff --git a/src/datachain/data_storage/metastore.py b/src/datachain/data_storage/metastore.py index c3548f6d8..7d091f0d7 100644 --- a/src/datachain/data_storage/metastore.py +++ b/src/datachain/data_storage/metastore.py @@ -22,6 +22,7 @@ UniqueConstraint, select, ) +from sqlalchemy.sql import func as f from datachain.data_storage import JobQueryType, JobStatus from datachain.data_storage.serializer import Serializable @@ -37,7 +38,9 @@ from datachain.error import ( DatasetNotFoundError, DatasetVersionNotFoundError, + NamespaceDeleteNotAllowedError, NamespaceNotFoundError, + ProjectDeleteNotAllowedError, ProjectNotFoundError, TableMissingError, ) @@ -141,6 +144,10 @@ def create_namespace( def get_namespace(self, name: str, conn=None) -> Namespace: """Gets a single namespace by name""" + @abstractmethod + def remove_namespace(self, namespace_id: int, conn=None) -> None: + """Removes a single namespace by id""" + @abstractmethod def list_namespaces(self, conn=None) -> list[Namespace]: """Gets a list of all namespaces""" @@ -190,10 +197,30 @@ def get_project( It also creates project if not found and create flag is set to True. """ + def is_default_project(self, project_name: str, namespace_name: str) -> bool: + return ( + project_name == self.default_project_name + and namespace_name == self.default_namespace_name + ) + + def is_listing_project(self, project_name: str, namespace_name: str) -> bool: + return ( + project_name == self.listing_project_name + and namespace_name == self.system_namespace_name + ) + @abstractmethod def get_project_by_id(self, project_id: int, conn=None) -> Project: """Gets a single project by id""" + @abstractmethod + def count_projects(self, namespace_id: Optional[int] = None) -> int: + """Counts projects in some namespace or in general.""" + + @abstractmethod + def remove_project(self, project_id: int, conn=None) -> None: + """Removes a single project by id""" + @abstractmethod def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]: """Gets list of projects in some namespace or in general (in all namespaces)""" @@ -270,6 +297,10 @@ def list_datasets( ) -> Iterator[DatasetListRecord]: """Lists all datasets in some project or in all projects.""" + @abstractmethod + def count_datasets(self, project_id: Optional[int] = None) -> int: + """Counts datasets in some project or in all projects.""" + @abstractmethod def list_datasets_by_prefix( self, prefix: str, project_id: Optional[int] = None @@ -735,6 +766,18 @@ def create_namespace( return self.get_namespace(name) + def remove_namespace(self, namespace_id: int, conn=None) -> None: + num_projects = self.count_projects(namespace_id) + if num_projects > 0: + raise NamespaceDeleteNotAllowedError( + f"Namespace cannot be removed. It contains {num_projects} project(s). " + "Please remove the project(s) first." + ) + + n = self._namespaces + with self.db.transaction(): + self.db.execute(self._namespaces_delete().where(n.c.id == namespace_id)) + def get_namespace(self, name: str, conn=None) -> Namespace: """Gets a single namespace by name""" n = self._namespaces @@ -796,18 +839,6 @@ def create_project( return self.get_project(name, namespace.name) - def _is_listing_project(self, project_name: str, namespace_name: str) -> bool: - return ( - project_name == self.listing_project_name - and namespace_name == self.system_namespace_name - ) - - def _is_default_project(self, project_name: str, namespace_name: str) -> bool: - return ( - project_name == self.default_project_name - and namespace_name == self.default_namespace_name - ) - def get_project( self, name: str, namespace_name: str, create: bool = False, conn=None ) -> Project: @@ -816,7 +847,7 @@ def get_project( p = self._projects validate = True - if self._is_listing_project(name, namespace_name) or self._is_default_project( + if self.is_listing_project(name, namespace_name) or self.is_default_project( name, namespace_name ): # we are always creating default and listing projects if they don't exist @@ -858,7 +889,31 @@ def get_project_by_id(self, project_id: int, conn=None) -> Project: raise ProjectNotFoundError(f"Project with id {project_id} not found.") return self.project_class.parse(*rows[0]) - def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]: + def count_projects(self, namespace_id: Optional[int] = None) -> int: + p = self._projects + query = self._projects_select() + if namespace_id: + query = query.where(p.c.namespace_id == namespace_id) + + query = select(f.count(1)).select_from(query.subquery()) + + return next(self.db.execute(query))[0] + + def remove_project(self, project_id: int, conn=None) -> None: + num_datasets = self.count_datasets(project_id) + if num_datasets > 0: + raise ProjectDeleteNotAllowedError( + f"Project cannot be removed. It contains {num_datasets} dataset(s). " + "Please remove the dataset(s) first." + ) + + p = self._projects + with self.db.transaction(): + self.db.execute(self._projects_delete().where(p.c.id == project_id)) + + def list_projects( + self, namespace_id: Optional[int] = None, conn=None + ) -> list[Project]: """ Gets a list of projects inside some namespace, or in all namespaces """ @@ -1189,7 +1244,6 @@ def _base_list_datasets_query(self) -> "Select": def list_datasets( self, project_id: Optional[int] = None ) -> Iterator["DatasetListRecord"]: - """Lists all datasets.""" d = self._datasets query = self._base_list_datasets_query().order_by( self._datasets.c.name, self._datasets_versions.c.version @@ -1198,6 +1252,16 @@ def list_datasets( query = query.where(d.c.project_id == project_id) yield from self._parse_dataset_list(self.db.execute(query)) + def count_datasets(self, project_id: Optional[int] = None) -> int: + d = self._datasets + query = self._datasets_select() + if project_id: + query = query.where(d.c.project_id == project_id) + + query = select(f.count(1)).select_from(query.subquery()) + + return next(self.db.execute(query))[0] + def list_datasets_by_prefix( self, prefix: str, project_id: Optional[int] = None, conn=None ) -> Iterator["DatasetListRecord"]: diff --git a/src/datachain/error.py b/src/datachain/error.py index 78b1c028a..cebb658c5 100644 --- a/src/datachain/error.py +++ b/src/datachain/error.py @@ -34,6 +34,14 @@ class ProjectCreateNotAllowedError(NotAllowedError): pass +class ProjectDeleteNotAllowedError(NotAllowedError): + pass + + +class NamespaceDeleteNotAllowedError(NotAllowedError): + pass + + class ProjectNotFoundError(NotFoundError): pass diff --git a/src/datachain/lib/namespaces.py b/src/datachain/lib/namespaces.py index 75c88cb94..36dccb6bb 100644 --- a/src/datachain/lib/namespaces.py +++ b/src/datachain/lib/namespaces.py @@ -1,7 +1,11 @@ from typing import Optional -from datachain.error import NamespaceCreateNotAllowedError -from datachain.namespace import Namespace +from datachain.error import ( + NamespaceCreateNotAllowedError, + NamespaceDeleteNotAllowedError, +) +from datachain.lib.projects import delete as delete_project +from datachain.namespace import Namespace, parse_name from datachain.query import Session @@ -71,3 +75,54 @@ def ls(session: Optional[Session] = None) -> list[Namespace]: ``` """ return Session.get(session).catalog.metastore.list_namespaces() + + +def delete(name: str, session: Optional[Session]) -> None: + """ + Removes a namespace by name. + + Raises: + NamespaceNotFoundError: If the namespace does not exist. + NamespaceDeleteNotAllowedError: If the namespace is non-empty, + is the default namespace, or is a system namespace, + as these cannot be removed. + + Parameters: + name : The name of the namespace. + session : Session to use for getting project. + + Example: + ```py + import datachain as dc + from datachain.lib.namespace import delete as delete_namespace + delete_namespace("dev") + ``` + """ + session = Session.get(session) + metastore = session.catalog.metastore + + namespace_name, project_name = parse_name(name) + + if project_name: + return delete_project(project_name, namespace_name, session) + + namespace = metastore.get_namespace(name) + + if name == metastore.system_namespace_name: + raise NamespaceDeleteNotAllowedError( + f"Namespace {metastore.system_namespace_name} cannot be removed" + ) + + if name == metastore.default_namespace_name: + raise NamespaceDeleteNotAllowedError( + f"Namespace {metastore.default_namespace_name} cannot be removed" + ) + + num_projects = metastore.count_projects(namespace.id) + if num_projects > 0: + raise NamespaceDeleteNotAllowedError( + f"Namespace cannot be removed. It contains {num_projects} project(s). " + "Please remove the project(s) first." + ) + + metastore.remove_namespace(namespace.id) diff --git a/src/datachain/lib/projects.py b/src/datachain/lib/projects.py index da8423bfd..ed0166275 100644 --- a/src/datachain/lib/projects.py +++ b/src/datachain/lib/projects.py @@ -1,6 +1,6 @@ from typing import Optional -from datachain.error import ProjectCreateNotAllowedError +from datachain.error import ProjectCreateNotAllowedError, ProjectDeleteNotAllowedError from datachain.project import Project from datachain.query import Session @@ -86,3 +86,49 @@ def ls( namespace_id = session.catalog.metastore.get_namespace(namespace).id return session.catalog.metastore.list_projects(namespace_id) + + +def delete(name: str, namespace: str, session: Optional[Session] = None) -> None: + """ + Removes a project by name within a namespace. + + Raises: + ProjectNotFoundError: If the project does not exist. + ProjectDeleteNotAllowedError: If the project is non-empty, + is the default project, or is a listing project, + as these cannot be removed. + + Parameters: + name : The name of the project. + namespace : The name of the namespace. + session : Session to use for getting project. + + Example: + ```py + import datachain as dc + dc.delete_project("my-project", "local") + ``` + """ + session = Session.get(session) + metastore = session.catalog.metastore + + project = metastore.get_project(name, namespace) + + if metastore.is_listing_project(name, namespace): + raise ProjectDeleteNotAllowedError( + f"Project {metastore.listing_project_name} cannot be removed" + ) + + if metastore.is_default_project(name, namespace): + raise ProjectDeleteNotAllowedError( + f"Project {metastore.default_project_name} cannot be removed" + ) + + num_datasets = metastore.count_datasets(project.id) + if num_datasets > 0: + raise ProjectDeleteNotAllowedError( + f"Project cannot be removed. It contains {num_datasets} dataset(s). " + "Please remove the dataset(s) first." + ) + + metastore.remove_project(project.id) diff --git a/src/datachain/namespace.py b/src/datachain/namespace.py index 86010568b..c7a4bcb98 100644 --- a/src/datachain/namespace.py +++ b/src/datachain/namespace.py @@ -9,6 +9,25 @@ NAMESPACE_NAME_RESERVED_CHARS = [".", "@"] +def parse_name(name: str) -> tuple[str, Optional[str]]: + """ + Parses namespace name into namespace and optional project name. + If both namespace and project are defined in name, they need to be split by dot + e.g dev.my-project + Valid inputs: + - dev.my-project + - dev + """ + parts = name.split(".") + if len(parts) == 1: + return name, None + if len(parts) == 2: + return parts[0], parts[1] + raise InvalidNamespaceNameError( + f"Invalid namespace format: {name}. Expected 'namespace' or 'ns1.ns2'." + ) + + @dataclass(frozen=True) class Namespace: id: int diff --git a/tests/unit/lib/test_namespace.py b/tests/unit/lib/test_namespace.py index 0241e93cc..77d7e8a1d 100644 --- a/tests/unit/lib/test_namespace.py +++ b/tests/unit/lib/test_namespace.py @@ -3,11 +3,14 @@ from datachain.error import ( InvalidNamespaceNameError, NamespaceCreateNotAllowedError, + NamespaceDeleteNotAllowedError, NamespaceNotFoundError, ) from datachain.lib.namespaces import create as create_namespace +from datachain.lib.namespaces import delete as delete_namespace from datachain.lib.namespaces import get as get_namespace from datachain.lib.namespaces import ls as ls_namespaces +from datachain.lib.projects import create as create_project from tests.utils import skip_if_not_sqlite @@ -77,3 +80,45 @@ def test_ls_namespaces_just_local(test_session): system_namespace_name = test_session.catalog.metastore.system_namespace_name namespaces = ls_namespaces(session=test_session) assert [n.name for n in namespaces] == [system_namespace_name] + + +def test_delete_namespace(test_session, dev_namespace): + delete_namespace(dev_namespace.name, session=test_session) + with pytest.raises(NamespaceNotFoundError): + get_namespace(dev_namespace.name, session=test_session) + + +def test_delete_namespace_not_found(test_session, dev_namespace): + with pytest.raises(NamespaceNotFoundError): + delete_namespace("missing", session=test_session) + + +def test_delete_namespace_system(test_session): + metastore = test_session.catalog.metastore + with pytest.raises(NamespaceDeleteNotAllowedError) as excinfo: + delete_namespace(metastore.system_namespace_name, session=test_session) + assert str(excinfo.value) == ( + f"Namespace {metastore.system_namespace_name} cannot be removed" + ) + + +def test_delete_namespace_default(test_session): + metastore = test_session.catalog.metastore + metastore.create_namespace(metastore.default_namespace_name, validate=False) + with pytest.raises(NamespaceDeleteNotAllowedError) as excinfo: + delete_namespace(metastore.default_namespace_name, session=test_session) + assert str(excinfo.value) == ( + f"Namespace {metastore.default_namespace_name} cannot be removed" + ) + + +def test_delete_namespace_non_empty(test_session, dev_namespace): + create_project(dev_namespace.name, "my-project") + + with pytest.raises(NamespaceDeleteNotAllowedError) as excinfo: + delete_namespace(dev_namespace.name, session=test_session) + + assert str(excinfo.value) == ( + "Namespace cannot be removed. It contains 1 project(s)." + " Please remove the project(s) first." + ) diff --git a/tests/unit/lib/test_project.py b/tests/unit/lib/test_project.py index 2ce35de52..2b557e4a8 100644 --- a/tests/unit/lib/test_project.py +++ b/tests/unit/lib/test_project.py @@ -4,9 +4,11 @@ from datachain.error import ( InvalidProjectNameError, ProjectCreateNotAllowedError, + ProjectDeleteNotAllowedError, ProjectNotFoundError, ) from datachain.lib.namespaces import create as create_namespace +from datachain.lib.namespaces import delete as delete_namespace from datachain.lib.namespaces import get as get_namespace from datachain.lib.projects import get as get_project from datachain.lib.projects import ls as ls_projects @@ -155,3 +157,61 @@ def test_ls_projects_empty_in_namespace(test_session): create_namespace("ns1") projects = ls_projects("ns1", session=test_session) assert [(p.namespace.name, p.name) for p in projects] == [] + + +def test_delete_project(test_session, chatbot_project, dev_namespace): + delete_namespace( + f"{dev_namespace.name}.{chatbot_project.name}", session=test_session + ) + with pytest.raises(ProjectNotFoundError): + get_project(chatbot_project.name, dev_namespace.name, session=test_session) + + # namespace should not be deleted + get_namespace(dev_namespace.name, session=test_session) + + +def test_delete_project_not_found(test_session, chatbot_project, dev_namespace): + with pytest.raises(ProjectNotFoundError): + delete_namespace(f"{dev_namespace.name}.missing", session=test_session) + + +def test_delete_project_listing(test_session): + metastore = test_session.catalog.metastore + with pytest.raises(ProjectDeleteNotAllowedError) as excinfo: + delete_namespace( + f"{metastore.system_namespace_name}.{metastore.listing_project_name}", + session=test_session, + ) + assert str(excinfo.value) == ( + f"Project {metastore.listing_project_name} cannot be removed" + ) + + +def test_delete_project_default(test_session): + metastore = test_session.catalog.metastore + with pytest.raises(ProjectDeleteNotAllowedError) as excinfo: + delete_namespace( + f"{metastore.default_namespace_name}.{metastore.default_project_name}", + session=test_session, + ) + assert str(excinfo.value) == ( + f"Project {metastore.default_project_name} cannot be removed" + ) + + +def test_delete_project_non_empty(test_session, chatbot_project, dev_namespace): + ( + dc.read_values(num=[1, 2, 3]) + .settings(namespace=dev_namespace.name, project=chatbot_project.name) + .save("numbers") + ) + + with pytest.raises(ProjectDeleteNotAllowedError) as excinfo: + delete_namespace( + f"{dev_namespace.name}.{chatbot_project.name}", session=test_session + ) + + assert str(excinfo.value) == ( + "Project cannot be removed. It contains 1 dataset(s)." + " Please remove the dataset(s) first." + )