Added delete_namespace and delete_project functions

src/datachain/__init__.py

@@ -37,7 +37,9 @@
     VideoFrame,
 )
 from datachain.lib.model_store import ModelStore
+from datachain.lib.namespaces import delete as delete_namespace
 from datachain.lib.projects import create as create_project
+from datachain.lib.projects import delete as delete_project
 from datachain.lib.udf import Aggregator, Generator, Mapper
 from datachain.lib.utils import AbstractUDF, DataChainError
 from datachain.query import metrics, param
@@ -74,6 +76,8 @@
     "create_project",
     "datasets",
     "delete_dataset",
+    "delete_namespace",
+    "delete_project",
     "is_chain_type",
     "is_studio",
     "listings",

src/datachain/data_storage/metastore.py

@@ -22,6 +22,7 @@
     UniqueConstraint,
     select,
 )
+from sqlalchemy.sql import func as f
 
 from datachain.data_storage import JobQueryType, JobStatus
 from datachain.data_storage.serializer import Serializable
@@ -37,7 +38,9 @@
 from datachain.error import (
     DatasetNotFoundError,
     DatasetVersionNotFoundError,
+    NamespaceDeleteNotAllowedError,
     NamespaceNotFoundError,
+    ProjectDeleteNotAllowedError,
     ProjectNotFoundError,
     TableMissingError,
 )
@@ -141,6 +144,10 @@ def create_namespace(
     def get_namespace(self, name: str, conn=None) -> Namespace:
         """Gets a single namespace by name"""
 
+    @abstractmethod
+    def remove_namespace(self, namespace_id: int, conn=None) -> None:
+        """Removes a single namespace by id"""
+
     @abstractmethod
     def list_namespaces(self, conn=None) -> list[Namespace]:
         """Gets a list of all namespaces"""
@@ -190,10 +197,30 @@ def get_project(
         It also creates project if not found and create flag is set to True.
         """
 
+    def is_default_project(self, project_name: str, namespace_name: str) -> bool:
+        return (
+            project_name == self.default_project_name
+            and namespace_name == self.default_namespace_name
+        )
+
+    def is_listing_project(self, project_name: str, namespace_name: str) -> bool:
+        return (
+            project_name == self.listing_project_name
+            and namespace_name == self.system_namespace_name
+        )
+
     @abstractmethod
     def get_project_by_id(self, project_id: int, conn=None) -> Project:
         """Gets a single project by id"""
 
+    @abstractmethod
+    def count_projects(self, namespace_id: Optional[int] = None) -> int:
+        """Counts projects in some namespace or in general."""
+
+    @abstractmethod
+    def remove_project(self, project_id: int, conn=None) -> None:
+        """Removes a single project by id"""
+
     @abstractmethod
     def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]:
         """Gets list of projects in some namespace or in general (in all namespaces)"""
@@ -270,6 +297,10 @@ def list_datasets(
     ) -> Iterator[DatasetListRecord]:
         """Lists all datasets in some project or in all projects."""
 
+    @abstractmethod
+    def count_datasets(self, project_id: Optional[int] = None) -> int:
+        """Counts datasets in some project or in all projects."""
+
     @abstractmethod
     def list_datasets_by_prefix(
         self, prefix: str, project_id: Optional[int] = None
@@ -735,6 +766,18 @@ def create_namespace(
 
         return self.get_namespace(name)
 
+    def remove_namespace(self, namespace_id: int, conn=None) -> None:
+        num_projects = self.count_projects(namespace_id)
+        if num_projects > 0:
+            raise NamespaceDeleteNotAllowedError(
+                f"Namespace cannot be removed. It contains {num_projects} project(s). "
+                "Please remove the project(s) first."
+            )
+
+        n = self._namespaces
+        with self.db.transaction():
+            self.db.execute(self._namespaces_delete().where(n.c.id == namespace_id))
+
     def get_namespace(self, name: str, conn=None) -> Namespace:
         """Gets a single namespace by name"""
         n = self._namespaces
@@ -796,18 +839,6 @@ def create_project(
 
         return self.get_project(name, namespace.name)
 
-    def _is_listing_project(self, project_name: str, namespace_name: str) -> bool:
-        return (
-            project_name == self.listing_project_name
-            and namespace_name == self.system_namespace_name
-        )
-
-    def _is_default_project(self, project_name: str, namespace_name: str) -> bool:
-        return (
-            project_name == self.default_project_name
-            and namespace_name == self.default_namespace_name
-        )
-
     def get_project(
         self, name: str, namespace_name: str, create: bool = False, conn=None
     ) -> Project:
@@ -816,7 +847,7 @@ def get_project(
         p = self._projects
         validate = True
 
-        if self._is_listing_project(name, namespace_name) or self._is_default_project(
+        if self.is_listing_project(name, namespace_name) or self.is_default_project(
             name, namespace_name
         ):
             # we are always creating default and listing projects if they don't exist
@@ -858,7 +889,31 @@ def get_project_by_id(self, project_id: int, conn=None) -> Project:
             raise ProjectNotFoundError(f"Project with id {project_id} not found.")
         return self.project_class.parse(*rows[0])
 
-    def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]:
+    def count_projects(self, namespace_id: Optional[int] = None) -> int:
+        p = self._projects
+        query = self._projects_select()
+        if namespace_id:
+            query = query.where(p.c.namespace_id == namespace_id)
+
+        query = select(f.count(1)).select_from(query.subquery())
+
+        return next(self.db.execute(query))[0]
+
+    def remove_project(self, project_id: int, conn=None) -> None:
+        num_datasets = self.count_datasets(project_id)
+        if num_datasets > 0:
+            raise ProjectDeleteNotAllowedError(
+                f"Project cannot be removed. It contains {num_datasets} dataset(s). "
+                "Please remove the dataset(s) first."
+            )
+
+        p = self._projects
+        with self.db.transaction():
+            self.db.execute(self._projects_delete().where(p.c.id == project_id))
+
+    def list_projects(
+        self, namespace_id: Optional[int] = None, conn=None
+    ) -> list[Project]:
         """
         Gets a list of projects inside some namespace, or in all namespaces
         """
@@ -1189,7 +1244,6 @@ def _base_list_datasets_query(self) -> "Select":
     def list_datasets(
         self, project_id: Optional[int] = None
     ) -> Iterator["DatasetListRecord"]:
-        """Lists all datasets."""
         d = self._datasets
         query = self._base_list_datasets_query().order_by(
             self._datasets.c.name, self._datasets_versions.c.version
@@ -1198,6 +1252,16 @@ def list_datasets(
             query = query.where(d.c.project_id == project_id)
         yield from self._parse_dataset_list(self.db.execute(query))
 
+    def count_datasets(self, project_id: Optional[int] = None) -> int:
+        d = self._datasets
+        query = self._datasets_select()
+        if project_id:
+            query = query.where(d.c.project_id == project_id)
+
+        query = select(f.count(1)).select_from(query.subquery())
+
+        return next(self.db.execute(query))[0]
+
     def list_datasets_by_prefix(
         self, prefix: str, project_id: Optional[int] = None, conn=None
     ) -> Iterator["DatasetListRecord"]:

src/datachain/error.py

@@ -34,6 +34,14 @@ class ProjectCreateNotAllowedError(NotAllowedError):
     pass
 
 
+class ProjectDeleteNotAllowedError(NotAllowedError):
+    pass
+
+
+class NamespaceDeleteNotAllowedError(NotAllowedError):
+    pass
+
+
 class ProjectNotFoundError(NotFoundError):
     pass
 

src/datachain/lib/namespaces.py

@@ -1,6 +1,9 @@
 from typing import Optional
 
-from datachain.error import NamespaceCreateNotAllowedError
+from datachain.error import (
+    NamespaceCreateNotAllowedError,
+    NamespaceDeleteNotAllowedError,
+)
 from datachain.namespace import Namespace
 from datachain.query import Session
 
@@ -71,3 +74,49 @@ def ls(session: Optional[Session] = None) -> list[Namespace]:
         ```
     """
     return Session.get(session).catalog.metastore.list_namespaces()
+
+
+def delete(name: str, session: Optional[Session]) -> None:
+    """
+    Removes a namespace by name.
+
+    Raises:
+        NamespaceNotFoundError: If the namespace does not exist.
+        NamespaceDeleteNotAllowedError: If the namespace is non-empty,
+            is the default namespace, or is a system namespace,
+            as these cannot be removed.
+
+    Parameters:
+        name : The name of the namespace.
+        session : Session to use for getting project.
+
+    Example:
+        ```py
+        import datachain as dc
+        from datachain.lib.namespace import delete as delete_namespace
+        delete_namespace("dev")
+        ```
+    """
+    session = Session.get(session)
+    metastore = session.catalog.metastore
+
+    namespace = metastore.get_namespace(name)
+
+    if name == metastore.system_namespace_name:
+        raise NamespaceDeleteNotAllowedError(
+            f"Namespace {metastore.system_namespace_name} cannot be removed"
+        )
+
+    if name == metastore.default_namespace_name:
+        raise NamespaceDeleteNotAllowedError(
+            f"Namespace {metastore.default_namespace_name} cannot be removed"
+        )
+
+    num_projects = metastore.count_projects(namespace.id)
+    if num_projects > 0:
+        raise NamespaceDeleteNotAllowedError(
+            f"Namespace cannot be removed. It contains {num_projects} project(s). "
+            "Please remove the project(s) first."
+        )
+
+    metastore.remove_namespace(namespace.id)

src/datachain/lib/projects.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from datachain.error import ProjectCreateNotAllowedError
+from datachain.error import ProjectCreateNotAllowedError, ProjectDeleteNotAllowedError
 from datachain.project import Project
 from datachain.query import Session
 
@@ -86,3 +86,49 @@ def ls(
         namespace_id = session.catalog.metastore.get_namespace(namespace).id
 
     return session.catalog.metastore.list_projects(namespace_id)
+
+
+def delete(name: str, namespace: str, session: Optional[Session] = None) -> None:
+    """
+    Removes a project by name within a namespace.
+
+    Raises:
+        ProjectNotFoundError: If the project does not exist.
+        ProjectDeleteNotAllowedError: If the project is non-empty,
+            is the default project, or is a listing project,
+            as these cannot be removed.
+
+    Parameters:
+        name : The name of the project.
+        namespace : The name of the namespace.
+        session : Session to use for getting project.
+
+    Example:
+        ```py
+        import datachain as dc
+        dc.delete_project("my-project", "local")
+        ```
+    """
+    session = Session.get(session)
+    metastore = session.catalog.metastore
+
+    project = metastore.get_project(name, namespace)
+
+    if metastore.is_listing_project(name, namespace):
+        raise ProjectDeleteNotAllowedError(
+            f"Project {metastore.listing_project_name} cannot be removed"
+        )
+
+    if metastore.is_default_project(name, namespace):
+        raise ProjectDeleteNotAllowedError(
+            f"Project {metastore.default_project_name} cannot be removed"
+        )
+
+    num_datasets = metastore.count_datasets(project.id)
+    if num_datasets > 0:
+        raise ProjectDeleteNotAllowedError(
+            f"Project cannot be removed. It contains {num_datasets} dataset(s). "
+            "Please remove the dataset(s) first."
+        )
+
+    metastore.remove_project(project.id)

tests/unit/lib/test_namespace.py

@@ -3,11 +3,14 @@
 from datachain.error import (
     InvalidNamespaceNameError,
     NamespaceCreateNotAllowedError,
+    NamespaceDeleteNotAllowedError,
     NamespaceNotFoundError,
 )
 from datachain.lib.namespaces import create as create_namespace
+from datachain.lib.namespaces import delete as delete_namespace
 from datachain.lib.namespaces import get as get_namespace
 from datachain.lib.namespaces import ls as ls_namespaces
+from datachain.lib.projects import create as create_project
 from tests.utils import skip_if_not_sqlite
 
 
@@ -77,3 +80,45 @@ def test_ls_namespaces_just_local(test_session):
     system_namespace_name = test_session.catalog.metastore.system_namespace_name
     namespaces = ls_namespaces(session=test_session)
     assert [n.name for n in namespaces] == [system_namespace_name]
+
+
+def test_delete_namespace(test_session, dev_namespace):
+    delete_namespace(dev_namespace.name, session=test_session)
+    with pytest.raises(NamespaceNotFoundError):
+        get_namespace(dev_namespace.name, session=test_session)
+
+
+def test_delete_namespace_not_found(test_session, dev_namespace):
+    with pytest.raises(NamespaceNotFoundError):
+        delete_namespace("missing", session=test_session)
+
+
+def test_delete_namespace_system(test_session):
+    metastore = test_session.catalog.metastore
+    with pytest.raises(NamespaceDeleteNotAllowedError) as excinfo:
+        delete_namespace(metastore.system_namespace_name, session=test_session)
+    assert str(excinfo.value) == (
+        f"Namespace {metastore.system_namespace_name} cannot be removed"
+    )
+
+
+def test_delete_namespace_default(test_session):
+    metastore = test_session.catalog.metastore
+    metastore.create_namespace(metastore.default_namespace_name, validate=False)
+    with pytest.raises(NamespaceDeleteNotAllowedError) as excinfo:
+        delete_namespace(metastore.default_namespace_name, session=test_session)
+    assert str(excinfo.value) == (
+        f"Namespace {metastore.default_namespace_name} cannot be removed"
+    )
+
+
+def test_delete_namespace_non_empty(test_session, dev_namespace):
+    create_project(dev_namespace.name, "my-project")
+
+    with pytest.raises(NamespaceDeleteNotAllowedError) as excinfo:
+        delete_namespace(dev_namespace.name, session=test_session)
+
+    assert str(excinfo.value) == (
+        "Namespace cannot be removed. It contains 1 project(s)."
+        " Please remove the project(s) first."
+    )