Fix bucket successfully deleted from S3 but DB update fails

SPRINTLOG.md

@@ -382,3 +382,4 @@ _Nothing merged in CLI during this sprint_
 # 2024-04-8 - 2024-04-19
 
 - New version: 2.6.4 ([#1526](https://github.com/ScilifelabDataCentre/dds_web/pull/1526))
+- Fix raising error when archiving project, bucket deleted but DB error ([#1524](https://github.com/ScilifelabDataCentre/dds_web/pull/1524))

dds_web/api/project.py

@@ -767,19 +767,26 @@ def delete_project_contents(project, delete_bucket=False):
             sqlalchemy.exc.OperationalError,
             AttributeError,
         ) as sqlerr:
-            raise DeletionError(
-                project=project.public_id,
-                message=str(sqlerr),
-                alt_message=(
+            if flask.request:
+                raise DeletionError(
+                    project=project.public_id,
+                    message=str(sqlerr),
+                    alt_message=(
+                        "Project bucket contents were deleted, but they were not deleted from the "
+                        "database. Please contact SciLifeLab Data Centre."
+                        + (
+                            "Database malfunction."
+                            if isinstance(sqlerr, sqlalchemy.exc.OperationalError)
+                            else "."
+                        )
+                    ),
+                ) from sqlerr
+            else:
+                error_msg = (
                     "Project bucket contents were deleted, but they were not deleted from the "
                     "database. Please contact SciLifeLab Data Centre."
-                    + (
-                        "Database malfunction."
-                        if isinstance(err, sqlalchemy.exc.OperationalError)
-                        else "."
-                    )
-                ),
-            ) from sqlerr
+                )
+                flask.current_app.logger.exception(error_msg)
 
 
 class CreateProject(flask_restful.Resource):

dds_web/commands.py

@@ -678,10 +678,24 @@ def set_expired_to_archived():
                         project.current_status,
                         project.current_deadline,
                     )
-                    new_status_row, delete_message = archive.archive_project(
-                        project=project,
-                        current_time=current_time(),
-                    )
+                    try:
+                        new_status_row, delete_message = archive.archive_project(
+                            project=project,
+                            current_time=current_time(),
+                        )
+                    except (
+                        sqlalchemy.exc.OperationalError,
+                        sqlalchemy.exc.SQLAlchemyError,
+                    ):
+                        # Save error message and continue to next project
+                        db.session.rollback()
+                        error_msg = (
+                            "Project bucket contents were deleted, but they were not deleted from the "
+                            "database. Please contact SciLifeLab Data Centre."
+                        )
+                        errors[unit.name][project.public_id] = error_msg
+                        continue
+
                     flask.current_app.logger.debug(delete_message.strip())
                     project.project_statuses.append(new_status_row)
 
@@ -694,6 +708,7 @@ def set_expired_to_archived():
                         sqlalchemy.exc.OperationalError,
                         sqlalchemy.exc.SQLAlchemyError,
                     ) as err:
+                        # commit operation failed, save error message, log it and continue to next project
                         flask.current_app.logger.exception(err)
                         db.session.rollback()
                         errors[unit.name][project.public_id] = str(err)

tests/api/test_project.py

@@ -9,6 +9,8 @@
 import datetime
 import time
 import unittest.mock
+from unittest.mock import MagicMock
+from unittest.mock import patch
 
 # Installed
 import boto3
@@ -510,6 +512,49 @@ def test_projectstatus_archived_project(module_client, boto3_session):
     assert project.researchusers
 
 
+def test_projectstatus_archived_project_db_fail(
+    module_client, boto3_session, capfd: LogCaptureFixture
+):
+    """Create a project and archive it fails in DB update"""
+
+    # Create unit admins to allow project creation
+    current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count()
+    if current_unit_admins < 3:
+        create_unit_admins(num_admins=2)
+    current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count()
+    assert current_unit_admins >= 3
+
+    response = module_client.post(
+        tests.DDSEndpoint.PROJECT_CREATE,
+        headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(module_client),
+        json=proj_data,
+    )
+    assert response.status_code == http.HTTPStatus.OK
+    project_id = response.json.get("project_id")
+
+    # change status and mock fail in DB operation
+    mock_query = MagicMock()
+    mock_query.filter.return_value.delete.side_effect = sqlalchemy.exc.OperationalError(
+        "OperationalError", "test", "sqlalchemy"
+    )
+    with patch("dds_web.database.models.File.query", mock_query):
+        new_status = {"new_status": "Archived"}
+        response = module_client.post(
+            tests.DDSEndpoint.PROJECT_STATUS,
+            headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(module_client),
+            query_string={"project": project_id},
+            json=new_status,
+        )
+        assert response.status_code == http.HTTPStatus.INTERNAL_SERVER_ERROR
+
+    _, err = capfd.readouterr()
+    assert "DeletionError" in err
+    assert (
+        "Project bucket contents were deleted, but they were not deleted from the database. Please contact SciLifeLab Data Centre"
+        in err
+    )
+
+
 def test_projectstatus_aborted_project(module_client, boto3_session):
     """Create a project and try to abort it"""
     # Create unit admins to allow project creation

tests/test_commands.py

@@ -1395,6 +1395,34 @@ def test_set_expired_to_archived(_: MagicMock, client, cli_runner):
     assert j == 6
 
 
+@mock.patch("boto3.session.Session")
+def test_set_expired_to_archived_db_failed(
+    _: MagicMock, client, cli_runner, capfd: LogCaptureFixture
+):
+    """Reproduce the error when the s3 bucket is deleted but the DB update fails."""
+    # Get the project and set up as expired
+    project = models.Project.query.filter_by(public_id="public_project_id").one_or_none()
+    for status in project.project_statuses:
+        status.deadline = current_time() - timedelta(weeks=1)
+        status.status = "Expired"
+
+    mock_query = MagicMock()
+    mock_query.filter.return_value.delete.side_effect = sqlalchemy.exc.OperationalError(
+        "OperationalError", "test", "sqlalchemy"
+    )
+    with patch("dds_web.database.models.File.query", mock_query):
+        with patch("flask.request", False):
+            cli_runner.invoke(set_expired_to_archived)
+
+    # Check the logs for the error message
+    _, err = capfd.readouterr()
+    print(err)
+    assert (
+        "Project bucket contents were deleted, but they were not deleted from the database. Please contact SciLifeLab Data Centre."
+    ) in err
+    assert ("SQL: OperationalError") in err
+
+
 # delete invites