Merge branch 'dev' into decision-regarding-2024

ScilifelabDataCentre · Nov 2, 2023 · 742f1dd · 742f1dd
2 parents 9887732 + 90614a2
commit 742f1dd
Show file tree

Hide file tree

Showing 9 changed files with 897 additions and 60 deletions.
diff --git a/SPRINTLOG.md b/SPRINTLOG.md
@@ -315,8 +315,9 @@ _Nothing merged in CLI during this sprint_
 - Add flag --verify-checksum to the comand in email template ([#1478])(https://github.com/ScilifelabDataCentre/dds_web/pull/1478)
 - Improved email layout; Highlighted information and commands when project is released ([#1479])(https://github.com/ScilifelabDataCentre/dds_web/pull/1479)
 
-# 2023-10-16 - 2023-10-27
+# 2023-10-16 - 2023-11-03 (Longer sprint due to OKR prep and höstlov)
 
 - Added new API endpoint ProjectStatus.patch to extend the deadline ([#1480])(https://github.com/ScilifelabDataCentre/dds_web/pull/1480)
 - New version: 2.5.2 ([#1482](https://github.com/ScilifelabDataCentre/dds_web/pull/1482))
+- New endpoint `AddFailedFiles` for adding failed files to database ([#1472])(https://github.com/ScilifelabDataCentre/dds_web/pull/1472)
 - New ADR record regarding OKR 2024 ([#1483](https://github.com/ScilifelabDataCentre/dds_web/pull/1483))
diff --git a/dds_web/api/__init__.py b/dds_web/api/__init__.py
@@ -50,6 +50,7 @@ def output_json(data, code, headers=None):
 api.add_resource(files.FileInfo, "/file/info", endpoint="file_info")
 api.add_resource(files.FileInfoAll, "/file/all/info", endpoint="all_file_info")
 api.add_resource(files.UpdateFile, "/file/update", endpoint="update_file")
+api.add_resource(files.AddFailedFiles, "/file/failed/add", endpoint="add_failed_files")
 
 # Projects ############################################################################## Projects #
 api.add_resource(project.UserProjects, "/proj/list", endpoint="list_projects")

diff --git a/dds_web/api/files.py b/dds_web/api/files.py
@@ -740,3 +740,22 @@ def put(self):
             db.session.commit()
 
         return {"message": "File info updated."}
+
+
+class AddFailedFiles(flask_restful.Resource):
+    """Get files from log file and save to database."""
+
+    @auth.login_required(role=["Unit Admin", "Unit Personnel"])
+    @json_required
+    @handle_validation_errors
+    def put(self):
+        """Run flask command with failed_delivery_log."""
+
+        # Verify project ID and access
+        project = project_schemas.ProjectRequiredSchema().load(flask.request.args)
+
+        # Get the request json and pass it to add_uploaded_files_to_db
+        request_json = flask.request.get_json(silent=True)
+
+        files_added, errors = dds_web.utils.add_uploaded_files_to_db(project, request_json)
+        return {"files_added": [file.name for file in files_added], "message": errors}
diff --git a/dds_web/commands.py b/dds_web/commands.py
@@ -211,71 +211,28 @@ def update_uploaded_file_with_log(project, path_to_log_file):
     """Update file details that weren't properly uploaded to db from cli log"""
     import botocore
     from dds_web.database import models
-    from dds_web import db
-    from dds_web.api.api_s3_connector import ApiS3Connector
+    from dds_web import utils
     import json
 
     proj_in_db = models.Project.query.filter_by(public_id=project).one_or_none()
     if not proj_in_db:
         flask.current_app.logger.error(f"The project '{project}' doesn't exist.")
         return
+    flask.current_app.logger.debug(f"Updating file in project '{project}'...")
 
     if not os.path.exists(path_to_log_file):
         flask.current_app.logger.error(f"The log file '{path_to_log_file}' doesn't exist.")
         return
+    flask.current_app.logger.debug(f"Reading file info from path '{path_to_log_file}'...")
 
     with open(path_to_log_file, "r") as f:
         log = json.load(f)
-    errors = {}
-    files_added = []
-    for file, vals in log.items():
-        status = vals.get("status")
-        if not status or not status.get("failed_op") == "add_file_db":
-            continue
-
-        with ApiS3Connector(project=proj_in_db) as s3conn:
-            try:
-                _ = s3conn.resource.meta.client.head_object(
-                    Bucket=s3conn.project.bucket, Key=vals["path_remote"]
-                )
-            except botocore.client.ClientError as err:
-                if err.response["Error"]["Code"] == "404":
-                    errors[file] = {"error": "File not found in S3", "traceback": err.__traceback__}
-            else:
-                file_object = models.File.query.filter(
-                    sqlalchemy.and_(
-                        models.File.name == sqlalchemy.func.binary(file),
-                        models.File.project_id == proj_in_db.id,
-                    )
-                ).first()
-                if file_object:
-                    errors[file] = {"error": "File already in database."}
-                else:
-                    new_file = models.File(
-                        name=file,
-                        name_in_bucket=vals["path_remote"],
-                        subpath=vals["subpath"],
-                        project_id=proj_in_db.id,
-                        size_original=vals["size_raw"],
-                        size_stored=vals["size_processed"],
-                        compressed=not vals["compressed"],
-                        public_key=vals["public_key"],
-                        salt=vals["salt"],
-                        checksum=vals["checksum"],
-                    )
-                    new_version = models.Version(
-                        size_stored=new_file.size_stored, time_uploaded=datetime.datetime.utcnow()
-                    )
-                    proj_in_db.file_versions.append(new_version)
-                    proj_in_db.files.append(new_file)
-                    new_file.versions.append(new_version)
+        flask.current_app.logger.debug("File contents were loaded...")
 
-                    db.session.add(new_file)
-                    files_added.append(new_file)
-                db.session.commit()
+    files_added, errors = utils.add_uploaded_files_to_db(proj_in_db=proj_in_db, log=log)
 
-        flask.current_app.logger.info(f"Files added: {files_added}")
-        flask.current_app.logger.info(f"Errors while adding files: {errors}")
+    flask.current_app.logger.info(f"Files added: {files_added}")
+    flask.current_app.logger.info(f"Errors while adding files: {errors}")
 
 
 @click.group(name="lost-files")

diff --git a/dds_web/utils.py b/dds_web/utils.py
@@ -13,6 +13,7 @@
 import time
 import smtplib
 from dateutil.relativedelta import relativedelta
+import gc
 
 # Installed
 import botocore
@@ -749,3 +750,171 @@ def use_sto4(unit_object, project_object) -> bool:
 
     flask.current_app.logger.info(f"{project_id_logging}sto2")
     return False
+
+
+def add_uploaded_files_to_db(proj_in_db, log: typing.Dict):
+    """Adds uploaded files to the database.
+
+    Args:
+        proj_in_db (dds_web.models.Project): The project to add the files to.
+        log (typing.Dict): A dictionary containing information about the uploaded files.
+
+    Returns:
+        A tuple containing a list of files that were successfully added to the database,
+        and a dictionary containing any errors that occurred while
+        adding the files.
+    """
+    # Import necessary modules and initialize variables
+    from dds_web import db
+    from dds_web.api.api_s3_connector import ApiS3Connector
+
+    errors = {}
+    files_added = []
+
+    flask.current_app.logger.info(type(log))
+    # Loop through each file in the log
+    for file, vals in log.items():
+        status = vals.get("status")
+        overwrite = vals.get("overwrite", False)
+
+        # Check if the file was successfully uploaded but database not updated
+        if not status or not status.get("failed_op") == "add_file_db":
+            errors[file] = {"error": "Incorrect 'failed_op'."}
+            continue
+
+        # Connect to S3 and check if the file exists
+        with ApiS3Connector(project=proj_in_db) as s3conn:
+            try:
+                _ = s3conn.resource.meta.client.head_object(
+                    Bucket=s3conn.project.bucket, Key=vals["path_remote"]
+                )
+            except botocore.client.ClientError as err:
+                if err.response["Error"]["Code"] == "404":
+                    errors[file] = {"error": "File not found in S3", "traceback": err.__traceback__}
+            else:
+                try:
+                    # Check if the file already exists in the database
+                    file_object = models.File.query.filter(
+                        sqlalchemy.and_(
+                            models.File.name == sqlalchemy.func.binary(file),
+                            models.File.project_id == proj_in_db.id,
+                        )
+                    ).first()
+
+                    # If the file already exists, create a new version of it if "--overwrite" was specified
+                    if file_object:
+                        if overwrite:
+                            try:
+                                new_file_version(existing_file=file_object, new_info=vals)
+                                files_added.append(file_object)
+                            except KeyError as err:
+                                errors[file] = {"error": f"Missing key: {err}"}
+                        else:
+                            errors[file] = {"error": "File already in database."}
+
+                    # If the file does not exist, create a new file and version
+                    else:
+                        new_file = models.File(
+                            name=file,
+                            name_in_bucket=vals["path_remote"],
+                            subpath=vals["subpath"],
+                            project_id=proj_in_db.id,
+                            size_original=vals["size_raw"],
+                            size_stored=vals["size_processed"],
+                            compressed=not vals["compressed"],
+                            public_key=vals["public_key"],
+                            salt=vals["salt"],
+                            checksum=vals["checksum"],
+                        )
+                        new_version = models.Version(
+                            size_stored=new_file.size_stored,
+                            time_uploaded=datetime.datetime.utcnow(),
+                        )
+                        proj_in_db.file_versions.append(new_version)
+                        proj_in_db.files.append(new_file)
+                        new_file.versions.append(new_version)
+
+                        db.session.add(new_file)
+                        db.session.commit()
+                        files_added.append(new_file)
+                except (
+                    sqlalchemy.exc.IntegrityError,
+                    sqlalchemy.exc.OperationalError,
+                    sqlalchemy.exc.SQLAlchemyError,
+                ) as err:
+                    errors[file] = {"error": str(err)}
+                    db.session.rollback()
+    if errors:
+        flask.current_app.logger.error(f"Error in new_file_version: {errors}")
+
+    return files_added, errors
+
+
+def new_file_version(existing_file, new_info):
+    """
+    Create new version of a file.
+
+    Args:
+        existing_file (dds_web.models.File): The existing file to create a new version of.
+        new_info (dict): A dictionary containing information about the new version of the file.
+
+    Returns:
+        None
+    """
+    from dds_web import db
+    import dds_web.utils
+
+    # Get project
+    project = existing_file.project
+
+    # Get versions
+    current_file_version = models.Version.query.filter(
+        sqlalchemy.and_(
+            models.Version.active_file == sqlalchemy.func.binary(existing_file.id),
+            models.Version.time_deleted.is_(None),
+        )
+    ).all()
+
+    # If there is more than one version of the file which does not yet have a deletion timestamp, log a warning
+    if len(current_file_version) > 1:
+        flask.current_app.logger.warning(
+            "There is more than one version of the file "
+            "which does not yet have a deletion timestamp."
+        )
+
+    # Same timestamp for deleted and created new version
+    new_timestamp = dds_web.utils.current_time()
+
+    # Set the deletion timestamp for the latests version of the file
+    for version in current_file_version:
+        if version.time_deleted is None:
+            version.time_deleted = new_timestamp
+
+    # Update file info
+    existing_file.subpath = new_info["subpath"]
+    existing_file.size_original = new_info["size_raw"]
+    existing_file.size_stored = new_info["size_processed"]
+    existing_file.compressed = not new_info["compressed"]
+    existing_file.salt = new_info["salt"]
+    existing_file.public_key = new_info["public_key"]
+    existing_file.time_uploaded = new_timestamp
+    existing_file.checksum = new_info["checksum"]
+
+    # Create a new version of the file
+    new_version = models.Version(
+        size_stored=new_info["size_processed"],
+        time_uploaded=new_timestamp,
+        active_file=existing_file.id,
+        project_id=project,
+    )
+
+    # Update foreign keys and relationships
+    project.file_versions.append(new_version)
+    existing_file.versions.append(new_version)
+
+    # Add the new version to the database and commit the changes
+    db.session.add(new_version)
+    db.session.commit()
+
+    # Clean up information
+    del new_info
diff --git a/tests/__init__.py b/tests/__init__.py
@@ -195,6 +195,7 @@ class DDSEndpoint:
     FILE_INFO = BASE_ENDPOINT + "/file/info"
     FILE_INFO_ALL = BASE_ENDPOINT + "/file/all/info"
     FILE_UPDATE = BASE_ENDPOINT + "/file/update"
+    FILE_ADD_FAILED = BASE_ENDPOINT + "/file/failed/add"
 
     # Project specific urls
     PROJECT_CREATE = BASE_ENDPOINT + "/proj/create"