Skip to content

Commit

Permalink
Merge pull request #1089 from ScilifelabDataCentre/operationalerror-i…
Browse files Browse the repository at this point in the history 
…n-files

Add catch of OperationalError in `files.py`
  • Loading branch information
@i-oden
i-oden authored Mar 25, 2022
2 parents 0e48c08 + 5c7e373 commit 5a20585
Show file tree
Hide file tree
Showing 2 changed files with 103 additions and 24 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,3 +59,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe
- Add possibility of deleting invites (temporary fix in delete user endpoint) ([#1075](https://github.com/ScilifelabDataCentre/dds_web/pull/1075))
- Flask command `create-unit` to create unit without having to interact with database directly ([#1075](https://github.com/ScilifelabDataCentre/dds_web/pull/1075))
- Let project description include . and , ([#1080](https://github.com/ScilifelabDataCentre/dds_web/pull/1080))
- Catch OperationalError if there is a database malfunction in `files.py` ([#1089](https://github.com/ScilifelabDataCentre/dds_web/pull/1089))
126 changes: 102 additions & 24 deletions dds_web/api/files.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,18 @@ def post(self):

try:
db.session.commit()
except sqlalchemy.exc.SQLAlchemyError as err:
except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err:
flask.current_app.logger.debug(err)
db.session.rollback()
raise DatabaseError("Failed to add new file to database.") from err
raise DatabaseError(
message=str(err),
alt_message="Failed to add new file to database"
+ (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
),
) from err

return {"message": f"File '{new_file.name}' added to db."}

Expand Down Expand Up @@ -176,9 +184,17 @@ def put(self):

db.session.add(new_version)
db.session.commit()
except sqlalchemy.exc.SQLAlchemyError as err:
except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err:
db.session.rollback()
raise DatabaseError(f"Failed updating file information: {err}") from err
raise DatabaseError(
message=str(err),
alt_message=f"Failed updating file information"
+ (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
),
) from err

return {"message": f"File '{file_info.get('name')}' updated in db."}

Expand All @@ -205,8 +221,16 @@ def get(self):
.filter(models.File.project_id == sqlalchemy.func.binary(project.id))
.all()
)
except sqlalchemy.exc.SQLAlchemyError as err:
raise DatabaseError(f"Failed to get matching files in db: {err}") from err
except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err:
raise DatabaseError(
message=str(err),
alt_message=f"Failed to get matching files in db"
+ (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
),
) from err

# The files checked are not in the db
if not matching_files or matching_files is None:
Expand Down Expand Up @@ -291,8 +315,16 @@ def get_folder_size(self, project, folder_name):
.first()
)

except sqlalchemy.exc.SQLAlchemyError as err:
raise DatabaseError(message=str(err)) from err
except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err:
raise DatabaseError(
message=str(err),
alt_message=f"Could not get size of folder '{folder_name}'"
+ (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
),
) from err

return file_info.sizeSum

Expand Down Expand Up @@ -354,8 +386,16 @@ def items_in_subpath(project, folder="."):
)
distinct_folders = list(split_paths)

except sqlalchemy.exc.SQLAlchemyError as err:
raise DatabaseError(message=str(err)) from err
except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err:
raise DatabaseError(
message=str(err),
alt_message=f"Could not get items in {f'folder {folder}' if folder != '.' else 'root'}"
+ (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
),
) from err
else:
return distinct_files, distinct_folders

Expand Down Expand Up @@ -404,9 +444,25 @@ def delete_multiple(self, project, files):
db.session.rollback()
not_exist_list.append(entry)
continue
except (sqlalchemy.exc.SQLAlchemyError, DatabaseError) as err:
except (
sqlalchemy.exc.SQLAlchemyError,
DatabaseError,
sqlalchemy.exc.OperationalError,
) as err:
db.session.rollback()
not_removed_dict[entry] = str(err)
flask.current_app.logger.exception(err)
not_removed_dict[entry] = (
str(err)
if isinstance(err, DatabaseError)
else (
"Could not collect the remote file name"
+ (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
)
)
)
continue

# Remove from s3 bucket
Expand All @@ -420,9 +476,14 @@ def delete_multiple(self, project, files):
# Commit to db if ok
try:
db.session.commit()
except sqlalchemy.exc.SQLAlchemyError as err:
except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err:
db.session.rollback()
not_removed_dict[entry] = str(err)
flask.current_app.logger.exception(err)
not_removed_dict[entry] = "Could not remove data" + (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
)
continue

return not_removed_dict, not_exist_list
Expand Down Expand Up @@ -505,13 +566,14 @@ def delete(self):
except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err:
db.session.rollback()
flask.current_app.logger.error(
"Files deleted in S3 but not in db. The entries must be synchronised!"
"Files deleted in S3 but not in db. The entries must be synchronised! "
f"Error: {str(err)}"
)
not_removed[folder_name] = "Could not remove files in folder" + (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
)
if isinstance(err, sqlalchemy.exc.OperationalError):
err_msg = "Database malfunction."
else:
err_msg = str(err)
not_removed[folder_name] = err_msg
fail_type = "db"
break

Expand Down Expand Up @@ -541,8 +603,16 @@ def get_files_for_deletion(self, project: str, folder: str):
)
.all()
)
except sqlalchemy.exc.SQLAlchemyError as err:
raise DatabaseError(message=str(err)) from err
except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err:
raise DatabaseError(
message=str(err),
alt_message="Could not collect files for deletion"
+ (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
),
) from err

return files

Expand Down Expand Up @@ -651,10 +721,18 @@ def put(self):
raise NoSuchFileError()

file.time_latest_download = dds_web.utils.current_time()
except sqlalchemy.exc.SQLAlchemyError as err:
except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err:
db.session.rollback()
flask.current_app.logger.exception(str(err))
raise DatabaseError("Update of file info failed.") from err
raise DatabaseError(
message=str(err),
alt_message="Update of file info failed"
+ (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
),
) from err
else:
# flask.current_app.logger.debug("File %s updated", file_name)
db.session.commit()
Expand Down

0 comments on commit 5a20585

Please sign in to comment.