Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add catch of OperationalError in files.py #1089

Merged
merged 3 commits into from
Mar 25, 2022
Merged

Add catch of OperationalError in files.py #1089

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4c7b607
operational error in files.py
Mar 25, 2022
acaadfe
changelog
Mar 25, 2022
5c7e373
Update dds_web/api/files.py
i-oden Mar 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,3 +59,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe
- Add possibility of deleting invites (temporary fix in delete user endpoint) ([#1075](https://github.com/ScilifelabDataCentre/dds_web/pull/1075))
- Flask command `create-unit` to create unit without having to interact with database directly ([#1075](https://github.com/ScilifelabDataCentre/dds_web/pull/1075))
- Let project description include . and , ([#1080](https://github.com/ScilifelabDataCentre/dds_web/pull/1080))
- Catch OperationalError if there is a database malfunction in `files.py` ([#1089](https://github.com/ScilifelabDataCentre/dds_web/pull/1089))
126 changes: 102 additions & 24 deletions dds_web/api/files.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,18 @@ def post(self):

try:
db.session.commit()
except sqlalchemy.exc.SQLAlchemyError as err:
except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err:
flask.current_app.logger.debug(err)
db.session.rollback()
raise DatabaseError("Failed to add new file to database.") from err
raise DatabaseError(
message=str(err),
alt_message="Failed to add new file to database"
+ (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
),
) from err

return {"message": f"File '{new_file.name}' added to db."}

Expand Down Expand Up @@ -176,9 +184,17 @@ def put(self):

db.session.add(new_version)
db.session.commit()
except sqlalchemy.exc.SQLAlchemyError as err:
except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err:
db.session.rollback()
raise DatabaseError(f"Failed updating file information: {err}") from err
raise DatabaseError(
message=str(err),
alt_message=f"Failed updating file information"
+ (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
),
) from err

return {"message": f"File '{file_info.get('name')}' updated in db."}

Expand All @@ -205,8 +221,16 @@ def get(self):
.filter(models.File.project_id == sqlalchemy.func.binary(project.id))
.all()
)
except sqlalchemy.exc.SQLAlchemyError as err:
raise DatabaseError(f"Failed to get matching files in db: {err}") from err
except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err:
raise DatabaseError(
message=str(err),
alt_message=f"Failed to get matching files in db"
+ (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
),
) from err

# The files checked are not in the db
if not matching_files or matching_files is None:
Expand Down Expand Up @@ -291,8 +315,16 @@ def get_folder_size(self, project, folder_name):
.first()
)

except sqlalchemy.exc.SQLAlchemyError as err:
raise DatabaseError(message=str(err)) from err
except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err:
raise DatabaseError(
message=str(err),
alt_message=f"Could not get size of folder '{folder_name}'"
+ (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
),
) from err

return file_info.sizeSum

Expand Down Expand Up @@ -354,8 +386,16 @@ def items_in_subpath(project, folder="."):
)
distinct_folders = list(split_paths)

except sqlalchemy.exc.SQLAlchemyError as err:
raise DatabaseError(message=str(err)) from err
except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err:
raise DatabaseError(
message=str(err),
alt_message=f"Could not get items in {f'folder {folder}' if folder != '.' else 'root'}"
+ (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
),
) from err
else:
return distinct_files, distinct_folders

Expand Down Expand Up @@ -404,9 +444,25 @@ def delete_multiple(self, project, files):
db.session.rollback()
not_exist_list.append(entry)
continue
except (sqlalchemy.exc.SQLAlchemyError, DatabaseError) as err:
except (
sqlalchemy.exc.SQLAlchemyError,
DatabaseError,
sqlalchemy.exc.OperationalError,
) as err:
db.session.rollback()
not_removed_dict[entry] = str(err)
flask.current_app.logger.exception(err)
not_removed_dict[entry] = (
str(err)
if isinstance(err, DatabaseError)
else (
"Could not collect the remote file name"
+ (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
)
)
)
continue

# Remove from s3 bucket
Expand All @@ -420,9 +476,14 @@ def delete_multiple(self, project, files):
# Commit to db if ok
try:
db.session.commit()
except sqlalchemy.exc.SQLAlchemyError as err:
except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err:
db.session.rollback()
not_removed_dict[entry] = str(err)
flask.current_app.logger.exception(err)
not_removed_dict[entry] = "Could not remove data" + (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
)
continue

return not_removed_dict, not_exist_list
Expand Down Expand Up @@ -505,13 +566,14 @@ def delete(self):
except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err:
db.session.rollback()
flask.current_app.logger.error(
"Files deleted in S3 but not in db. The entries must be synchronised!"
"Files deleted in S3 but not in db. The entries must be synchronised! "
f"Error: {str(err)}"
)
not_removed[folder_name] = "Could not remove files in folder" + (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
)
if isinstance(err, sqlalchemy.exc.OperationalError):
err_msg = "Database malfunction."
else:
err_msg = str(err)
not_removed[folder_name] = err_msg
fail_type = "db"
break

Expand Down Expand Up @@ -541,8 +603,16 @@ def get_files_for_deletion(self, project: str, folder: str):
)
.all()
)
except sqlalchemy.exc.SQLAlchemyError as err:
raise DatabaseError(message=str(err)) from err
except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err:
raise DatabaseError(
message=str(err),
alt_message="Could not collect files for deletion"
+ (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
),
) from err

return files

Expand Down Expand Up @@ -651,10 +721,18 @@ def put(self):
raise NoSuchFileError()

file.time_latest_download = dds_web.utils.current_time()
except sqlalchemy.exc.SQLAlchemyError as err:
except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err:
db.session.rollback()
flask.current_app.logger.exception(str(err))
raise DatabaseError("Update of file info failed.") from err
raise DatabaseError(
message=str(err),
alt_message="Update of file info failed"
+ (
": Database malfunction."
if isinstance(err, sqlalchemy.exc.OperationalError)
else "."
),
) from err
else:
# flask.current_app.logger.debug("File %s updated", file_name)
db.session.commit()
Expand Down