Merge branch 'dev' into trivy-scan-local

CHANGELOG.md

@@ -199,3 +199,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe
 - Version bump: 2.2.6 ([#1375](https://github.com/ScilifelabDataCentre/dds_web/pull/1375))
 - Workflow: Add option to publish dev image manually ([#1376](https://github.com/ScilifelabDataCentre/dds_web/pull/1376))
 - Workflow: Scan branch with trivy ([#1377](https://github.com/ScilifelabDataCentre/dds_web/pull/1377))
+- Bug: Add value to `Unit.warning_level` for existing units ([#1378](https://github.com/ScilifelabDataCentre/dds_web/pull/1379))

dds_web/api/db_tools.py

@@ -23,7 +23,6 @@
 
 
 def remove_user_self_deletion_request(user):
-
     try:
         request_row = models.DeletionRequest.query.filter(
             models.DeletionRequest.requester_id == user.username

dds_web/api/dds_decorators.py

@@ -39,10 +39,9 @@
 def handle_validation_errors(func):
     @functools.wraps(func)
     def handle_error(*args, **kwargs):
-
         try:
             result = func(*args, **kwargs)
-        except (marshmallow.exceptions.ValidationError) as valerr:
+        except marshmallow.exceptions.ValidationError as valerr:
             if "_schema" in valerr.messages:
                 return valerr.messages["_schema"][0], 400
             else:
@@ -56,7 +55,6 @@ def handle_error(*args, **kwargs):
 def json_required(func):
     @functools.wraps(func)
     def verify_json(*args, **kwargs):
-
         if not flask.request.json:
             raise MissingJsonError(message="Required data missing from request!")
 
@@ -68,7 +66,6 @@ def verify_json(*args, **kwargs):
 def args_required(func):
     @functools.wraps(func)
     def verify_args(*args, **kwargs):
-
         if not flask.request.args:
             raise DDSArgumentError(message="Required information missing from request!")
 
@@ -80,7 +77,6 @@ def verify_args(*args, **kwargs):
 def dbsession(func):
     @functools.wraps(func)
     def make_commit(*args, **kwargs):
-
         # Run function, catch errors
         try:
             result = func(*args, **kwargs)
@@ -106,7 +102,6 @@ def make_commit(*args, **kwargs):
 def handle_db_error(func):
     @functools.wraps(func)
     def perform_get(*args, **kwargs):
-
         # Run function, catch errors
         try:
             result = func(*args, **kwargs)
@@ -134,7 +129,6 @@ def connect_cloud(func):
 
     @functools.wraps(func)
     def init_resource(self, *args, **kwargs):
-
         try:
             _, self.keys, self.url, self.bucketname = self.get_s3_info()
             # Connect to service
@@ -190,7 +184,6 @@ def wrapper_logging_bind_request(*args, **kwargs):
             project=flask.request.args.get("project") if flask.request.args else None,
             user=get_username_or_request_ip(),
         ):
-
             try:
                 value = func(*args, **kwargs)
 

dds_web/api/project.py

@@ -46,6 +46,7 @@
 from dds_web.security.auth import get_user_roles_common
 from dds_web.api.files import check_eligibility_for_deletion
 
+
 ####################################################################################################
 # ENDPOINTS ############################################################################ ENDPOINTS #
 ####################################################################################################
@@ -546,7 +547,6 @@ def format_project_dict(self, current_user):
 
     @staticmethod
     def project_usage(project):
-
         # Calculate approximate cost per gbhour: kr per gb per month / (days * hours)
         cost_gbhour = 0.09 / (30 * 24)
         bhours = 0.0

dds_web/api/schemas/project_schemas.py

@@ -215,7 +215,6 @@ class ProjectContentSchema(ProjectRequiredSchema):
     get_all = marshmallow.fields.Boolean(required=False, default=False)
 
     def find_contents(self, project, contents):
-
         # All contents
         all_contents_query = models.File.query.filter(
             models.File.project_id == sqlalchemy.func.binary(project.id)

dds_web/api/user.py

@@ -44,6 +44,7 @@
 # initiate bound logger
 action_logger = structlog.getLogger("actions")
 
+
 ####################################################################################################
 # ENDPOINTS ############################################################################ ENDPOINTS #
 ####################################################################################################
@@ -962,7 +963,6 @@ class SecondFactor(flask_restful.Resource):
     @auth.login_required
     @handle_validation_errors
     def get(self):
-
         token_schemas.TokenSchema().load(flask.request.json)
 
         token_claims = dds_web.security.auth.obtain_current_encrypted_token_claims()
@@ -1038,7 +1038,6 @@ class RequestHOTPActivation(flask_restful.Resource):
     # Using Basic auth since TOTP might have been lost, will still need access to email
     @basic_auth.login_required
     def post(self):
-
         user = auth.current_user()
         json_info = flask.request.json
 
@@ -1136,7 +1135,6 @@ def get(self):
         # Project (bucket) specific info
         usage = {}
         for p in unit_info.projects:
-
             # Define fields in usage dict
             usage[p.public_id] = {"gbhours": 0.0, "cost": 0.0}
 

dds_web/commands.py

@@ -385,7 +385,6 @@ def set_available_to_expired():
                 )
                 .with_for_update()
             ):
-
                 if (
                     project.current_status == "Available"
                     and project.current_deadline <= current_time()
@@ -473,7 +472,6 @@ def set_expired_to_archived():
                 )
                 .with_for_update()
             ):
-
                 if (
                     project.current_status == "Expired"
                     and project.current_deadline <= current_time()

dds_web/errors.py

@@ -159,7 +159,6 @@ def __init__(
         pass_message=False,
         project=None,
     ):
-
         general_logger.error(message)
 
         if project:
@@ -178,7 +177,6 @@ class EmptyProjectException(LoggedHTTPException):
     code = http.HTTPStatus.BAD_REQUEST
 
     def __init__(self, project, username=None, message="The project is empty."):
-
         if not username:
             username = auth.current_user()
         structlog.threadlocal.bind_threadlocal(user=username)
@@ -195,7 +193,6 @@ class DeletionError(LoggedHTTPException):
     code = http.HTTPStatus.INTERNAL_SERVER_ERROR
 
     def __init__(self, project, message, alt_message=None, pass_message=False):
-
         if project:
             structlog.threadlocal.bind_threadlocal(project=project)
 
@@ -209,7 +206,6 @@ class NoSuchProjectError(LoggedHTTPException):
     code = http.HTTPStatus.BAD_REQUEST  # 400
 
     def __init__(self, project, message="The specified project does not exist."):
-
         if project:
             structlog.threadlocal.bind_threadlocal(project=project)
 
@@ -271,7 +267,6 @@ class JwtTokenGenerationError(LoggedHTTPException):
     code = http.HTTPStatus.INTERNAL_SERVER_ERROR
 
     def __init__(self, message="Error during JWT Token generation.", pass_message=False):
-
         general_logger.warning(message)
 
         super().__init__(
@@ -366,7 +361,6 @@ class UserDeletionError(LoggedHTTPException):
     code = http.HTTPStatus.BAD_REQUEST
 
     def __init__(self, message="User deletion failed.", alt_message=None):
-
         general_logger.warning(message)
         super().__init__(alt_message or message)
 
@@ -394,28 +388,23 @@ def __init__(self, message="Specified file does not exist."):
 
 
 class TooManyRequestsError(LoggedHTTPException):
-
     code = http.HTTPStatus.TOO_MANY_REQUESTS
     description = "Too many authentication requests in one hour"
 
     def __init__(self):
-
         super().__init__(self.description)
         general_logger.warning(self.description)
 
 
 class RoleException(LoggedHTTPException):
-
     code = http.HTTPStatus.FORBIDDEN
 
     def __init__(self, message="Invalid role."):
-
         super().__init__(message)
         general_logger.warning(message)
 
 
 class VersionMismatchError(LoggedHTTPException):
-
     code = http.HTTPStatus.FORBIDDEN
 
     def __init__(
@@ -426,7 +415,6 @@ def __init__(
 
 
 class MaintenanceOngoingException(LoggedHTTPException):
-
     code = http.HTTPStatus.SERVICE_UNAVAILABLE
 
     def __init__(self, message="Maintenance of DDS is ongoing."):

dds_web/utils.py

@@ -44,6 +44,7 @@
 
 # General ################################################################################ General #
 
+
 # Cannot have type hint for return due to models.Project giving circular import
 def collect_project(project_id: str):
     """Get project object from database."""

dds_web/web/user.py

@@ -303,7 +303,6 @@ def confirm_2fa():
         return flask.redirect(flask.url_for("auth_blueprint.login", next=next_target))
 
     if form.validate_on_submit():
-
         if user.totp_enabled:
             twofactor_value = form.totp.data
             twofactor_verify = user.verify_TOTP
@@ -637,7 +636,6 @@ def confirm_self_deletion(token):
     s = itsdangerous.URLSafeTimedSerializer(flask.current_app.config.get("SECRET_KEY"))
 
     try:
-
         # Get email from token, overwrite the one from login if applicable
         email = s.loads(token, salt="email-delete", max_age=604800)
 
@@ -655,7 +653,6 @@ def confirm_self_deletion(token):
         ).first()
 
     except itsdangerous.exc.SignatureExpired as exc:
-
         email = db_tools.remove_user_self_deletion_request(flask_login.current_user)
         raise ddserr.UserDeletionError(
             message=f"Deletion request for {email} has expired. Please login to the DDS and request deletion anew."

migrations/versions/0c9c237cced5_unit_quota_and_warning.py

@@ -24,10 +24,11 @@ def upgrade():
 
     # Update existing columns
     # 1. Load table - need to load columns in order to use
-    unit_table = sa.sql.table("units", sa.sql.column("quota", mysql.BIGINT))
-    # 2. Update column value - set value to 100 TB
-    op.execute(unit_table.update().values(quota=100 * (10**12)))
-
+    unit_table = sa.sql.table(
+        "units", sa.sql.column("quota", mysql.BIGINT), sa.sql.column("warning_level", mysql.FLOAT)
+    )
+    # 2. Update column value - set value to 100 TB and 0.8
+    op.execute(unit_table.update().values(quota=100 * (10**12), warning_level=0.8))
     # ### end Alembic commands ###
 
 

tests/__init__.py

@@ -43,6 +43,7 @@
 
 DEFAULT_HEADER = {"X-CLI-Version": __version__}
 
+
 ###############################################################################
 # CLASSES ########################################################### CLASSES #
 ###############################################################################

tests/api/test_superadmin_only.py

@@ -33,6 +33,7 @@
 # TESTS #################################################################################### TESTS #
 ####################################################################################################
 
+
 # Tools ############################################################################################
 def get_token(username: str, client: flask.testing.FlaskClient) -> typing.Dict:
     return tests.UserAuth(tests.USER_CREDENTIALS[username]).token(client)

tests/test_auth.py

@@ -3,6 +3,7 @@
 from dds_web.database import models
 from dds_web import db
 
+
 # verify_token
 def test_verify_token_user_not_exists_after_deletion(client):
     """Log in, delete, log out. Should give exception."""

tests/test_commands.py

@@ -250,6 +250,7 @@ def test_update_uploaded_file_with_log_nonexisting_file(client, runner, fs: Fake
 
 # monitor_usage
 
+
 # usage = 0 --> check log
 def test_monitor_usage_no_usage(client, cli_runner, capfd):
     """If a unit has no uploaded data, there's no need to do the calculations or send email warning."""

tests/test_errors.py

@@ -10,7 +10,6 @@
 
 
 class LoggedHTTPExceptionTest(errors.LoggedHTTPException):
-
     code = http.HTTPStatus.INTERNAL_SERVER_ERROR
 
 

tests/test_files_new.py

@@ -784,7 +784,6 @@ def test_new_file_publickey_wrong_length(client):
 
 
 def test_new_file_salt_wrong_length(client):
-
     project_1 = project_row(project_id="file_testing_project")
     assert project_1
 
@@ -801,7 +800,6 @@ def test_new_file_salt_wrong_length(client):
 
 
 def test_new_file_checksum_wrong_length(client):
-
     project_1 = project_row(project_id="file_testing_project")
     assert project_1
 

tests/test_project_info.py

@@ -25,6 +25,7 @@
 
 # TESTS #################################################################################### TESTS #
 
+
 # Info listing
 def test_list_proj_info_no_token(client):
     """Token required to list project information"""

tests/test_user_delete.py

@@ -65,7 +65,6 @@ def test_del_self_nouser(client):
 def test_del_self(client):
     """Request self deletion."""
     with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send:
-
         response = client.delete(
             tests.DDSEndpoint.USER_DELETE_SELF,
             headers=tests.UserAuth(tests.USER_CREDENTIALS["delete_me_researcher"]).token(client),