Create cronjob for reporting

CHANGELOG.md

@@ -175,3 +175,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe
 
 - Changed support email ([#1324](https://github.com/ScilifelabDataCentre/dds_web/pull/1324))
 - Allow Super Admin login during maintenance ([#1333](https://github.com/ScilifelabDataCentre/dds_web/pull/1333))
+- Cronjob: Get number of units and users for reporting ([#1324](https://github.com/ScilifelabDataCentre/dds_web/pull/1335))

dds_web/config.py

@@ -56,6 +56,7 @@ class Config(object):
     MAIL_USE_TLS = False
     MAIL_USE_SSL = False
     MAIL_DEFAULT_SENDER = ("SciLifeLab DDS", "[email protected]")
+    MAIL_DDS = "[email protected]"
 
     TOKEN_ENDPOINT_ACCESS_LIMIT = "10/hour"
     RATELIMIT_STORAGE_URI = os.environ.get(

dds_web/scheduled_tasks.py

@@ -1,4 +1,5 @@
 from datetime import datetime, timedelta
+import typing
 
 import flask_apscheduler
 import flask
@@ -304,3 +305,79 @@ def quarterly_usage():
             flask.current_app.logger.exception(err)
             db.session.rollback()
             raise
+
+
+# @scheduler.task("interval", id="reporting", seconds=30, misfire_grace_time=1)
+@scheduler.task("cron", id="reporting", day="1", hour=0, minute=1)
+def reporting_units_and_users():
+    """At the start of every month, get number of units and users."""
+    # Imports
+    import csv
+    import flask_mail
+    import flask_sqlalchemy
+    import pathlib
+    from dds_web import errors, utils
+    from dds_web.database.models import User, Unit
+
+    # Get current date
+    current_date: str = utils.timestamp(ts_format="%Y-%m-%d")
+
+    # Location of reporting file
+    reporting_file: pathlib.Path = pathlib.Path("doc/reporting/dds-reporting.csv")
+
+    # Error default
+    error: str = None
+
+    # App context required
+    with scheduler.app.app_context():
+        # Get email address
+        recipient: str = scheduler.app.config.get("MAIL_DDS")
+        default_subject: str = "DDS Unit / User report"
+        default_body: str = f"This email contains the DDS unit- and user statistics. The data was collected on: {current_date}."
+        error_subject: str = f"Error in {default_subject}"
+        error_body: str = "The cronjob 'reporting' experienced issues"
+
+        # Get units and count them
+        units: flask_sqlalchemy.BaseQuery = Unit.query
+        num_units: int = units.count()
+
+        # Count users
+        users: flask_sqlalchemy.BaseQuery = User.query
+        num_users_total: int = users.count()  # All users
+        num_superadmins: int = users.filter_by(type="superadmin").count()  # Super Admins
+        num_unit_users: int = users.filter_by(type="unituser").count()  # Unit Admins / Personnel
+        num_researchers: int = users.filter_by(type="researchuser").count()  # Researchers
+
+        # Verify that sum is correct
+        if sum([num_superadmins, num_unit_users, num_researchers]) != num_users_total:
+            error: str = "Sum of number of users incorrect."
+        # Define csv file and verify that it exists
+        elif not reporting_file.exists():
+            error: str = "Could not find the csv file."
+
+        if error:
+            # Send email about error
+            file_error_msg: flask_mail.Message = flask_mail.Message(
+                subject=error_subject,
+                recipients=[recipient],
+                body=f"{error_body}: {error}",
+            )
+            utils.send_email_with_retry(msg=file_error_msg)
+            raise Exception(error)
+
+        # Add row with new info
+        with reporting_file.open(mode="a") as repfile:
+            writer = csv.writer(repfile)
+            writer.writerow(
+                [current_date, num_units, num_researchers, num_unit_users, num_users_total]
+            )
+
+        # Create email
+        msg: flask_mail.Message = flask_mail.Message(
+            subject=default_subject,
+            recipients=[recipient],
+            body=default_body,
+        )
+        with reporting_file.open(mode="r") as file:  # Attach file
+            msg.attach(filename=reporting_file.name, content_type="text/csv", data=file.read())
+        utils.send_email_with_retry(msg=msg)  # Send

doc/reporting/dds-reporting.csv

@@ -0,0 +1,2 @@
+Date,Units using DDS in production,Researchers,Unit users,Total number of users
+2022-11-30,2,108,11,119

tests/test_scheduled_tasks.py

@@ -1,9 +1,18 @@
 from datetime import timedelta
 
 import flask
+import flask_mail
+import unittest
+import pathlib
+import csv
+from datetime import datetime
+import typing
+import pytest
 
 from unittest import mock
 from unittest.mock import MagicMock
+from pyfakefs.fake_filesystem import FakeFilesystem
+import freezegun
 
 from dds_web import db
 from dds_web.database import models
@@ -14,6 +23,7 @@
     set_expired_to_archived,
     delete_invite,
     quarterly_usage,
+    reporting_units_and_users,
 )
 
 from typing import List
@@ -109,3 +119,89 @@ def test_delete_invite_timestamp_issue(client: flask.testing.FlaskClient) -> Non
 def test_quarterly_usage(client: flask.testing.FlaskClient) -> None:
     """Test the quarterly_usage cron job."""
     quarterly_usage()
+
+
+def test_reporting_units_and_users(client: flask.testing.FlaskClient, fs: FakeFilesystem) -> None:
+    """Test that the reporting is giving correct values."""
+    # Create reporting file
+    reporting_file: pathlib.Path = pathlib.Path("doc/reporting/dds-reporting.csv")
+    assert not fs.exists(reporting_file)
+    fs.create_file(reporting_file)
+    assert fs.exists(reporting_file)
+
+    # Rows for csv
+    header: typing.List = [
+        "Date",
+        "Units using DDS in production",
+        "Researchers",
+        "Unit users",
+        "Total number of users",
+    ]
+    first_row: typing.List = [f"2022-12-10", 2, 108, 11, 119]
+
+    # Fill reporting file with headers and one row
+    with reporting_file.open(mode="a") as csv_file:
+        writer = csv.writer(csv_file)
+        writer.writerow(header)  # Header - Columns
+        writer.writerow(first_row)  # First row
+
+    time_now = datetime(year=2022, month=12, day=10, hour=10, minute=54, second=10)
+    with freezegun.freeze_time(time_now):
+        # Run scheduled job now
+        with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send:
+            reporting_units_and_users()
+            assert mock_mail_send.call_count == 1
+
+    # Check correct numbers
+    num_units: int = models.Unit.query.count()
+    num_users_total: int = models.User.query.count()
+    num_unit_users: int = models.UnitUser.query.count()
+    num_researchers: int = models.ResearchUser.query.count()
+    num_superadmins: int = models.SuperAdmin.query.count()
+
+    # Expected new row:
+    new_row: typing.List = [
+        f"{time_now.year}-{time_now.month}-{time_now.day}",
+        num_units,
+        num_researchers,
+        num_unit_users,
+        num_users_total,
+    ]
+
+    # Check csv file contents
+    with reporting_file.open(mode="r") as result:
+        reader = csv.reader(result)
+        line: int = 0
+        for row in reader:
+            if line == 0:
+                assert row == header
+            elif line == 1:
+                assert row == [str(x) for x in first_row]
+            elif line == 2:
+                assert row == [str(x) for x in new_row]
+            line += 1
+
+    # Delete file to test error
+    fs.remove(reporting_file)
+    assert not fs.exists(reporting_file)
+
+    # Test no file found
+    with freezegun.freeze_time(time_now):
+        # Run scheduled job now
+        with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send:
+            with pytest.raises(Exception) as err:
+                reporting_units_and_users()
+                assert mock_mail_send.call_count == 1
+            assert str(err.value) == "Could not find the csv file."
+
+    # Change total number of users to test error
+    with unittest.mock.patch("dds_web.scheduled_tasks.sum") as mocker:
+        mocker.return_value = num_users_total + 1
+        # Test incorrect number of users
+        with freezegun.freeze_time(time_now):
+            # Run scheduled job now
+            with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send:
+                with pytest.raises(Exception) as err:
+                    reporting_units_and_users()
+                    assert mock_mail_send.call_count == 1
+                assert str(err.value) == "Sum of number of users incorrect."