integrates invenio-cern-sync and jobs

- closes #196

Pipfile

@@ -16,6 +16,7 @@ uwsgi = ">=2.0"
 uwsgitop = ">=0.11"
 uwsgi-tools = ">=1.1.1"
 flask-mail = ">=0.9.0,<0.10.0"
+# invenio-cern-sync = {git = "https://github.com/cerndocumentserver/invenio-cern-sync.git", ref = "v1.0.0"}
 
 [requires]
 python_version = "3.9"

invenio.cfg

@@ -186,6 +186,7 @@ DATACITE_DATACENTER_SYMBOL = ""
 # See https://github.com/inveniosoftware/invenio-accounts/blob/master/invenio_accounts/config.py
 ACCOUNTS_DEFAULT_USERS_VERIFIED = True # ensure that users are verified by default
 ACCOUNTS_DEFAULT_USER_VISIBILITY = "public" # enables users to be searchable for invites
+ACCOUNTS_DEFAULT_EMAIL_VISIBILITY = "public"
 ACCOUNTS_LOCAL_LOGIN_ENABLED = True  # enable local login
 PERMANENT_SESSION_LIFETIME = timedelta(days=10)
 SECURITY_REGISTERABLE = True  # local login: allow users to register
@@ -201,119 +202,61 @@ SECURITY_SEND_PASSWORD_RESET_EMAIL = False
 SECURITY_SEND_PASSWORD_RESET_NOTICE_EMAIL = False
 SECURITY_SEND_REGISTER_EMAIL = False
 
-# Invenio-OAuthclient
-# ===================
-# See https://github.com/inveniosoftware/invenio-oauthclient/blob/master/invenio_oauthclient/config.py
-from invenio_oauthclient.contrib.keycloak import KeycloakSettingsHelper
-from cds_rdm.oidc import (
-    cern_info_handler,
-    cern_info_serializer,
-    confirm_registration_form,
-    cern_groups_serializer,
-    cern_groups_handler,
-    cern_setup_handler,
-)
-from urllib.parse import quote
-
-CERN_KEYCLOAK_BASE_URL = os.environ.get("INVENIO_CERN_KEYCLOAK_BASE_URL", "https://keycloak-qa.cern.ch/")
-
-_keycloak_helper = KeycloakSettingsHelper(
-    title="CERN",
-    description="CERN SSO authentication",
-    base_url=CERN_KEYCLOAK_BASE_URL,
-    realm="cern",
-    app_key="CERN_APP_CREDENTIALS",
-    logout_url="{}auth/realms/cern/protocol/openid-connect/logout?redirect_uri={}".format(
-        CERN_KEYCLOAK_BASE_URL,
-        quote(os.environ.get("INVENIO_SITE_UI_URL", SITE_UI_URL))
-    ),
-)
-OAUTHCLIENT_CERN_REALM_URL = _keycloak_helper.realm_url
-OAUTHCLIENT_CERN_USER_INFO_URL = _keycloak_helper.user_info_url
-OAUTHCLIENT_CERN_VERIFY_EXP = True
-OAUTHCLIENT_CERN_VERIFY_AUD = False
-OAUTHCLIENT_CERN_USER_INFO_FROM_ENDPOINT = True
-
-handlers = _keycloak_helper.get_handlers()
-handlers["signup_handler"] = {
-    **handlers["signup_handler"],
-    "info": cern_info_handler,
-    "info_serializer": cern_info_serializer,
-    "groups_serializer": cern_groups_serializer,
-    "groups": cern_groups_handler,
-    "setup": cern_setup_handler,
-}
-rest_handlers = _keycloak_helper.get_rest_handlers()
-rest_handlers["signup_handler"] = {
-    **rest_handlers["signup_handler"],
-    "info": cern_info_handler,
-    "info_serializer": cern_info_serializer,
-    "groups_serializer": cern_groups_serializer,
-    "groups": cern_groups_handler,
-    "setup": cern_setup_handler,
-}
-
-OAUTHCLIENT_SIGNUP_FORM = confirm_registration_form
-
-OAUTH_REMOTE_APP_NAME = "cern"
-
+# Invenio-CERN-Sync/CERN SSO
+# ==========================
+from invenio_cern_sync.sso import cern_remote_app_name, cern_keycloak
 OAUTHCLIENT_REMOTE_APPS = {
-    OAUTH_REMOTE_APP_NAME: _keycloak_helper.remote_app,
+    cern_remote_app_name: cern_keycloak.remote_app,
 }
 
 CERN_APP_CREDENTIALS = {
     "consumer_key": "CHANGE ME",
     "consumer_secret": "CHANGE ME",
 }
+CERN_SYNC_KEYCLOAK_BASE_URL = "https://auth.cern.ch/"
+CERN_SYNC_AUTHZ_BASE_URL = "https://authorization-service-api.web.cern.ch/"
+INVENIO_CERN_SYNC_KEYCLOAK_BASE_URL = "https://auth.cern.ch/"  # set env var when testing
+
+
+OAUTHCLIENT_CERN_REALM_URL = cern_keycloak.realm_url
+OAUTHCLIENT_CERN_USER_INFO_URL = cern_keycloak.user_info_url
+OAUTHCLIENT_CERN_VERIFY_EXP = True
+OAUTHCLIENT_CERN_VERIFY_AUD = False
+OAUTHCLIENT_CERN_USER_INFO_FROM_ENDPOINT = True
 
 from invenio_oauthclient.views.client import auto_redirect_login
-ACCOUNTS_LOGIN_VIEW_FUNCTION = auto_redirect_login  # autoredirect to external login if enabled
-OAUTHCLIENT_AUTO_REDIRECT_TO_EXTERNAL_LOGIN = True  # autoredirect to external login
+OAUTHCLIENT_AUTO_REDIRECT_TO_EXTERNAL_LOGIN = True  # enable autoredirect to external login
+ACCOUNTS_LOGIN_VIEW_FUNCTION = auto_redirect_login  # autoredirect to external login
+
+from invenio_cern_sync.sso.api import confirm_registration_form
+OAUTHCLIENT_SIGNUP_FORM = confirm_registration_form
+
+from invenio_cern_sync.users.profile import CERNUserProfileSchema
+ACCOUNTS_USER_PROFILE_SCHEMA = CERNUserProfileSchema()
 
 # Invenio-UserProfiles
 # ====================
-USERPROFILES_READ_ONLY = False  # allow users to change profile info (name, email, etc...)
-USERPROFILES_EXTEND_SECURITY_FORMS = True
+USERPROFILES_READ_ONLY = True  # disable change of user profile
+USERPROFILES_EXTEND_SECURITY_FORMS = True  # automatically use user's email address as account email
 
 # OAI-PMH
 # =======
-# See https://github.com/inveniosoftware/invenio-oaiserver/blob/master/invenio_oaiserver/config.py
 OAISERVER_ID_PREFIX = "cds-rdm.com"
 """The prefix that will be applied to the generated OAI-PMH ids."""
 
 # Invenio-Search
 # ==============
 SEARCH_INDEX_PREFIX = "cds-rdm-"
 
-# Celery
-# ======
-CELERY_BEAT_SCHEDULE = {
-    **APP_RDM_CELERY_BEAT_SCHEDULE,
-    "user-sync": {
-        "task": "cds_rdm.tasks.sync_users",
-        "schedule": crontab(minute=0, hour=3),  # Every day at 03:00 UTC
-    },
-    "groups-sync": {
-        "task": "cds_rdm.tasks.sync_groups",
-        "schedule": crontab(minute=0, hour=2),  # Every day at 02:00 UTC
-    },
-}
-
 ###############################################################################
 # CDS-RDM configuration
 ###############################################################################
 CDS_SERVICE_ELEMENT_URL = "https://cern.service-now.com/service-portal?id=service_element&name=CDS-Service"
 
-# AUTH/LDAP
-CERN_LDAP_URL = "ldap://xldap.cern.ch"
-CERN_AUTHORIZATION_SERVICE_API = "https://authorization-service-api-qa.web.cern.ch/api/v1.0/"
-CERN_AUTHORIZATION_SERVICE_API_GROUP = "Group"
-
 # Permissions: define who can create new communities
 CDS_EMAILS_ALLOW_CREATE_COMMUNITIES = []
 CDS_GROUPS_ALLOW_CREATE_COMMUNITIES = []
 
-
 # Invenio-Files-REST
 # ==================
 XROOTD_ENABLED = False
@@ -355,7 +298,7 @@ RDM_NAMESPACES = {
 #     VocabularyCF(
 #         name="cern:experiment",
 #         vocabulary_id="experiments",
-#         dump_options=True, 
+#         dump_options=True,
 #         multiple=False,
 #     ),
 #     VocabularyCF(
@@ -401,4 +344,12 @@ RDM_NAMESPACES = {
 #             ),
 #         ]
 #     }
-# ]
+# ]
+
+
+RDM_FILES_DEFAULT_QUOTA_SIZE = 50 * 10**9     # 50GB
+RDM_FILES_DEFAULT_MAX_FILE_SIZE = 50 * 10**9  # 50GB
+
+# Invenio-Jobs
+# ============
+JOBS_ADMINISTRATION_ENABLED = True

site/cds_rdm/cli.py

@@ -15,14 +15,12 @@
 from invenio_pidstore.models import PersistentIdentifier
 from invenio_rdm_records.proxies import current_rdm_records_service
 from invenio_rdm_records.records.api import RDMDraft, RDMRecord
-from invenio_rdm_records.records.models import (
-    RDMDraftMetadata,
-    RDMFileDraftMetadata,
-    RDMFileRecordMetadata,
-    RDMParentCommunity,
-    RDMRecordMetadata,
-    RDMVersionsState,
-)
+from invenio_rdm_records.records.models import (RDMDraftMetadata,
+                                                RDMFileDraftMetadata,
+                                                RDMFileRecordMetadata,
+                                                RDMParentCommunity,
+                                                RDMRecordMetadata,
+                                                RDMVersionsState)
 from invenio_requests.proxies import current_requests_service
 from invenio_requests.records.api import Request
 from invenio_requests.records.models import RequestMetadata

site/cds_rdm/files.py

@@ -26,7 +26,8 @@
     HTTPKerberosAuth = type("obj", (object,), {})
     DISABLED = 3
     # use base PyFSFileStorage instead
-    from invenio_files_rest.storage.pyfs import PyFSFileStorage as BaseFileStorage
+    from invenio_files_rest.storage.pyfs import \
+        PyFSFileStorage as BaseFileStorage
 
 
 class OffloadFileStorage(BaseFileStorage):

site/cds_rdm/jobs.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2023 CERN.
+#
+# Invenio is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""Jobs."""
+
+from invenio_jobs.jobs import JobType
+
+from .tasks import sync_groups, sync_users
+
+
+class SyncUsers(JobType):
+    """Sync users with CERN database."""
+
+    id = "sync_cern_users"
+    title = "Sync CERN users"
+    description = "Sync CERN users with the AuthZ service"
+
+    task = sync_users
+
+    @classmethod
+    def build_task_arguments(cls, _, since=None, **kwargs):
+        """Build task arguments."""
+        return {"since": since}
+
+
+class SyncGroups(JobType):
+    """Sync groups with CERN database."""
+
+    id = "sync_cern_groups"
+    title = "Sync CERN groups"
+    description = "Sync CERN groups with the AuthZ service"
+
+    task = sync_groups
+
+    @classmethod
+    def build_task_arguments(cls, _, since=None, **kwargs):
+        """Build task arguments."""
+        return {"since": since}