Skip to content

Commit

Permalink
integrates invenio-cern-sync and jobs
Browse files Browse the repository at this point in the history
- closes #196
  • Loading branch information
ntarocco committed Oct 12, 2024
1 parent 87d7763 commit 4e43cc0
Show file tree
Hide file tree
Showing 19 changed files with 311 additions and 1,426 deletions.
1 change: 1 addition & 0 deletions Pipfile
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ uwsgi = ">=2.0"
uwsgitop = ">=0.11"
uwsgi-tools = ">=1.1.1"
flask-mail = ">=0.9.0,<0.10.0"
# invenio-cern-sync = {git = "https://github.com/cerndocumentserver/invenio-cern-sync.git", ref = "v1.0.0"}

[requires]
python_version = "3.9"
Expand Down
438 changes: 207 additions & 231 deletions Pipfile.lock

Large diffs are not rendered by default.

119 changes: 35 additions & 84 deletions invenio.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ DATACITE_DATACENTER_SYMBOL = ""
# See https://github.com/inveniosoftware/invenio-accounts/blob/master/invenio_accounts/config.py
ACCOUNTS_DEFAULT_USERS_VERIFIED = True # ensure that users are verified by default
ACCOUNTS_DEFAULT_USER_VISIBILITY = "public" # enables users to be searchable for invites
ACCOUNTS_DEFAULT_EMAIL_VISIBILITY = "public"
ACCOUNTS_LOCAL_LOGIN_ENABLED = True # enable local login
PERMANENT_SESSION_LIFETIME = timedelta(days=10)
SECURITY_REGISTERABLE = True # local login: allow users to register
Expand All @@ -201,119 +202,61 @@ SECURITY_SEND_PASSWORD_RESET_EMAIL = False
SECURITY_SEND_PASSWORD_RESET_NOTICE_EMAIL = False
SECURITY_SEND_REGISTER_EMAIL = False

# Invenio-OAuthclient
# ===================
# See https://github.com/inveniosoftware/invenio-oauthclient/blob/master/invenio_oauthclient/config.py
from invenio_oauthclient.contrib.keycloak import KeycloakSettingsHelper
from cds_rdm.oidc import (
cern_info_handler,
cern_info_serializer,
confirm_registration_form,
cern_groups_serializer,
cern_groups_handler,
cern_setup_handler,
)
from urllib.parse import quote

CERN_KEYCLOAK_BASE_URL = os.environ.get("INVENIO_CERN_KEYCLOAK_BASE_URL", "https://keycloak-qa.cern.ch/")

_keycloak_helper = KeycloakSettingsHelper(
title="CERN",
description="CERN SSO authentication",
base_url=CERN_KEYCLOAK_BASE_URL,
realm="cern",
app_key="CERN_APP_CREDENTIALS",
logout_url="{}auth/realms/cern/protocol/openid-connect/logout?redirect_uri={}".format(
CERN_KEYCLOAK_BASE_URL,
quote(os.environ.get("INVENIO_SITE_UI_URL", SITE_UI_URL))
),
)
OAUTHCLIENT_CERN_REALM_URL = _keycloak_helper.realm_url
OAUTHCLIENT_CERN_USER_INFO_URL = _keycloak_helper.user_info_url
OAUTHCLIENT_CERN_VERIFY_EXP = True
OAUTHCLIENT_CERN_VERIFY_AUD = False
OAUTHCLIENT_CERN_USER_INFO_FROM_ENDPOINT = True

handlers = _keycloak_helper.get_handlers()
handlers["signup_handler"] = {
**handlers["signup_handler"],
"info": cern_info_handler,
"info_serializer": cern_info_serializer,
"groups_serializer": cern_groups_serializer,
"groups": cern_groups_handler,
"setup": cern_setup_handler,
}
rest_handlers = _keycloak_helper.get_rest_handlers()
rest_handlers["signup_handler"] = {
**rest_handlers["signup_handler"],
"info": cern_info_handler,
"info_serializer": cern_info_serializer,
"groups_serializer": cern_groups_serializer,
"groups": cern_groups_handler,
"setup": cern_setup_handler,
}

OAUTHCLIENT_SIGNUP_FORM = confirm_registration_form

OAUTH_REMOTE_APP_NAME = "cern"

# Invenio-CERN-Sync/CERN SSO
# ==========================
from invenio_cern_sync.sso import cern_remote_app_name, cern_keycloak
OAUTHCLIENT_REMOTE_APPS = {
OAUTH_REMOTE_APP_NAME: _keycloak_helper.remote_app,
cern_remote_app_name: cern_keycloak.remote_app,
}

CERN_APP_CREDENTIALS = {
"consumer_key": "CHANGE ME",
"consumer_secret": "CHANGE ME",
}
CERN_SYNC_KEYCLOAK_BASE_URL = "https://auth.cern.ch/"
CERN_SYNC_AUTHZ_BASE_URL = "https://authorization-service-api.web.cern.ch/"
INVENIO_CERN_SYNC_KEYCLOAK_BASE_URL = "https://auth.cern.ch/" # set env var when testing


OAUTHCLIENT_CERN_REALM_URL = cern_keycloak.realm_url
OAUTHCLIENT_CERN_USER_INFO_URL = cern_keycloak.user_info_url
OAUTHCLIENT_CERN_VERIFY_EXP = True
OAUTHCLIENT_CERN_VERIFY_AUD = False
OAUTHCLIENT_CERN_USER_INFO_FROM_ENDPOINT = True

from invenio_oauthclient.views.client import auto_redirect_login
ACCOUNTS_LOGIN_VIEW_FUNCTION = auto_redirect_login # autoredirect to external login if enabled
OAUTHCLIENT_AUTO_REDIRECT_TO_EXTERNAL_LOGIN = True # autoredirect to external login
OAUTHCLIENT_AUTO_REDIRECT_TO_EXTERNAL_LOGIN = True # enable autoredirect to external login
ACCOUNTS_LOGIN_VIEW_FUNCTION = auto_redirect_login # autoredirect to external login

from invenio_cern_sync.sso.api import confirm_registration_form
OAUTHCLIENT_SIGNUP_FORM = confirm_registration_form

from invenio_cern_sync.users.profile import CERNUserProfileSchema
ACCOUNTS_USER_PROFILE_SCHEMA = CERNUserProfileSchema()

# Invenio-UserProfiles
# ====================
USERPROFILES_READ_ONLY = False # allow users to change profile info (name, email, etc...)
USERPROFILES_EXTEND_SECURITY_FORMS = True
USERPROFILES_READ_ONLY = True # disable change of user profile
USERPROFILES_EXTEND_SECURITY_FORMS = True # automatically use user's email address as account email

# OAI-PMH
# =======
# See https://github.com/inveniosoftware/invenio-oaiserver/blob/master/invenio_oaiserver/config.py
OAISERVER_ID_PREFIX = "cds-rdm.com"
"""The prefix that will be applied to the generated OAI-PMH ids."""

# Invenio-Search
# ==============
SEARCH_INDEX_PREFIX = "cds-rdm-"

# Celery
# ======
CELERY_BEAT_SCHEDULE = {
**APP_RDM_CELERY_BEAT_SCHEDULE,
"user-sync": {
"task": "cds_rdm.tasks.sync_users",
"schedule": crontab(minute=0, hour=3), # Every day at 03:00 UTC
},
"groups-sync": {
"task": "cds_rdm.tasks.sync_groups",
"schedule": crontab(minute=0, hour=2), # Every day at 02:00 UTC
},
}

###############################################################################
# CDS-RDM configuration
###############################################################################
CDS_SERVICE_ELEMENT_URL = "https://cern.service-now.com/service-portal?id=service_element&name=CDS-Service"

# AUTH/LDAP
CERN_LDAP_URL = "ldap://xldap.cern.ch"
CERN_AUTHORIZATION_SERVICE_API = "https://authorization-service-api-qa.web.cern.ch/api/v1.0/"
CERN_AUTHORIZATION_SERVICE_API_GROUP = "Group"

# Permissions: define who can create new communities
CDS_EMAILS_ALLOW_CREATE_COMMUNITIES = []
CDS_GROUPS_ALLOW_CREATE_COMMUNITIES = []


# Invenio-Files-REST
# ==================
XROOTD_ENABLED = False
Expand Down Expand Up @@ -355,7 +298,7 @@ RDM_NAMESPACES = {
# VocabularyCF(
# name="cern:experiment",
# vocabulary_id="experiments",
# dump_options=True,
# dump_options=True,
# multiple=False,
# ),
# VocabularyCF(
Expand Down Expand Up @@ -401,4 +344,12 @@ RDM_NAMESPACES = {
# ),
# ]
# }
# ]
# ]


RDM_FILES_DEFAULT_QUOTA_SIZE = 50 * 10**9 # 50GB
RDM_FILES_DEFAULT_MAX_FILE_SIZE = 50 * 10**9 # 50GB

# Invenio-Jobs
# ============
JOBS_ADMINISTRATION_ENABLED = True
14 changes: 0 additions & 14 deletions site/cds_rdm/assets/semantic-ui/js/cds_rdm/src/records/detail.js

This file was deleted.

14 changes: 6 additions & 8 deletions site/cds_rdm/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,12 @@
from invenio_pidstore.models import PersistentIdentifier
from invenio_rdm_records.proxies import current_rdm_records_service
from invenio_rdm_records.records.api import RDMDraft, RDMRecord
from invenio_rdm_records.records.models import (
RDMDraftMetadata,
RDMFileDraftMetadata,
RDMFileRecordMetadata,
RDMParentCommunity,
RDMRecordMetadata,
RDMVersionsState,
)
from invenio_rdm_records.records.models import (RDMDraftMetadata,
RDMFileDraftMetadata,
RDMFileRecordMetadata,
RDMParentCommunity,
RDMRecordMetadata,
RDMVersionsState)
from invenio_requests.proxies import current_requests_service
from invenio_requests.records.api import Request
from invenio_requests.records.models import RequestMetadata
Expand Down
3 changes: 2 additions & 1 deletion site/cds_rdm/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@
HTTPKerberosAuth = type("obj", (object,), {})
DISABLED = 3
# use base PyFSFileStorage instead
from invenio_files_rest.storage.pyfs import PyFSFileStorage as BaseFileStorage
from invenio_files_rest.storage.pyfs import \
PyFSFileStorage as BaseFileStorage


class OffloadFileStorage(BaseFileStorage):
Expand Down
43 changes: 43 additions & 0 deletions site/cds_rdm/jobs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2023 CERN.
#
# Invenio is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.

"""Jobs."""

from invenio_jobs.jobs import JobType

from .tasks import sync_groups, sync_users


class SyncUsers(JobType):
"""Sync users with CERN database."""

id = "sync_cern_users"
title = "Sync CERN users"
description = "Sync CERN users with the AuthZ service"

task = sync_users

@classmethod
def build_task_arguments(cls, _, since=None, **kwargs):
"""Build task arguments."""
return {"since": since}


class SyncGroups(JobType):
"""Sync groups with CERN database."""

id = "sync_cern_groups"
title = "Sync CERN groups"
description = "Sync CERN groups with the AuthZ service"

task = sync_groups

@classmethod
def build_task_arguments(cls, _, since=None, **kwargs):
"""Build task arguments."""
return {"since": since}
8 changes: 0 additions & 8 deletions site/cds_rdm/ldap/__init__.py

This file was deleted.

Loading

0 comments on commit 4e43cc0

Please sign in to comment.