From e41c456feaa5464ce95dbfaf4d55161dfbf92138 Mon Sep 17 00:00:00 2001 From: Sidney Richards Date: Tue, 4 Jun 2024 12:40:41 +0200 Subject: [PATCH] [#2526] Allow for 0..n ZGW API groups to be configured This is a first step towards allowing multiple ZGW backends throughout the platform. To maintain backwards compatibility with the current datamodel, we proxy the current service config fields to the new ZGWApiGroupConfig model as we work our way towards making all ZGW client invocations multi-backend aware. --- src/open_inwoner/openzaak/admin.py | 19 +- .../0049_add_multiple_zgw_backends_config.py | 91 ++++++++++ ...igrate_zgw_root_fields_to_multi_backend.py | 55 ++++++ .../migrations/0051_drop_root_zgw_fields.py | 34 ++++ src/open_inwoner/openzaak/models.py | 171 ++++++++++++++---- .../openzaak/tests/test_migrations.py | 68 +++++++ 6 files changed, 395 insertions(+), 43 deletions(-) create mode 100644 src/open_inwoner/openzaak/migrations/0049_add_multiple_zgw_backends_config.py create mode 100644 src/open_inwoner/openzaak/migrations/0050_migrate_zgw_root_fields_to_multi_backend.py create mode 100644 src/open_inwoner/openzaak/migrations/0051_drop_root_zgw_fields.py create mode 100644 src/open_inwoner/openzaak/tests/test_migrations.py diff --git a/src/open_inwoner/openzaak/admin.py b/src/open_inwoner/openzaak/admin.py index f80a1aec70..4d1acee815 100644 --- a/src/open_inwoner/openzaak/admin.py +++ b/src/open_inwoner/openzaak/admin.py @@ -18,23 +18,20 @@ ZaakTypeInformatieObjectTypeConfig, ZaakTypeResultaatTypeConfig, ZaakTypeStatusTypeConfig, + ZGWApiGroupConfig, ) +class ZGWApiGroupConfig(admin.StackedInline): + model = ZGWApiGroupConfig + extra = 0 + + @admin.register(OpenZaakConfig) class OpenZaakConfigAdmin(SingletonModelAdmin): + inlines = [ZGWApiGroupConfig] + fieldsets = ( - ( - None, - { - "fields": [ - "zaak_service", - "catalogi_service", - "document_service", - "form_service", - ] - }, - ), ( "Advanced options", { diff --git a/src/open_inwoner/openzaak/migrations/0049_add_multiple_zgw_backends_config.py b/src/open_inwoner/openzaak/migrations/0049_add_multiple_zgw_backends_config.py new file mode 100644 index 0000000000..7555c710cc --- /dev/null +++ b/src/open_inwoner/openzaak/migrations/0049_add_multiple_zgw_backends_config.py @@ -0,0 +1,91 @@ +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("zgw_consumers", "0019_alter_service_uuid"), + ("openzaak", "0047_delete_statustranslation"), + ] + + operations = [ + migrations.CreateModel( + name="ZGWApiGroupConfig", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "name", + models.CharField( + help_text="A recognisable name for this set of ZGW APIs.", + max_length=255, + verbose_name="name", + ), + ), + ( + "drc_service", + models.ForeignKey( + limit_choices_to={"api_type": "drc"}, + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name="zgwset_drc_config", + to="zgw_consumers.service", + verbose_name="Documenten API", + ), + ), + ( + "form_service", + models.OneToOneField( + limit_choices_to={"api_type": "orc"}, + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name="zgwset_orc_form_config", + to="zgw_consumers.service", + verbose_name="Form API", + ), + ), + ( + "open_zaak_config", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="api_groups", + to="openzaak.openzaakconfig", + ), + ), + ( + "zrc_service", + models.ForeignKey( + limit_choices_to={"api_type": "zrc"}, + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name="zgwset_zrc_config", + to="zgw_consumers.service", + verbose_name="Zaken API", + ), + ), + ( + "ztc_service", + models.ForeignKey( + limit_choices_to={"api_type": "ztc"}, + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name="zgwset_ztc_config", + to="zgw_consumers.service", + verbose_name="Catalogi API", + ), + ), + ], + options={ + "verbose_name": "ZGW API set", + "verbose_name_plural": "ZGW API sets", + }, + ), + ] diff --git a/src/open_inwoner/openzaak/migrations/0050_migrate_zgw_root_fields_to_multi_backend.py b/src/open_inwoner/openzaak/migrations/0050_migrate_zgw_root_fields_to_multi_backend.py new file mode 100644 index 0000000000..85dedb7ee6 --- /dev/null +++ b/src/open_inwoner/openzaak/migrations/0050_migrate_zgw_root_fields_to_multi_backend.py @@ -0,0 +1,55 @@ +import logging +from django.db import migrations + + +logger = logging.getLogger(__name__) + + +def migrate_zgw_service_config_to_default_group_config(apps, schema_editor): + ZGWApiGroupConfig = apps.get_model("openzaak", "ZGWApiGroupConfig") + OpenZaakConfig = apps.get_model("openzaak", "OpenZaakConfig") + + for config in OpenZaakConfig.objects.all(): + # OpenZaakConfig.objects.select_for_update().get(pk=config.pk) # Lock + ZGWApiGroupConfig.objects.create( + name="Migrated default config", + zrc_service=config.zaak_service, + drc_service=config.document_service, + ztc_service=config.catalogi_service, + form_service=config.form_service, + open_zaak_config=config, + ) + + +def reverse_migrate_zgw_service_config_to_default_group_config(apps, schema_editor): + OpenZaakConfig = apps.get_model("openzaak", "OpenZaakConfig") + + for config in OpenZaakConfig.objects.all(): + if config.api_groups.count() == 0: + continue + + if config.api_groups.count() > 1: + logger.warning("Multiple API groups to choose from, picking first") + + group_config = config.api_groups.first() + config.zaak_service = group_config.zrc_service + config.document_service = group_config.drc_service + config.catalogi_service = group_config.ztc_service + config.form_service = group_config.form_service + config.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ("zgw_consumers", "0019_alter_service_uuid"), + ("openzaak", "0049_add_multiple_zgw_backends_config"), + ] + + operations = [ + migrations.RunPython( + migrate_zgw_service_config_to_default_group_config, + reverse_code=reverse_migrate_zgw_service_config_to_default_group_config, + atomic=True, + ), + ] diff --git a/src/open_inwoner/openzaak/migrations/0051_drop_root_zgw_fields.py b/src/open_inwoner/openzaak/migrations/0051_drop_root_zgw_fields.py new file mode 100644 index 0000000000..7e9f1c484f --- /dev/null +++ b/src/open_inwoner/openzaak/migrations/0051_drop_root_zgw_fields.py @@ -0,0 +1,34 @@ +# Generated by Django 4.2.11 on 2024-06-04 10:02 + +import logging +from django.db import migrations + + +logger = logging.getLogger(__name__) + + +class Migration(migrations.Migration): + + dependencies = [ + ("zgw_consumers", "0019_alter_service_uuid"), + ("openzaak", "0050_migrate_zgw_root_fields_to_multi_backend"), + ] + + operations = [ + migrations.RemoveField( + model_name="openzaakconfig", + name="catalogi_service", + ), + migrations.RemoveField( + model_name="openzaakconfig", + name="document_service", + ), + migrations.RemoveField( + model_name="openzaakconfig", + name="form_service", + ), + migrations.RemoveField( + model_name="openzaakconfig", + name="zaak_service", + ), + ] diff --git a/src/open_inwoner/openzaak/models.py b/src/open_inwoner/openzaak/models.py index d72f457d8c..0948a594cd 100644 --- a/src/open_inwoner/openzaak/models.py +++ b/src/open_inwoner/openzaak/models.py @@ -1,6 +1,7 @@ +import warnings from datetime import timedelta -from django.db import models +from django.db import models, transaction from django.db.models import Q, UniqueConstraint from django.utils import timezone from django.utils.translation import gettext_lazy as _ @@ -46,45 +47,160 @@ def generate_default_file_extensions(): ) -class OpenZaakConfig(SingletonModel): - """ - Global configuration and defaults for zaken and catalogi services. - """ +class ZGWApiGroupConfig(models.Model): + """A set of of ZGW service configurations.""" + + open_zaak_config = models.ForeignKey( + "openzaak.OpenZaakConfig", on_delete=models.PROTECT, related_name="api_groups" + ) - zaak_service = models.OneToOneField( + name = models.CharField( + _("name"), + max_length=255, + help_text=_("A recognisable name for this set of ZGW APIs."), + ) + zrc_service = models.ForeignKey( "zgw_consumers.Service", - verbose_name=_("Open Zaak API"), + verbose_name=_("Zaken API"), on_delete=models.PROTECT, limit_choices_to={"api_type": APITypes.zrc}, - related_name="+", - blank=True, + related_name="zgwset_zrc_config", null=True, ) - zaak_max_confidentiality = models.CharField( - max_length=32, - choices=VertrouwelijkheidsAanduidingen.choices, - default=VertrouwelijkheidsAanduidingen.openbaar, - verbose_name=_("Case confidentiality"), - help_text=_("Select maximum confidentiality level of cases"), + drc_service = models.ForeignKey( + "zgw_consumers.Service", + verbose_name=_("Documenten API"), + on_delete=models.PROTECT, + limit_choices_to={"api_type": APITypes.drc}, + related_name="zgwset_drc_config", + null=True, ) - catalogi_service = models.OneToOneField( + ztc_service = models.ForeignKey( "zgw_consumers.Service", verbose_name=_("Catalogi API"), on_delete=models.PROTECT, limit_choices_to={"api_type": APITypes.ztc}, - related_name="+", - blank=True, + related_name="zgwset_ztc_config", null=True, ) - document_service = models.OneToOneField( + form_service = models.OneToOneField( "zgw_consumers.Service", - verbose_name=_("Documents API"), + verbose_name=_("Form API"), on_delete=models.PROTECT, - limit_choices_to={"api_type": APITypes.drc}, - related_name="+", - blank=True, + limit_choices_to={"api_type": APITypes.orc}, + related_name="zgwset_orc_form_config", null=True, ) + + class Meta: + verbose_name = _("ZGW API set") + verbose_name_plural = _("ZGW API sets") + + def __str__(self): + return self.name + + +# This will help us track legacy invocations of the root-level service config +_default_zgw_api_group_deprecation_message = ( + "This usage of `default_zgw_api_group` should be refactored to " + "support multiple ZGW backends" +) + +warnings.filterwarnings( + "once", _default_zgw_api_group_deprecation_message, category=DeprecationWarning +) + + +class OpenZaakConfig(SingletonModel): + """ + Global configuration and defaults for zaken and catalogi services. + """ + + @property + def default_zgw_api_group(self): + # TODO: This is a temporary solution to the new mult-backend + # ZGW configuration to avoid breaking the existing API. + # The *_service fields are proxied through this field to + # avoid having two sources of truth regarding the configured + # ZGW services. The legacy code will simply have a single + # backend configured and retrieve this through the proxy. + + if (api_groups_count := self.api_groups.count()) == 0: + return None + + if api_groups_count > 0: + warnings.warn( + _default_zgw_api_group_deprecation_message, + DeprecationWarning, + ) + + return self.api_groups.first() + + def _set_zgw_service(self, field: str, service): + if self.pk is None: + raise ValueError( + f"Please save your {self.__class__} instance before setting services" + ) + + with transaction.atomic(): + if self.default_zgw_api_group is None: + ZGWApiGroupConfig.objects.create(open_zaak_config=self) + + default_group = self.default_zgw_api_group + setattr(default_group, field, service) + default_group.save() + + return getattr(default_group, field) + + @property + def zaak_service(self): + if self.default_zgw_api_group is None: + return None + + return self.default_zgw_api_group.zrc_service + + @zaak_service.setter + def zaak_service(self, service): + return self._set_zgw_service("zrc_service", service) + + @property + def catalogi_service(self): + if self.default_zgw_api_group is None: + return None + return self.default_zgw_api_group.ztc_service + + @catalogi_service.setter + def catalogi_service(self, service): + return self._set_zgw_service("ztc_service", service) + + @property + def document_service(self): + if self.default_zgw_api_group is None: + return None + return self.default_zgw_api_group.drc_service + + @document_service.setter + def document_service(self, service): + return self._set_zgw_service("drc_service", service) + + @property + def form_service(self): + if self.default_zgw_api_group is None: + return None + + return self.default_zgw_api_group.form_service + + @form_service.setter + def form_service(self, service): + return self._set_zgw_service("form_service", service) + + zaak_max_confidentiality = models.CharField( + max_length=32, + choices=VertrouwelijkheidsAanduidingen.choices, + default=VertrouwelijkheidsAanduidingen.openbaar, + verbose_name=_("Case confidentiality"), + help_text=_("Select maximum confidentiality level of cases"), + ) document_max_confidentiality = models.CharField( max_length=32, choices=VertrouwelijkheidsAanduidingen.choices, @@ -105,15 +221,6 @@ class OpenZaakConfig(SingletonModel): default=generate_default_file_extensions, help_text=_("A list of the allowed file extensions."), ) - form_service = models.OneToOneField( - "zgw_consumers.Service", - verbose_name=_("Form API"), - on_delete=models.PROTECT, - limit_choices_to={"api_type": APITypes.orc}, - related_name="+", - blank=True, - null=True, - ) skip_notification_statustype_informeren = models.BooleanField( verbose_name=_("Use StatusType.informeren workaround"), diff --git a/src/open_inwoner/openzaak/tests/test_migrations.py b/src/open_inwoner/openzaak/tests/test_migrations.py new file mode 100644 index 0000000000..27bb327bb9 --- /dev/null +++ b/src/open_inwoner/openzaak/tests/test_migrations.py @@ -0,0 +1,68 @@ +from zgw_consumers.constants import APITypes + +from open_inwoner.openzaak.tests.factories import ServiceFactory +from open_inwoner.utils.tests.test_migrations import TestMigrations + + +class TestMultiZGWBackendMigrations(TestMigrations): + migrate_from = "0047_delete_statustranslation" + migrate_to = "0051_drop_root_zgw_fields" + app = "openzaak" + + def setUpBeforeMigration(self, apps): + OpenZaakConfig = apps.get_model("openzaak", "OpenZaakConfig") + Service = apps.get_model("zgw_consumers", "Service") + + self.catalogi_service = ServiceFactory(api_type=APITypes.ztc) + self.zaken_service = ServiceFactory(api_type=APITypes.zrc) + self.documenten_service = ServiceFactory(api_type=APITypes.drc) + self.forms_service = ServiceFactory(api_type=APITypes.orc) + + # Note we have to refetch the service instances here: the factories + # create models that differ from the between-migration models + # expected by this OpenZaakConfig + OpenZaakConfig.objects.create( + zaak_service=Service.objects.get(id=self.zaken_service.id), + catalogi_service=Service.objects.get(id=self.catalogi_service.id), + document_service=Service.objects.get(id=self.documenten_service.id), + form_service=Service.objects.get(id=self.forms_service.id), + ) + + def test_migration_0048_to_0051_multi_zgw_backend(self): + ZGWApiGroupConfig = self.apps.get_model("openzaak", "ZGWApiGroupConfig") + OpenZaakConfig = self.apps.get_model("openzaak", "OpenZaakConfig") + + config = OpenZaakConfig.objects.get() + with self.assertRaises( + AttributeError, msg="Root-level service fields should be gone" + ): + for field in ( + "zaak_service", + "catalogi_service", + "document_service", + "form_service", + ): + getattr(config, field) + + value = list( + ZGWApiGroupConfig.objects.values_list( + "zrc_service__id", + "drc_service__id", + "ztc_service__id", + "form_service__id", + ) + ) + expected = [ + ( + self.zaken_service.id, + self.documenten_service.id, + self.catalogi_service.id, + self.forms_service.id, + ) + ] + + self.assertEqual( + value, + expected, + msg="Service config should have been moved to a new ZGWApiGroupConfig", + )