From 8c7624f28cd2d33ac68c941d14ae498fe562765b Mon Sep 17 00:00:00 2001 From: Sidney Richards Date: Wed, 14 Aug 2024 18:17:50 +0200 Subject: [PATCH] expose the ZGW import/export functionality through the admin page --- src/open_inwoner/openzaak/admin.py | 108 ++++++++++++- src/open_inwoner/openzaak/tests/test_admin.py | 144 +++++++++++++++++- .../admin/catalogusconfig_change_list.html | 12 ++ .../admin/import_zgw_export_form.html | 13 ++ 4 files changed, 275 insertions(+), 2 deletions(-) create mode 100644 src/open_inwoner/templates/admin/catalogusconfig_change_list.html create mode 100644 src/open_inwoner/templates/admin/import_zgw_export_form.html diff --git a/src/open_inwoner/openzaak/admin.py b/src/open_inwoner/openzaak/admin.py index 34c56a7483..b4b5e01ce3 100644 --- a/src/open_inwoner/openzaak/admin.py +++ b/src/open_inwoner/openzaak/admin.py @@ -1,14 +1,27 @@ +import datetime +import logging + from django.contrib import admin, messages from django.core.exceptions import ValidationError from django.db.models import BooleanField, Count, ExpressionWrapper, Q -from django.forms import ModelForm, Textarea +from django.forms import Form, ModelForm, Textarea from django.forms.models import BaseInlineFormSet +from django.http import HttpResponseRedirect, StreamingHttpResponse +from django.template.defaultfilters import filesizeformat +from django.template.response import TemplateResponse +from django.urls import path, reverse from django.utils.translation import gettext_lazy as _, ngettext from import_export.admin import ImportExportMixin +from privates.storages import PrivateMediaFileSystemStorage from solo.admin import SingletonModelAdmin from open_inwoner.ckeditor5.widgets import CKEditorWidget +from open_inwoner.openzaak.import_export import ( + CatalogusConfigExport, + CatalogusConfigImport, +) +from open_inwoner.utils.forms import LimitedUploadFileField from .models import ( CatalogusConfig, @@ -22,6 +35,8 @@ ZGWApiGroupConfig, ) +logger = logging.getLogger(__name__) + class ZGWApiGroupConfig(admin.StackedInline): model = ZGWApiGroupConfig @@ -67,8 +82,26 @@ class OpenZaakConfigAdmin(SingletonModelAdmin): ) +class ImportZGWExportFileForm(Form): + MAX_UPLOAD_SIZE = 1024**2 * 25 # 25MB + zgw_export_file = LimitedUploadFileField( + label=_("ZGW export bestand"), + help_text=_( + "Upload a file generated by the export function on the 'Catalogus Config' administration page. The maximum size of this file is %(file_size)s" + % {"file_size": filesizeformat(MAX_UPLOAD_SIZE)} + ), + allow_empty_file=False, + required=True, + max_upload_size=MAX_UPLOAD_SIZE, + min_upload_size=1, + allowed_mime_types=["application/json", "text/plain"], + ) + + @admin.register(CatalogusConfig) class CatalogusConfigAdmin(admin.ModelAdmin): + change_list_template = "admin/catalogusconfig_change_list.html" + list_display = [ "domein", "rsin", @@ -90,6 +123,79 @@ class CatalogusConfigAdmin(admin.ModelAdmin): ordering = ("domein", "rsin") list_filter = ("service",) + actions = ["export_catalogus_configs"] + + def get_urls(self): + urls = super().get_urls() + custom_urls = [ + path( + "import-catalogus-dump/", + self.admin_site.admin_view(self.process_file_view), + name="upload_zgw_import_file", + ), + ] + return custom_urls + urls + + @admin.action(description=_("Export to file")) + def export_catalogus_configs(modeladmin, request, queryset): + export = CatalogusConfigExport.from_catalogus_configs(queryset) + response = StreamingHttpResponse( + export.as_jsonl_iter(), + content_type="application/json", + ) + response[ + "Content-Disposition" + ] = 'attachment; filename="zgw-catalogi-export.json"' + return response + + def process_file_view(self, request): + form = ImportZGWExportFileForm() + + if request.method == "POST": + form = ImportZGWExportFileForm(request.POST, request.FILES) + if form.is_valid(): + storage = PrivateMediaFileSystemStorage() + timestamp = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + target_file_name = f"zgw_import_dump_{timestamp}.json" + storage.save(target_file_name, request.FILES["zgw_export_file"]) + + try: + import_result = ( + CatalogusConfigImport.import_from_jsonl_file_in_django_storage( + target_file_name, + storage, + ) + ) + self.message_user( + request, + _( + "Successfully processed %(num_rows)d items" + % {"num_rows": import_result.total_rows_processed} + ), + messages.SUCCESS, + ) + + return HttpResponseRedirect( + reverse( + "admin:openzaak_catalogusconfig_changelist", + ) + ) + except Exception: + logger.exception("Unable to process ZGW import") + self.message_user( + request, + _( + "We were unable to process your upload. Please regenerate the file and try again." + ), + messages.ERROR, + ) + finally: + storage.delete(target_file_name) + + return TemplateResponse( + request, "admin/import_zgw_export_form.html", {"form": form} + ) + class HasDocNotifyListFilter(admin.SimpleListFilter): title = _("notify document attachment") diff --git a/src/open_inwoner/openzaak/tests/test_admin.py b/src/open_inwoner/openzaak/tests/test_admin.py index 28cd9afe88..5ec4fd7b8f 100644 --- a/src/open_inwoner/openzaak/tests/test_admin.py +++ b/src/open_inwoner/openzaak/tests/test_admin.py @@ -1,12 +1,25 @@ +import json +from unittest import mock + +from django.test import override_settings from django.urls import reverse from django.utils.translation import gettext_lazy as _ +import freezegun from django_webtest import WebTest from maykin_2fa.test import disable_admin_mfa +from privates.storages import PrivateMediaFileSystemStorage +from privates.test import temp_private_root +from webtest import Upload from open_inwoner.accounts.tests.factories import UserFactory -from .factories import ZaakTypeConfigFactory, ZaakTypeInformatieObjectTypeConfigFactory +from .factories import ( + CatalogusConfigFactory, + ServiceFactory, + ZaakTypeConfigFactory, + ZaakTypeInformatieObjectTypeConfigFactory, +) @disable_admin_mfa() @@ -109,3 +122,132 @@ def test_both_can_be_disabled(self): ) self.assertFalse(self.ztc.document_upload_enabled) self.assertFalse(self.ztiotc.document_upload_enabled) + + +@disable_admin_mfa() +@temp_private_root() +@freezegun.freeze_time("2024-08-14 17:50:01") +@override_settings(CELERY_TASK_ALWAYS_EAGER=True) +class TestCatalogusConfigExportAdmin(WebTest): + csrf_checks = False + + def setUp(self): + self.user = UserFactory(is_superuser=True, is_staff=True) + self.service = ServiceFactory(slug="service-1") + self.catalogus = CatalogusConfigFactory( + url="https://foo.maykinmedia.nl", + domein="DM", + rsin="123456789", + service=self.service, + ) + self.mock_file = Upload("dump.json", b"foobar", "application/json") + + def test_export_action_returns_correct_export(self): + response = self.app.post( + reverse( + "admin:openzaak_catalogusconfig_changelist", + ), + { + "action": "export_catalogus_configs", + "_selected_action": [self.catalogus.id], + }, + user=self.user, + ) + + self.assertEqual( + response.content_disposition, + 'attachment; filename="zgw-catalogi-export.json"', + ) + self.assertEqual(response.status_code, 200) + self.assertEqual( + [json.loads(row) for row in response.text.split("\n")[:-1]], + [ + { + "model": "openzaak.catalogusconfig", + "fields": { + "url": "https://foo.maykinmedia.nl", + "domein": "DM", + "rsin": "123456789", + "service": ["service-1"], + }, + }, + ], + msg="Response should be valid JSONL matching the input object", + ) + + @mock.patch( + "open_inwoner.openzaak.import_export.CatalogusConfigImport.import_from_jsonl_file_in_django_storage" + ) + def test_import_flow_reports_success(self, m) -> None: + form = self.app.get( + reverse( + "admin:upload_zgw_import_file", + ), + user=self.user, + ).form + form["zgw_export_file"] = self.mock_file + + response = form.submit().follow() + + self.assertEqual(m.call_count, 1) + self.assertEqual(m.call_args[0][0], "zgw_import_dump_2024-08-14-17-50-01.json") + self.assertTrue(isinstance(m.call_args[0][1], PrivateMediaFileSystemStorage)) + + messages = [str(msg) for msg in response.context["messages"]] + self.assertEqual( + messages, + [_("Successfully processed 1 items")], + ) + self.assertFalse( + PrivateMediaFileSystemStorage().exists( + "zgw_import_dump_2024-08-14-17-50-01.jsonl" + ), + msg="File should always be deleted regardless of success or failure", + ) + self.assertEqual( + response.request.path, + reverse( + "admin:openzaak_catalogusconfig_changelist", + ), + ) + + @mock.patch( + "open_inwoner.openzaak.import_export.CatalogusConfigImport.import_from_jsonl_file_in_django_storage" + ) + def test_import_flow_errors_reports_failure_to_user(self, m) -> None: + m.side_effect = Exception("something went wrong") + form = self.app.get( + reverse( + "admin:upload_zgw_import_file", + ), + user=self.user, + ).form + form["zgw_export_file"] = self.mock_file + + response = form.submit() + + self.assertEqual(m.call_count, 1) + self.assertEqual(m.call_args[0][0], "zgw_import_dump_2024-08-14-17-50-01.json") + self.assertTrue(isinstance(m.call_args[0][1], PrivateMediaFileSystemStorage)) + + messages = [str(msg) for msg in response.context["messages"]] + self.assertEqual( + messages, + [ + _( + "We were unable to process your upload. Please regenerate the file and try again." + ) + ], + ) + self.assertFalse( + PrivateMediaFileSystemStorage().exists( + "zgw_import_dump_2024-08-14-17-50-01.json" + ), + msg="File should always be deleted regardless of success or failure", + ) + self.assertEqual( + response.request.path, + reverse( + "admin:upload_zgw_import_file", + ), + ) diff --git a/src/open_inwoner/templates/admin/catalogusconfig_change_list.html b/src/open_inwoner/templates/admin/catalogusconfig_change_list.html new file mode 100644 index 0000000000..41eb9eac60 --- /dev/null +++ b/src/open_inwoner/templates/admin/catalogusconfig_change_list.html @@ -0,0 +1,12 @@ +{% extends "admin/change_list.html" %} +{% load i18n admin_urls %} + +{% block object-tools-items %} +{{ block.super }} +
  • + + {% trans "Import from file" %} + +
    • +{% endblock %} + diff --git a/src/open_inwoner/templates/admin/import_zgw_export_form.html b/src/open_inwoner/templates/admin/import_zgw_export_form.html new file mode 100644 index 0000000000..dacdcd08e1 --- /dev/null +++ b/src/open_inwoner/templates/admin/import_zgw_export_form.html @@ -0,0 +1,13 @@ +{% extends "admin/base_site.html" %} +{% load i18n admin_urls %} + +{% block content %} +
    +
    + {% csrf_token %} + {{ form.as_div }} + +
    +
    +{% endblock %} +